markdown_splitter.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 211,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pathlib import Path\n",
    "\n",
    "VERSIONS_DIR = \"../data/platform-docs-versions/\"\n",
    "\n",
    "from langchain_community.document_loaders import UnstructuredMarkdownLoader, DirectoryLoader, TextLoader\n",
    "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
    "\n",
    "import os\n",
    "\n",
    "def list_folders(directory):\n",
    "    # Ensure the directory exists\n",
    "    if not os.path.isdir(directory):\n",
    "        print(f\"Error: '{directory}' is not a directory.\")\n",
    "        return\n",
    "\n",
    "    # List all items in the directory\n",
    "    items = os.listdir(directory)\n",
    "\n",
    "    # Filter out only the directories\n",
    "    return [item for item in items if os.path.isdir(os.path.join(directory, item))]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "This is a document\n",
      "\n",
      "# Heading 1\n",
      "\n",
      "Some text under heading 1.\n",
      "\n",
      "## Heading 2\n",
      "\n",
      "Some text under heading 2.\n",
      "\n",
      "## Another Heading | With Pipe\n",
      "\n",
      "# heading\n",
      "\n",
      "## heading\n",
      "\n",
      "### heading\n",
      "\n",
      "| Table |\n",
      "|-------|\n",
      "| Value |\n",
      "\n"
     ]
    }
   ],
   "source": [
    "import re\n",
    "\n",
    "def replace_alternate_headings(text):\n",
    "    lines = text.split('\\n')\n",
    "    new_lines = []\n",
    "    i = 0\n",
    "    while i < len(lines):\n",
    "        if i + 1 < len(lines) and lines[i].strip():\n",
    "            # '=' underline headings\n",
    "            if re.match(r'^=+$', lines[i + 1]):\n",
    "                new_lines.append(f'# {lines[i]}')\n",
    "                i += 2\n",
    "                continue\n",
    "            # '-' underline headings\n",
    "            elif re.match(r'^-+$', lines[i + 1]):\n",
    "                new_lines.append(f'## {lines[i]}')\n",
    "                i += 2\n",
    "                continue\n",
    "        new_lines.append(lines[i])\n",
    "        i += 1\n",
    "    return '\\n'.join(new_lines)\n",
    "\n",
    "markdown_text = \"\"\"This is a document\n",
    "\n",
    "Heading 1\n",
    "=========\n",
    "\n",
    "Some text under heading 1.\n",
    "\n",
    "Heading 2\n",
    "---------\n",
    "\n",
    "Some text under heading 2.\n",
    "\n",
    "Another Heading | With Pipe\n",
    "----------------------------\n",
    "\n",
    "# heading\n",
    "\n",
    "## heading\n",
    "\n",
    "### heading\n",
    "\n",
    "| Table |\n",
    "|-------|\n",
    "| Value |\n",
    "\"\"\"\n",
    "\n",
    "new_markdown = replace_alternate_headings(markdown_text)\n",
    "print(new_markdown)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['']"
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\"\".split('\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "metadata": {},
   "outputs": [],
   "source": [
    "def find_substring_line(content, substring):\n",
    "    all_lines = content.split('\\n')\n",
    "\n",
    "    substring_lines = substring.split('\\n')\n",
    "    for line_num, line in enumerate(all_lines):\n",
    "        if substring_lines[0].strip(' ') == line.strip(' '):\n",
    "            line_end = line_num + len(substring_lines)\n",
    "            return line_num, line_end\n",
    "    return None, None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain.text_splitter import MarkdownHeaderTextSplitter\n",
    "\n",
    "def process_document(document, merge_h3=False):\n",
    "    if merge_h3:\n",
    "        headers_to_split = [(\"#\", \"H1\"),(\"##\", \"H2\")]\n",
    "    \n",
    "    headers_to_split = [(\"#\", \"H1\"),(\"##\", \"H2\"), ('###', \"H3\")]\n",
    "    new_markdown = replace_alternate_headings(document.page_content)\n",
    "\n",
    "    markdown_splitter = MarkdownHeaderTextSplitter(\n",
    "        headers_to_split_on=headers_to_split, strip_headers=False\n",
    "    )\n",
    "    md_header_splits = markdown_splitter.split_text(new_markdown)\n",
    "    filtered_header_splits = []\n",
    "    threshold = 6000 # length of allowed threshold\n",
    "\n",
    "    metadata = item = None\n",
    "\n",
    "    for d in md_header_splits:\n",
    "        # skip if empty\n",
    "        if len(d.page_content) <= 50:\n",
    "            continue\n",
    "\n",
    "        # add to previous if short and have exactly the same metadata\n",
    "        if metadata == d.metadata.items() and item is not None and len(item) + d.page_content < threshold:\n",
    "            item = item + '\\n' + d.page_content\n",
    "            filtered_header_splits[-1][0] = item\n",
    "            continue\n",
    "        \n",
    "        # create a new split\n",
    "        source = \"Source document: \" + document.metadata['source']\n",
    "        metadata = d.metadata.items()\n",
    "        hierarchy = \"Paragraph location: \" + '\\n'\n",
    "        for k, v in metadata:\n",
    "            if k.startswith('H'):\n",
    "                hierarchy = hierarchy + f\"\\t{v}\\n\"\n",
    "\n",
    "        content = \"Content: \\n\" + d.page_content  + '\\n'\n",
    "        \n",
    "        item = f\"{source}\\n{hierarchy}\\n{content}\" \n",
    "\n",
    "        if new_markdown is not None and d.page_content is not None:\n",
    "            line_start, line_end = find_substring_line(new_markdown, d.page_content)\n",
    "            metadata=(line_start, line_end)\n",
    "        \n",
    "        # if line_start is not None:\n",
    "        #     print(f\"The substring '{d.page_content[:100]}' starts at line {line_start}, end at line {line_end}.\")\n",
    "        # else:\n",
    "        #     print(f\"The substring '{d.page_content[:100]}' was not found in the file.\")       \n",
    "\n",
    "        filtered_header_splits.append((item, metadata))\n",
    "\n",
    "\n",
    "    return filtered_header_splits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 168,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Document(page_content='## Ad Repository API\\n\\nJanuary 2024\\n\\n###### Contents\\n\\n* Getting Started................\\n* 3 Versioning\\n* 3 Usability\\n* 3 Get a list of app and developer names\\n* 4 Request example\\n* 4 Request parameters\\n* 4 Response example\\n* 5 Response properties\\n* 6 Get a list of countries or regions\\n* 7 Response example\\n* 7 Response properties\\n* 8 Get a list of ads\\n* 9 Request example\\n* 9 Request parameters\\n* 9 Response example\\n* 10 Response properties\\n* 11 Get ad variations\\n* 14 Request parameters\\n* 14 Response example\\n* 15 Response properties\\n* 17 Get advertising-restrictions\\n* 20 Request example\\n* 20 Request parameters\\n* 20 Response example\\n* 21 Response properties\\n* 22 Response properties\\n* 22 Changelog\\n\\n## Getting Started\\n\\nUse the Ad Repository API to look up Apple-delivered advertising on the App Store in select European countries and regions as well as information about qualifying advertising restrictions. You can manage API calls programmatically using API tools and programs of your choice.\\n\\n### Versioning\\n\\nThe current version of the API is v1.\\n\\n### Usability\\n\\n* Communication with the web service must use HTTPS.\\n* RSQL query patterns are supported in some endpoints.\\n* The API doesn\\'t accept headers with */*. Disallow Accept */* in your cURL commands.\\n* API responses will have a header Content-Type: application/json.\\n\\n## Get a list of app and developer names\\n\\nGET [https://adrepository.apple.com/api/v1/ad-repository-entities](https://adrepository.apple.com/api/v1/ad-repository-entities)\\n\\nUse this endpoint to return list of app and developer names based on custom query parameter input. A developer name refers to the developer as identified on the App Store, which is the entity on whose behalf the subject advertising was presented. Only apps that have been advertised in countries or regions in the European Union in which Apple-delivered advertising is available on the App Store are searchable.\\n\\n### Request example\\n\\nGET -/api/v1/ad-repository-entities?name={URL encoded search input text}  &types==APP  &offset=0  &limit=10\\n\\n### Request parameters\\n\\n\\\\begin{tabular}{|l|c|c|c|} \\\\hline\\n**Parameter** & **Type** & **Required** & **Description** \\\\\\\\ \\\\hline name & string & yes & Custom query for app and developer names with a minimum of 2 characters required. \\\\\\\\ \\\\hline  & & & Values are: \\\\\\\\ types & string & no & DEVELOPER \\\\\\\\  & & & APP, DEVELOPER (The default value or blank which returns all types.) \\\\\\\\ \\\\hline offset & integer & no & The offset pagination that limits the number of returned records. The default is 0. \\\\\\\\ \\\\hline limit & integer & no & The maximum number of entries to return per page. The default is 50. \\\\\\\\ \\\\hline \\\\end{tabular}\\n\\nResponse example The AdRepositoryEntity object is returned:\\n\\n{  \"data\": [  \"id\": 1582276305,  \"name\": \"TripTrek\",  \"type\": \"App\"  },  {  \"id\": 819221,  \"name\": \"TripTrek, Inc.\",  \"type\": \"DEVELOPER\"  },  \"pagination\": {  \"totalResults\": 2,  \"startIndex\": 0,  \"itemsPerPage\": 15  } } ```\\n\\nAd Repository API January 2024Response properties\\n\\n**Parameter** & **Type** & **Description**\\n\\nid & integer & A unique identifier. If the type is APP, the id will be an appId. If the type is DEVELOPER, the id is a developerId. name & string & The entity name on whose behalf the advertising was presented. type & string & Values are APP, DEVELOPER\\n\\npagination & integer & Returned pagination results. totalResults: The total number of entries that return for the operation. startIndex: Specifies the position of the first element in the results. The default is zero. itemsPerPage: Specifies how many items is displayed per page.\\n\\n### Get a list of countries or regions\\n\\nUse this endpoint to get a list of countries or regions in the European Union in which Apple-delivered advertising is available on the App Store. Use the country codes returned in the response in the Get a list of ads call.\\n\\n#### Request example\\n\\nGET [https://adrepository.apple.com/api/v1/countries-or-regions](https://adrepository.apple.com/api/v1/countries-or-regions)\\n\\n#### Response example\\n\\n{  \"data\": [  {  \"name\": \"Austria\",  \"code\": \"AT\"  },  {  \"name\": \"Belgium\",  \"code\": \"BE\"  },  {  \"name\": \"Croatia\",  \"code\": \"HR\"  },  {  \"name\": \"Czech Republic\",  \"code\": \"CZ\"  },  {  \"name\": \"Denmark\",  \"code\": \"DK\"  },  {  \"name\": \"Finland\",  \"code\": \"FI\"  },  {  \"name\": \"France\",  \"code\": \"FR\"  },  {  \"name\": \"Germany\",  \"code\": \"DE\"  },  {  \"name\": \"Greece\",  \"code\": \"GR\"  },  {  \"name\": \"Hungary\",  \"code\": \"HU\"  },  {  \"name\": \"Ireland\",  \"code\": \"IE\"  },  {  \"name\": \"Italy\",\"code\":\"IT\"  },  {  \"name\":\"Netherlands\",  \"code\":\"NL\"  },  {  \"name\":\"Poland\",  \"code\":\"PL\"  },  {  \"name\":\"Portugal\",  \"code\":\"PT\"  },  {  \"name\":\"Romania\",  \"code\":\"RO\"  },  {  \"name\":\"Spain\",  \"code\":\"ES\"  },  {  \"name\":\"Sweden\",  \"code\":\"SE\"  }  } }\\n\\nResponse properties\\n\\nParameterType\\n\\nDescription\\n\\nnamestring\\n\\nThe name of the country or region in the European Union in which Apple-delivered advertising is available on the App Store. For example: \"Sweden\".\\n\\ncodestring\\n\\nThe ISO 3166-1 alpha-2 two-letter country code of the country or region in the European Union in which Apple-delivered advertising is available on the App Store. For example: ES, SE, DE.\\n\\n## Get a list of ads\\n\\nUse this endpoint to return ad metadata and an adId to use in the Get Ad Variations call.\\n\\nGET [https://adrepository.apple.com/api/v1/ad-repository-ads](https://adrepository.apple.com/api/v1/ad-repository-ads)\\n\\n### Request example\\n\\n* Use ql= URL encoded RSQL filter to allow special characters.\\n* You can have both id or countryOrRegion or either in a request. If id is used then type must also be present.\\n\\nGET -/api/v1/ad-repository-ads ?ql=  type==APP;  id==1582276305;  countryOrRegion=in=(FR,DE);  datePreset==LAST_90_DAYS; &offset=0  &limit=50\\n\\n### Request parameters\\n\\n\\\\begin{tabular}{|l|c|c|c|} \\\\hline\\n**Parameter** & **Type** & **Required** & **Description** \\\\\\\\ \\\\hline type & string & yes & Values are APP, DEVELOPER. \\\\\\\\ \\\\hline id & string & yes & A unique identifier. If the type is APP, the id is an appl. If the type is DEVELOPER, the id is a developerId. \\\\\\\\ \\\\hline countryOrRegion & string & yes & A country or region in the European Union in which Apple-delivered advertising is available on the App Store, expressed in ISO 3166-1 alpha-2 format. For example: ES, SE, DE. \\\\\\\\ \\\\hline datePreset & string & no & The date range of the request. Values are \\\\\\\\  & & & Last_90_DAYS,LAST_YEAR. \\\\\\\\  & & & The default isLAST_90_DAYS. \\\\\\\\ \\\\hline limit & integer & no & The maximum number of entries to return per page. The default is 50. \\\\\\\\ \\\\hline offset & integer & no & The offset pagination that limits the number of returned records. The default is 0. \\\\\\\\ \\\\hline \\\\end{tabular}\\n\\n### Response example\\n\\nThe AdRepositoryAd object is returned.\\n\\n{  \"data\": [  {  \"adId\": \"76324182\",  \"appId\": 1582276305,  \"appName\": \"TripTrek - travel app\",  \"developerId\": 819221,  \"developerName\": \"Apple Inc\",  \"legalName\": \"Apple Inc.\",  \"placement\": \"APPSTORE_SEARCH_RESULT\",  \"format\": \"Icon Ad\",  \"countryOrRegion\": \"es\",  \"audienceRefinement\": {  \"ageTarget\": \"false\",  \"genderTarget\": \"false\",  \"locationTarget\": \"true\",  \"customerTypeTarget\": \"false\"  },  \"firstImpressionDate\": \"2022-08-25\",  \"lastImpressionDate\": \"2023-04-01\",  \"defaultLanguageTag\": \"es-ES\",  \"defaultLanguageDisplayName\": \"Spanish (Spain)\",  \"defaultPreviewDevice\": \"iPhone_6.5\",  \"adBanner\": {  \"joeColor\": \"b:ff4c00p:04@404s:17lc@1t:361303q:452501\",  \"iconPictureUrl\": \"\",  \"subtitle\": \"Planifique sus viajes con confianza.\",  \"primaryCategory\": \"viajar\",  \"inAppurchases\": \"false\",  \"promotionalText\": \"Descripcion de la aplicacion mostrada en la App Store.\",  \"editorialBadge\": \"false\"  \"shortDescription\": \"una descripcion de la aplicacion asociada con el anuncio.\"  },  \"adAssets\": [  {  \"videoUrl\": \"https://...x.png\",  \"pictureUrl\": \"null\",  \"order\": 1,  \"height\": 1920,  \"width\": 1080,  \"orientation\": \"LANDSCAPE\"  }  ],  \"appIconVariations\": [],  \"adLocaleVariations\": []  }  ],  \"dataStartDate\": \"2023-01-01\",  \"dataEndDate\": \"2023-04-01\",  \"pagination\": {  \"totalResults\": 330,  \"startIndex\": 0,  \"itemsPerPage\": 15  }  }\\n\\n## Response properties\\n\\n\\\\begin{tabular}{|p{113.8pt}|p{113.8pt}|p{113.8pt}|} \\\\hline\\n**Parameter** & **Type** & **Description** \\\\\\\\ \\\\hline adId & string & A unique ad identifier. Use adId as a resource in the ad variations call. \\\\\\\\ \\\\hline appId & integer & The unique identifier of the app associated with an ad. \\\\\\\\ \\\\hline appName & string & The name of the app associated with an ad. \\\\\\\\ \\\\hline developerId & integer & The unique identifier of the developer identified on the App Store. \\\\\\\\ \\\\hline developerName & string & The developer name refers to the developer as identified on the App Store which is the entity on whose behalf the subject advertising was presented. In some instances, the developerName may be different from the legalName field for the same entity. \\\\\\\\ \\\\hline legalName & string & The legal name of the person or entity identified under the seller field on the App Store. In some instances, the legalName may be different from the developerName for the same entity. \\\\\\\\ \\\\hline placement & string & The placement of the ad on the App Store. Values are APPSTORE_SEARCH_RESULTS, APPSTORE_TODAY_TAB, APPSTORE_SEARCH_TAB \\\\\\\\ \\\\hline format & string & The format of the ad. Values are Icon Ad, Icon + Asset Ad. \\\\\\\\ \\\\hline countryOrRegion & string & A country or region in the European Union in which Apple-delivered advertising is available on the App Store, expressed in ISO 3166-1 alpha-2 format. For example: ES, SE, DE. \\\\\\\\ \\\\hline audienceRefinement & object & Audience refinement criteria used in an ad campaign. \\\\\\\\ \\\\hline ageTarget & boolean & Indicates if age parameters are used in an ad campaign. \\\\\\\\ \\\\hline genderTarget & boolean & Indicates if gender parameters are used in an ad campaign. \\\\\\\\ \\\\hline locationTarget & boolean & Indicates if country or region parameters are used in an ad campaign. \\\\\\\\ \\\\hline customerTypeTarget & boolean & Indicates app downloaded type. Values are: ALL_USERS NEW_USERS RETURNING_USERS USERS_OF_MY_OTHER_APPS \\\\\\\\ \\\\hline \\\\end{tabular}\\n\\n[MISSING_PAGE_EMPTY:12]\\n\\n[MISSING_PAGE_FAIL:13]\\n\\n### Get ad variations\\n\\nUse this endpoint to return ad variations by date range. Use the adId returned in the Get a list of ads call as a resource identifier in the URI.\\n\\nGET [https://adrepository.apple.com/api/v1/ad-repository-ads/](https://adrepository.apple.com/api/v1/ad-repository-ads/){adId}? datePreset=LAST_90_DAYS\\n\\n### Request parameters\\n\\n[0.5cm] ParameterType Required Description adId string yes A unique resource ad identifier.\\n\\n[0.5cm] The date range of the request. Values are LAST_90_DAYS, LAST_180_DAYS, LAST_YEAR. The default is LAST_90_DAYS.\\n\\n### Response example\\n\\nThe AdRepositoryAd object is returned.\\n\\n{  \"data\": {  \"adId\": \"397563857\",  \"appld\": 1582276305,  \"apName\": \"TriPrek - travel app\",  \"developerId\": 819221,  \"developerName\": \"Apple Inc.\",  \"legalName\": \"Apple Inc.\",  \"placement\": \"APPSTORE_SEARCH_RESULTS\",  \"format\": \"Icon Ad\",  \"countryMregion\": \"Spain\",  \"audienceRefinement\": {  \"ageTarget\": \"true\",  \"genderTarget\": \"false\",  \"locationTarget\": \"true\",  \"customerTypeTarget\": \"false\"  },  \"firstImpressionDate\": \"2022-08-25\",  \"lastImpressionDate\": \"2023-04-01\",  \"defaultLanguageTag\": \"es-ES\",  \"defaultLanguageDisplayName\": \"Spanish (Spain)\",  \"defaultPreviewDevice\": \"iPhone_6.5\",  \"adBanner\": {  \"joeColor\": \"b:ff4c0@p0:04@4@4s:17lc01t:361303q:452501\",  \"iconPictureUr1\": \"https//...x_png/x.png\",  \"subtitle\": \"Plamiqique sus viajes con confinara.\",  \"primaryCategory\": \"viajar\",  \"inAppPurchases\": \"false\",  \"promotionalText\": \"Texto promocional que se muestra en la App Store.\",  \"editorialBadge\": \"false\"  \"shortDescription\": \"una descripcion de la aplicacion asociada con el anuncio.\"  },  \"adAssets\": [  {  \"videoUrl\": \"https://...x_png/x.png\",  \"pictureUrl\": \"null\",  \"order\": \"l\",  \"height\": \"1920\",  \"width\": \"1080\",  \"orientation\": \"PORTRAIT\"  },  \"appIconVariations\": [  \"https://..._png/x.png, https://...\\\\(x\\\\).png/x.png\"  ],  \"adLocalevariations\": [  {  \"languageTag\": \"en-GB\",  \"languageDisplayName\": \"English (Great Britain)\",  \"appName\": \"TriPrek\",  \"subTitle\": \"Plan your trips with confidence.\",  \"primaryCategory\": \"travel\",  \"promotionalText\": \"App description shown on the App Store.\",  \"deviceAssets\": [  {  \"previewDevice\": \"iphone_6_5\",  \"adAssets\": [{  \"videoUr1\": \"https...png/x.png\",  \"pictureurl\": \"null\",  \"order\": \"1\",  \"height\": \"1920\",  \"width\": \"1080\",  \"orientation\": \"PORTRAIT\"  }  }  }  }  }  }  }  }  } } ```\\n\\nAd Repository API\\n\\n## Response properties\\n\\n\\\\begin{tabular}{|p{142.3pt}|p{142.3pt}|p{142.3pt}|} \\\\hline\\n**Parameter** & **Type** & **Description** \\\\\\\\ \\\\hline adId & string & A unique ad identifier. Use adId as a resource in the ad variations call. \\\\\\\\ \\\\hline appId & integer & The unique identifier of the app associated with an ad. \\\\\\\\ \\\\hline appName & string & The name of the app associated with an ad. \\\\\\\\ \\\\hline developerId & integer & The unique identifier of the developer as identified on the App Store. \\\\\\\\ \\\\hline  & & The developer name refers to the developer as identified on the App Store which is the entity on whose behalf the subject advertising was presented. In some instances, the developerName may be different from the legalName field for the same entity. \\\\\\\\ \\\\hline legalName & string & The legal name of the person or entity identified under the seller field on the App Store. In some instances, the legalName may be different from the developerName for the same entity. \\\\\\\\ \\\\hline placement & string & The placement of the ad on the App Store. Values are APPSTORE_SEARCH_RESULTS, APPSTORE_TODAY_TAB, APPSTORE_SEARCH_TAB \\\\\\\\ \\\\hline format & string & The format of the ad. Values are Icon Ad, Icon + Asset Ad. \\\\\\\\ \\\\hline countryOrRegion & string & A country or region in the European Union in which Apple-delivered advertising is available on the App Store, expressed in ISO 3166-1 alpha-2 format. For example: ES, SE, DE. \\\\\\\\ \\\\hline audienceRefinement & object & Audience refinement criteria used in an ad campaign. \\\\\\\\ \\\\hline ageTarget & boolean & Indicates if age parameters are used in an ad campaign. \\\\\\\\ \\\\hline genderTarget & boolean & Indicates if gender parameters are used in an ad campaign. \\\\\\\\ \\\\hline locationTarget & boolean & Indicates if country or region parameters are used in an ad campaign. \\\\\\\\ \\\\hline customerTypeTarget & boolean & Indicates app downloader type. Values are: \\\\\\\\ \\\\hline  & & ALL_USERS NEW_USERS RETRURNING_USERS USERS_OF_MY_OTHER_APPS \\\\\\\\ \\\\hline firstImpressionDate & string & The first date of the first recorded delivered impression for the subject advertising. \\\\\\\\ \\\\hline \\\\end{tabular}\\n\\n\\\\begin{tabular}{|p{142.3pt}|p{142.3pt}|p{142.3pt}|} \\\\hline  & & The last recorded delivered impression for the subject advertising. This date will also correspond to the last impression date for advertising that was restricted but not suspended, stopped, or removed. \\\\\\\\ \\\\hline defaultLanguageTag & string & The locale code and default language used in the advertiser\\'s App Store geoterritory. For example, es-ES. \\\\\\\\ \\\\hline defaultLanguageDisplayN am & string & The default language used in the advertiser\\'s App Store geoterritory. For example, “Spanish (Spain)”. \\\\\\\\ \\\\hline defaultPreviewDevice & string & The default device model used in the advertiser\\'s App Store geoterritory. For example, “iPhone_6.5”. \\\\\\\\ \\\\hline adBanner & object & The ad\\'s App Store metadata properties. \\\\\\\\ \\\\hline joeColor & string & An encoded value that defines colors for text and background of the app as it appears on the App Store. \\\\\\\\ \\\\hline iconPictureUrl & string & A resolved URL of the ad icon. \\\\\\\\ \\\\hline subtitle & string & The subtitle of the app as it appears on the App Store. \\\\\\\\ \\\\hline primaryCategory & string & The app category on the App Store. \\\\\\\\ \\\\hline inAppurchases & boolean & Indicates if the app has in-app purchases enabled. \\\\\\\\ \\\\hline promotionalText & string & The app description on the App Store. \\\\\\\\ \\\\hline editorialBadge & boolean & The app\\'s App Store metadata approval shown next to the app icon. \\\\\\\\ \\\\hline shortDescription & string & If promotionalText is not present, a shortDescription matching the default product page is displayed. \\\\\\\\ \\\\hline adAssets & object & A array of ad assets. \\\\\\\\ \\\\hline videoUrl & string & The resolved URL for a video asset. \\\\\\\\ \\\\hline pictureUrl & string & The resolved URL for the image asset. \\\\\\\\ \\\\hline order & integer & The order of the ad asset that was uploaded to App Store Connect. \\\\\\\\ \\\\hline height & integer & The height of the ad asset that was uploaded to App Store Connect. \\\\\\\\ \\\\hline width & integer & The width of the ad asset that was uploaded to App Store Connect. \\\\\\\\ \\\\hline orientation & string & The orientation of the ad asset that was uploaded to App Store Connect. Values are LANDSCAPE, PORTRAIT, UNKNOWN. \\\\\\\\ \\\\hline \\\\end{tabular}\\n\\n[MISSING_PAGE_EMPTY:19]\\n\\n## Get advertising-restrictions\\n\\nUse this endpoint to return details of advertising-restrictions applied during a date range. Ads must have at least one advertising impression to be included in the response.\\n\\nGET [https://adrepository.apple.com/api/v1/restrictions](https://adrepository.apple.com/api/v1/restrictions)\\n\\n### Request example\\n\\nUse ql= URL encoded RSQL filter to allow special characters.\\n\\nGET -/api/v1/restrictions ?ql=  actionTaken==ACTION_ADVERTISING_REMOVED;  datePreset==LAST_90_DAYS  &offset=0  &limit=50\\n\\n### Request parameters\\n\\n\\\\begin{tabular}{|l|c|c|c|} \\\\hline \\\\multicolumn{1}{|c|}{Parameter} & \\\\multicolumn{1}{c|}{Type} & \\\\multicolumn{1}{c|}{Required} & \\\\multicolumn{1}{c|}{Description} \\\\\\\\ \\\\hline \\\\multirow{2}{*}{actionTaken} & \\\\multirow{2}{*}{string} & \\\\multirow{2}{*}{yes} & The action taken on the restriction, Values are: \\\\\\\\  & & & & ACTION_ACCOUNT_SUSPENDED \\\\\\\\  & & & ACTION_ADVERTISING_REMOVED \\\\\\\\ \\\\hline \\\\multirow{2}{*}{datePreset} & \\\\multirow{2}{*}{string} & \\\\multirow{2}{*}{no} & The date range of the request. Values are \\\\\\\\  & & & LAST_90_DAYS, LAST_180_DAYS, LAST_YEAR. The \\\\\\\\  & & & default is LAST_90_DAYS. \\\\\\\\ \\\\hline \\\\multirow{2}{*}{limit} & \\\\multirow{2}{*}{integer} & \\\\multirow{2}{*}{no} & The maximum number of entries to return per page. \\\\\\\\  & & & The default is 50. \\\\\\\\ \\\\hline \\\\multirow{2}{*}{offset} & \\\\multirow{2}{*}{integer} & \\\\multirow{2}{*}{no} & The offset negation that limits the number of \\\\\\\\  & & & returned records. The default is 0. \\\\\\\\ \\\\hline \\\\end{tabular}\\n\\n### Response example\\n\\nThe restriction object is returned.\\n\\n{  \"data\": [  {  \"actionDate\": \"2022-08-26\",  \"actionTaken\": \"ACTION_ADVERITISING_REMOVED\",  \"reason\": \"REASON_TOS_INCOMPATABILITY\",  \"basis\": \"Government order.\",  \"details\": \"Incompatible with Terms and Conditions.\",  \"firstImpressionDate\": \"2022-08-25\",  \"affectedplacements\": [  {  \"placement\": \"APPSTORE_TODAY_TAB\",  \"countriesOrRegions\": [  \"BE\",  \"FR\"  ]  }  ],  \"audienceRefinement\": {  \"ageTarget\": true,  \"genderTarget\": false,  \"locationTarget\": true,  \"customerTypeTarget\": false  }  }  ],  \"dataStartDate\": \"2022-01-01\",  \"dataEndDate\": \"2022-09-01\",  \"pagination\": {  \"totalResults\": 2,  \"startIndex\": 0,  \"itemsPerPage\": 50  }  }\\n\\n### Response properties\\n\\n\\\\begin{tabular}{|p{142.3pt}|p{142.3pt}|p{142.3pt}|} \\\\hline\\n**Parameter** & **Type** & **Description** \\\\\\\\ \\\\hline actionDate & string & The date the restriction was enforced. \\\\\\\\ \\\\hline actionTaken & string & The action taken on the restriction, Values are: \\\\(\\\\text{ACTION\\\\_ACCount\\\\_SUSPENDED}\\\\) \\\\(\\\\text{ACTION\\\\_ADVERTISING\\\\_REMOVED}\\\\) \\\\\\\\ \\\\hline reason & string & The reason on the basis of which the advertising restriction was imposed. For example: \\\\(\\\\text{REASON\\\\_GOVERNMENT\\\\_ORDER}\\\\) \\\\(\\\\text{REASON\\\\_THIRD\\\\_PARTY\\\\_REPORT}\\\\) \\\\(\\\\text{REASON\\\\_POLICY\\\\_RESTRICION\\\\_AGE}\\\\) \\\\(\\\\text{REASON\\\\_POLICY\\\\_RESTRICITION}\\\\) \\\\(\\\\text{REASON\\\\_TOS\\\\_INCOMPATABILITY}\\\\) \\\\\\\\ \\\\hline basis & string & The basis of the restriction. For example, \"Incompatible with Terms and Conditions\". \\\\\\\\ \\\\hline details & string & Details of the restriction. For example, \"Removed due to governmental order\". \\\\\\\\ \\\\hline firstImpressionDate & string & The first date of the first recorded delivered impression for the subject advertising. \\\\\\\\ \\\\hline affectedPlacements & object & Metadata of the ad restriction\\'s App Store placement. \\\\\\\\ \\\\hline placement & string & The placement of the ad on the App Store. Values are \\\\(\\\\text{APPSTORE\\\\_SEARCH\\\\_RESULTS}\\\\), \\\\(\\\\text{APPSTORE\\\\_TODAY\\\\_TAB}\\\\), \\\\(\\\\text{APPSTORE\\\\_SEARCH\\\\_TAB}\\\\) \\\\\\\\ \\\\hline countriesOrRegions & string & Countries or regions in the European Union in which Apple-delivered advertising is available on the App Store, expressed in ISO 3166-1 alpha-2 format. For example: ES, SE, DE. \\\\\\\\ \\\\hline audienceRefinement & object & Audience refinement criteria used in an ad campaign. \\\\\\\\ \\\\hline ageTarget & boolean & Indicates if age parameters are used in the ad campaign. \\\\\\\\ \\\\hline genderTarget & boolean & Indicates if gender parameters are used in the ad campaign. \\\\\\\\ \\\\hline locationTarget & boolean & Indicates if country or region parameters are used in the ad campaign. \\\\\\\\ \\\\hline customerTypeTarget & boolean & Indicates app downloader type. Values are: \\\\(\\\\text{ALL\\\\_USERS}\\\\) \\\\(\\\\text{NEW\\\\_USERS}\\\\) \\\\(\\\\text{RETURNING\\\\_USERS}\\\\) \\\\(\\\\text{USERS\\\\_OF\\\\_MY\\\\_OTHER\\\\_APPS}\\\\) \\\\\\\\ \\\\hline dataStartDate & string & Returned data is delayed by 7 days. For example, if your datePreset is the \\\\(\\\\text{LAST\\\\_90\\\\_DAYS}\\\\), the system will fetch matching ads by factoring today (UTC) as the dataEndDate minus 7 days. The dataStartDate is reported as the dataEndDate minus 90 days. \\\\\\\\ \\\\hline \\\\end{tabular}\\n\\n[MISSING_PAGE_EMPTY:23]\\n\\n[MISSING_PAGE_EMPTY:25]', metadata={'source': 'data/pdf_outputs/Apple_API-Ad-Repository/Ad Repository API.md'})]"
      ]
     },
     "execution_count": 168,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "docs = TextLoader(\"data/pdf_outputs/Apple_API-Ad-Repository/Ad Repository API.md\").load()\n",
    "docs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 217,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "\n",
    "def save_splits(folder_name, file_name, splits):\n",
    "\n",
    "    data = {}\n",
    "    for idx, split in enumerate(splits):\n",
    "        data[idx] = {\n",
    "            'content': split[0],\n",
    "            'line_start': split[1][0],\n",
    "            'line_end': split[1][1]\n",
    "        }\n",
    "\n",
    "    if not os.path.exists(folder_name):\n",
    "        os.makedirs(folder_name)\n",
    "\n",
    "    with open(f\"{folder_name}/{file_name}\", \"w\") as f:\n",
    "        json.dump(data, f, indent=4)\n",
    "        print(f\"Dumping in {folder_name}...\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 189,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('Source document: data/pdf_outputs/Apple_API-Ad-Repository/Ad Repository API.md\\nParagraph location: \\n\\tAd Repository API\\n\\nContent: \\n## Ad Repository API  \\nJanuary 2024  \\n###### Contents  \\n* Getting Started................\\n* 3 Versioning\\n* 3 Usability\\n* 3 Get a list of app and developer names\\n* 4 Request example\\n* 4 Request parameters\\n* 4 Response example\\n* 5 Response properties\\n* 6 Get a list of countries or regions\\n* 7 Response example\\n* 7 Response properties\\n* 8 Get a list of ads\\n* 9 Request example\\n* 9 Request parameters\\n* 9 Response example\\n* 10 Response properties\\n* 11 Get ad variations\\n* 14 Request parameters\\n* 14 Response example\\n* 15 Response properties\\n* 17 Get advertising-restrictions\\n* 20 Request example\\n* 20 Request parameters\\n* 20 Response example\\n* 21 Response properties\\n* 22 Response properties\\n* 22 Changelog\\n', (0, 30))\n",
      "---------------------------\n",
      "('Source document: data/pdf_outputs/Apple_API-Ad-Repository/Ad Repository API.md\\nParagraph location: \\n\\tGetting Started\\n\\nContent: \\n## Getting Started  \\nUse the Ad Repository API to look up Apple-delivered advertising on the App Store in select European countries and regions as well as information about qualifying advertising restrictions. You can manage API calls programmatically using API tools and programs of your choice.\\n', (34, 36))\n",
      "---------------------------\n",
      "('Source document: data/pdf_outputs/Apple_API-Ad-Repository/Ad Repository API.md\\nParagraph location: \\n\\tGetting Started\\n\\tVersioning\\n\\nContent: \\n### Versioning  \\nThe current version of the API is v1.\\n', (38, 40))\n",
      "---------------------------\n",
      "(\"Source document: data/pdf_outputs/Apple_API-Ad-Repository/Ad Repository API.md\\nParagraph location: \\n\\tGetting Started\\n\\tUsability\\n\\nContent: \\n### Usability  \\n* Communication with the web service must use HTTPS.\\n* RSQL query patterns are supported in some endpoints.\\n* The API doesn't accept headers with */*. Disallow Accept */* in your cURL commands.\\n* API responses will have a header Content-Type: application/json.\\n\", (42, 47))\n",
      "---------------------------\n",
      "('Source document: data/pdf_outputs/Apple_API-Ad-Repository/Ad Repository API.md\\nParagraph location: \\n\\tGet a list of app and developer names\\n\\nContent: \\n## Get a list of app and developer names  \\nGET [https://adrepository.apple.com/api/v1/ad-repository-entities](https://adrepository.apple.com/api/v1/ad-repository-entities)  \\nUse this endpoint to return list of app and developer names based on custom query parameter input. A developer name refers to the developer as identified on the App Store, which is the entity on whose behalf the subject advertising was presented. Only apps that have been advertised in countries or regions in the European Union in which Apple-delivered advertising is available on the App Store are searchable.\\n', (49, 52))\n",
      "---------------------------\n",
      "('Source document: data/pdf_outputs/Apple_API-Ad-Repository/Ad Repository API.md\\nParagraph location: \\n\\tGet a list of app and developer names\\n\\tRequest example\\n\\nContent: \\n### Request example  \\nGET -/api/v1/ad-repository-entities?name={URL encoded search input text}  &types==APP  &offset=0  &limit=10\\n', (55, 57))\n",
      "---------------------------\n",
      "('Source document: data/pdf_outputs/Apple_API-Ad-Repository/Ad Repository API.md\\nParagraph location: \\n\\tGet a list of app and developer names\\n\\tRequest parameters\\n\\nContent: \\n### Request parameters  \\n\\\\begin{tabular}{|l|c|c|c|} \\\\hline\\n**Parameter** & **Type** & **Required** & **Description** \\\\\\\\ \\\\hline name & string & yes & Custom query for app and developer names with a minimum of 2 characters required. \\\\\\\\ \\\\hline  & & & Values are: \\\\\\\\ types & string & no & DEVELOPER \\\\\\\\  & & & APP, DEVELOPER (The default value or blank which returns all types.) \\\\\\\\ \\\\hline offset & integer & no & The offset pagination that limits the number of returned records. The default is 0. \\\\\\\\ \\\\hline limit & integer & no & The maximum number of entries to return per page. The default is 50. \\\\\\\\ \\\\hline \\\\end{tabular}  \\nResponse example The AdRepositoryEntity object is returned:  \\n{  \"data\": [  \"id\": 1582276305,  \"name\": \"TripTrek\",  \"type\": \"App\"  },  {  \"id\": 819221,  \"name\": \"TripTrek, Inc.\",  \"type\": \"DEVELOPER\"  },  \"pagination\": {  \"totalResults\": 2,  \"startIndex\": 0,  \"itemsPerPage\": 15  } } ```  \\nAd Repository API January 2024Response properties  \\n**Parameter** & **Type** & **Description**  \\nid & integer & A unique identifier. If the type is APP, the id will be an appId. If the type is DEVELOPER, the id is a developerId. name & string & The entity name on whose behalf the advertising was presented. type & string & Values are APP, DEVELOPER  \\npagination & integer & Returned pagination results. totalResults: The total number of entries that return for the operation. startIndex: Specifies the position of the first element in the results. The default is zero. itemsPerPage: Specifies how many items is displayed per page.\\n', (59, 68))\n",
      "---------------------------\n",
      "('Source document: data/pdf_outputs/Apple_API-Ad-Repository/Ad Repository API.md\\nParagraph location: \\n\\tGet a list of app and developer names\\n\\tGet a list of countries or regions\\n\\nContent: \\n### Get a list of countries or regions  \\nUse this endpoint to get a list of countries or regions in the European Union in which Apple-delivered advertising is available on the App Store. Use the country codes returned in the response in the Get a list of ads call.  \\n#### Request example  \\nGET [https://adrepository.apple.com/api/v1/countries-or-regions](https://adrepository.apple.com/api/v1/countries-or-regions)  \\n#### Response example  \\n{  \"data\": [  {  \"name\": \"Austria\",  \"code\": \"AT\"  },  {  \"name\": \"Belgium\",  \"code\": \"BE\"  },  {  \"name\": \"Croatia\",  \"code\": \"HR\"  },  {  \"name\": \"Czech Republic\",  \"code\": \"CZ\"  },  {  \"name\": \"Denmark\",  \"code\": \"DK\"  },  {  \"name\": \"Finland\",  \"code\": \"FI\"  },  {  \"name\": \"France\",  \"code\": \"FR\"  },  {  \"name\": \"Germany\",  \"code\": \"DE\"  },  {  \"name\": \"Greece\",  \"code\": \"GR\"  },  {  \"name\": \"Hungary\",  \"code\": \"HU\"  },  {  \"name\": \"Ireland\",  \"code\": \"IE\"  },  {  \"name\": \"Italy\",\"code\":\"IT\"  },  {  \"name\":\"Netherlands\",  \"code\":\"NL\"  },  {  \"name\":\"Poland\",  \"code\":\"PL\"  },  {  \"name\":\"Portugal\",  \"code\":\"PT\"  },  {  \"name\":\"Romania\",  \"code\":\"RO\"  },  {  \"name\":\"Spain\",  \"code\":\"ES\"  },  {  \"name\":\"Sweden\",  \"code\":\"SE\"  }  } }  \\nResponse properties  \\nParameterType  \\nDescription  \\nnamestring  \\nThe name of the country or region in the European Union in which Apple-delivered advertising is available on the App Store. For example: \"Sweden\".  \\ncodestring  \\nThe ISO 3166-1 alpha-2 two-letter country code of the country or region in the European Union in which Apple-delivered advertising is available on the App Store. For example: ES, SE, DE.\\n', (76, 89))\n",
      "---------------------------\n",
      "('Source document: data/pdf_outputs/Apple_API-Ad-Repository/Ad Repository API.md\\nParagraph location: \\n\\tGet a list of ads\\n\\nContent: \\n## Get a list of ads  \\nUse this endpoint to return ad metadata and an adId to use in the Get Ad Variations call.  \\nGET [https://adrepository.apple.com/api/v1/ad-repository-ads](https://adrepository.apple.com/api/v1/ad-repository-ads)\\n', (102, 105))\n",
      "---------------------------\n",
      "('Source document: data/pdf_outputs/Apple_API-Ad-Repository/Ad Repository API.md\\nParagraph location: \\n\\tGet a list of ads\\n\\tRequest example\\n\\nContent: \\n### Request example  \\n* Use ql= URL encoded RSQL filter to allow special characters.\\n* You can have both id or countryOrRegion or either in a request. If id is used then type must also be present.  \\nGET -/api/v1/ad-repository-ads ?ql=  type==APP;  id==1582276305;  countryOrRegion=in=(FR,DE);  datePreset==LAST_90_DAYS; &offset=0  &limit=50\\n', (55, 59))\n",
      "---------------------------\n",
      "('Source document: data/pdf_outputs/Apple_API-Ad-Repository/Ad Repository API.md\\nParagraph location: \\n\\tGet a list of ads\\n\\tRequest parameters\\n\\nContent: \\n### Request parameters  \\n\\\\begin{tabular}{|l|c|c|c|} \\\\hline\\n**Parameter** & **Type** & **Required** & **Description** \\\\\\\\ \\\\hline type & string & yes & Values are APP, DEVELOPER. \\\\\\\\ \\\\hline id & string & yes & A unique identifier. If the type is APP, the id is an appl. If the type is DEVELOPER, the id is a developerId. \\\\\\\\ \\\\hline countryOrRegion & string & yes & A country or region in the European Union in which Apple-delivered advertising is available on the App Store, expressed in ISO 3166-1 alpha-2 format. For example: ES, SE, DE. \\\\\\\\ \\\\hline datePreset & string & no & The date range of the request. Values are \\\\\\\\  & & & Last_90_DAYS,LAST_YEAR. \\\\\\\\  & & & The default isLAST_90_DAYS. \\\\\\\\ \\\\hline limit & integer & no & The maximum number of entries to return per page. The default is 50. \\\\\\\\ \\\\hline offset & integer & no & The offset pagination that limits the number of returned records. The default is 0. \\\\\\\\ \\\\hline \\\\end{tabular}\\n', (59, 62))\n",
      "---------------------------\n",
      "('Source document: data/pdf_outputs/Apple_API-Ad-Repository/Ad Repository API.md\\nParagraph location: \\n\\tGet a list of ads\\n\\tResponse example\\n\\nContent: \\n### Response example  \\nThe AdRepositoryAd object is returned.  \\n{  \"data\": [  {  \"adId\": \"76324182\",  \"appId\": 1582276305,  \"appName\": \"TripTrek - travel app\",  \"developerId\": 819221,  \"developerName\": \"Apple Inc\",  \"legalName\": \"Apple Inc.\",  \"placement\": \"APPSTORE_SEARCH_RESULT\",  \"format\": \"Icon Ad\",  \"countryOrRegion\": \"es\",  \"audienceRefinement\": {  \"ageTarget\": \"false\",  \"genderTarget\": \"false\",  \"locationTarget\": \"true\",  \"customerTypeTarget\": \"false\"  },  \"firstImpressionDate\": \"2022-08-25\",  \"lastImpressionDate\": \"2023-04-01\",  \"defaultLanguageTag\": \"es-ES\",  \"defaultLanguageDisplayName\": \"Spanish (Spain)\",  \"defaultPreviewDevice\": \"iPhone_6.5\",  \"adBanner\": {  \"joeColor\": \"b:ff4c00p:04@404s:17lc@1t:361303q:452501\",  \"iconPictureUrl\": \"\",  \"subtitle\": \"Planifique sus viajes con confianza.\",  \"primaryCategory\": \"viajar\",  \"inAppurchases\": \"false\",  \"promotionalText\": \"Descripcion de la aplicacion mostrada en la App Store.\",  \"editorialBadge\": \"false\"  \"shortDescription\": \"una descripcion de la aplicacion asociada con el anuncio.\"  },  \"adAssets\": [  {  \"videoUrl\": \"https://...x.png\",  \"pictureUrl\": \"null\",  \"order\": 1,  \"height\": 1920,  \"width\": 1080,  \"orientation\": \"LANDSCAPE\"  }  ],  \"appIconVariations\": [],  \"adLocaleVariations\": []  }  ],  \"dataStartDate\": \"2023-01-01\",  \"dataEndDate\": \"2023-04-01\",  \"pagination\": {  \"totalResults\": 330,  \"startIndex\": 0,  \"itemsPerPage\": 15  }  }\\n', (120, 123))\n",
      "---------------------------\n",
      "('Source document: data/pdf_outputs/Apple_API-Ad-Repository/Ad Repository API.md\\nParagraph location: \\n\\tResponse properties\\n\\nContent: \\n## Response properties  \\n\\\\begin{tabular}{|p{113.8pt}|p{113.8pt}|p{113.8pt}|} \\\\hline\\n**Parameter** & **Type** & **Description** \\\\\\\\ \\\\hline adId & string & A unique ad identifier. Use adId as a resource in the ad variations call. \\\\\\\\ \\\\hline appId & integer & The unique identifier of the app associated with an ad. \\\\\\\\ \\\\hline appName & string & The name of the app associated with an ad. \\\\\\\\ \\\\hline developerId & integer & The unique identifier of the developer identified on the App Store. \\\\\\\\ \\\\hline developerName & string & The developer name refers to the developer as identified on the App Store which is the entity on whose behalf the subject advertising was presented. In some instances, the developerName may be different from the legalName field for the same entity. \\\\\\\\ \\\\hline legalName & string & The legal name of the person or entity identified under the seller field on the App Store. In some instances, the legalName may be different from the developerName for the same entity. \\\\\\\\ \\\\hline placement & string & The placement of the ad on the App Store. Values are APPSTORE_SEARCH_RESULTS, APPSTORE_TODAY_TAB, APPSTORE_SEARCH_TAB \\\\\\\\ \\\\hline format & string & The format of the ad. Values are Icon Ad, Icon + Asset Ad. \\\\\\\\ \\\\hline countryOrRegion & string & A country or region in the European Union in which Apple-delivered advertising is available on the App Store, expressed in ISO 3166-1 alpha-2 format. For example: ES, SE, DE. \\\\\\\\ \\\\hline audienceRefinement & object & Audience refinement criteria used in an ad campaign. \\\\\\\\ \\\\hline ageTarget & boolean & Indicates if age parameters are used in an ad campaign. \\\\\\\\ \\\\hline genderTarget & boolean & Indicates if gender parameters are used in an ad campaign. \\\\\\\\ \\\\hline locationTarget & boolean & Indicates if country or region parameters are used in an ad campaign. \\\\\\\\ \\\\hline customerTypeTarget & boolean & Indicates app downloaded type. Values are: ALL_USERS NEW_USERS RETURNING_USERS USERS_OF_MY_OTHER_APPS \\\\\\\\ \\\\hline \\\\end{tabular}  \\n[MISSING_PAGE_EMPTY:12]  \\n[MISSING_PAGE_FAIL:13]\\n', (126, 131))\n",
      "---------------------------\n",
      "('Source document: data/pdf_outputs/Apple_API-Ad-Repository/Ad Repository API.md\\nParagraph location: \\n\\tResponse properties\\n\\tGet ad variations\\n\\nContent: \\n### Get ad variations  \\nUse this endpoint to return ad variations by date range. Use the adId returned in the Get a list of ads call as a resource identifier in the URI.  \\nGET [https://adrepository.apple.com/api/v1/ad-repository-ads/](https://adrepository.apple.com/api/v1/ad-repository-ads/){adId}? datePreset=LAST_90_DAYS\\n', (135, 138))\n",
      "---------------------------\n",
      "('Source document: data/pdf_outputs/Apple_API-Ad-Repository/Ad Repository API.md\\nParagraph location: \\n\\tResponse properties\\n\\tRequest parameters\\n\\nContent: \\n### Request parameters  \\n[0.5cm] ParameterType Required Description adId string yes A unique resource ad identifier.  \\n[0.5cm] The date range of the request. Values are LAST_90_DAYS, LAST_180_DAYS, LAST_YEAR. The default is LAST_90_DAYS.\\n', (59, 62))\n",
      "---------------------------\n",
      "('Source document: data/pdf_outputs/Apple_API-Ad-Repository/Ad Repository API.md\\nParagraph location: \\n\\tResponse properties\\n\\tResponse example\\n\\nContent: \\n### Response example  \\nThe AdRepositoryAd object is returned.  \\n{  \"data\": {  \"adId\": \"397563857\",  \"appld\": 1582276305,  \"apName\": \"TriPrek - travel app\",  \"developerId\": 819221,  \"developerName\": \"Apple Inc.\",  \"legalName\": \"Apple Inc.\",  \"placement\": \"APPSTORE_SEARCH_RESULTS\",  \"format\": \"Icon Ad\",  \"countryMregion\": \"Spain\",  \"audienceRefinement\": {  \"ageTarget\": \"true\",  \"genderTarget\": \"false\",  \"locationTarget\": \"true\",  \"customerTypeTarget\": \"false\"  },  \"firstImpressionDate\": \"2022-08-25\",  \"lastImpressionDate\": \"2023-04-01\",  \"defaultLanguageTag\": \"es-ES\",  \"defaultLanguageDisplayName\": \"Spanish (Spain)\",  \"defaultPreviewDevice\": \"iPhone_6.5\",  \"adBanner\": {  \"joeColor\": \"b:ff4c0@p0:04@4@4s:17lc01t:361303q:452501\",  \"iconPictureUr1\": \"https//...x_png/x.png\",  \"subtitle\": \"Plamiqique sus viajes con confinara.\",  \"primaryCategory\": \"viajar\",  \"inAppPurchases\": \"false\",  \"promotionalText\": \"Texto promocional que se muestra en la App Store.\",  \"editorialBadge\": \"false\"  \"shortDescription\": \"una descripcion de la aplicacion asociada con el anuncio.\"  },  \"adAssets\": [  {  \"videoUrl\": \"https://...x_png/x.png\",  \"pictureUrl\": \"null\",  \"order\": \"l\",  \"height\": \"1920\",  \"width\": \"1080\",  \"orientation\": \"PORTRAIT\"  },  \"appIconVariations\": [  \"https://..._png/x.png, https://...\\\\(x\\\\).png/x.png\"  ],  \"adLocalevariations\": [  {  \"languageTag\": \"en-GB\",  \"languageDisplayName\": \"English (Great Britain)\",  \"appName\": \"TriPrek\",  \"subTitle\": \"Plan your trips with confidence.\",  \"primaryCategory\": \"travel\",  \"promotionalText\": \"App description shown on the App Store.\",  \"deviceAssets\": [  {  \"previewDevice\": \"iphone_6_5\",  \"adAssets\": [{  \"videoUr1\": \"https...png/x.png\",  \"pictureurl\": \"null\",  \"order\": \"1\",  \"height\": \"1920\",  \"width\": \"1080\",  \"orientation\": \"PORTRAIT\"  }  }  }  }  }  }  }  }  } } ```  \\nAd Repository API\\n', (120, 124))\n",
      "---------------------------\n",
      "(\"Source document: data/pdf_outputs/Apple_API-Ad-Repository/Ad Repository API.md\\nParagraph location: \\n\\tResponse properties\\n\\nContent: \\n## Response properties  \\n\\\\begin{tabular}{|p{142.3pt}|p{142.3pt}|p{142.3pt}|} \\\\hline\\n**Parameter** & **Type** & **Description** \\\\\\\\ \\\\hline adId & string & A unique ad identifier. Use adId as a resource in the ad variations call. \\\\\\\\ \\\\hline appId & integer & The unique identifier of the app associated with an ad. \\\\\\\\ \\\\hline appName & string & The name of the app associated with an ad. \\\\\\\\ \\\\hline developerId & integer & The unique identifier of the developer as identified on the App Store. \\\\\\\\ \\\\hline  & & The developer name refers to the developer as identified on the App Store which is the entity on whose behalf the subject advertising was presented. In some instances, the developerName may be different from the legalName field for the same entity. \\\\\\\\ \\\\hline legalName & string & The legal name of the person or entity identified under the seller field on the App Store. In some instances, the legalName may be different from the developerName for the same entity. \\\\\\\\ \\\\hline placement & string & The placement of the ad on the App Store. Values are APPSTORE_SEARCH_RESULTS, APPSTORE_TODAY_TAB, APPSTORE_SEARCH_TAB \\\\\\\\ \\\\hline format & string & The format of the ad. Values are Icon Ad, Icon + Asset Ad. \\\\\\\\ \\\\hline countryOrRegion & string & A country or region in the European Union in which Apple-delivered advertising is available on the App Store, expressed in ISO 3166-1 alpha-2 format. For example: ES, SE, DE. \\\\\\\\ \\\\hline audienceRefinement & object & Audience refinement criteria used in an ad campaign. \\\\\\\\ \\\\hline ageTarget & boolean & Indicates if age parameters are used in an ad campaign. \\\\\\\\ \\\\hline genderTarget & boolean & Indicates if gender parameters are used in an ad campaign. \\\\\\\\ \\\\hline locationTarget & boolean & Indicates if country or region parameters are used in an ad campaign. \\\\\\\\ \\\\hline customerTypeTarget & boolean & Indicates app downloader type. Values are: \\\\\\\\ \\\\hline  & & ALL_USERS NEW_USERS RETRURNING_USERS USERS_OF_MY_OTHER_APPS \\\\\\\\ \\\\hline firstImpressionDate & string & The first date of the first recorded delivered impression for the subject advertising. \\\\\\\\ \\\\hline \\\\end{tabular}  \\n\\\\begin{tabular}{|p{142.3pt}|p{142.3pt}|p{142.3pt}|} \\\\hline  & & The last recorded delivered impression for the subject advertising. This date will also correspond to the last impression date for advertising that was restricted but not suspended, stopped, or removed. \\\\\\\\ \\\\hline defaultLanguageTag & string & The locale code and default language used in the advertiser's App Store geoterritory. For example, es-ES. \\\\\\\\ \\\\hline defaultLanguageDisplayN am & string & The default language used in the advertiser's App Store geoterritory. For example, “Spanish (Spain)”. \\\\\\\\ \\\\hline defaultPreviewDevice & string & The default device model used in the advertiser's App Store geoterritory. For example, “iPhone_6.5”. \\\\\\\\ \\\\hline adBanner & object & The ad's App Store metadata properties. \\\\\\\\ \\\\hline joeColor & string & An encoded value that defines colors for text and background of the app as it appears on the App Store. \\\\\\\\ \\\\hline iconPictureUrl & string & A resolved URL of the ad icon. \\\\\\\\ \\\\hline subtitle & string & The subtitle of the app as it appears on the App Store. \\\\\\\\ \\\\hline primaryCategory & string & The app category on the App Store. \\\\\\\\ \\\\hline inAppurchases & boolean & Indicates if the app has in-app purchases enabled. \\\\\\\\ \\\\hline promotionalText & string & The app description on the App Store. \\\\\\\\ \\\\hline editorialBadge & boolean & The app's App Store metadata approval shown next to the app icon. \\\\\\\\ \\\\hline shortDescription & string & If promotionalText is not present, a shortDescription matching the default product page is displayed. \\\\\\\\ \\\\hline adAssets & object & A array of ad assets. \\\\\\\\ \\\\hline videoUrl & string & The resolved URL for a video asset. \\\\\\\\ \\\\hline pictureUrl & string & The resolved URL for the image asset. \\\\\\\\ \\\\hline order & integer & The order of the ad asset that was uploaded to App Store Connect. \\\\\\\\ \\\\hline height & integer & The height of the ad asset that was uploaded to App Store Connect. \\\\\\\\ \\\\hline width & integer & The width of the ad asset that was uploaded to App Store Connect. \\\\\\\\ \\\\hline orientation & string & The orientation of the ad asset that was uploaded to App Store Connect. Values are LANDSCAPE, PORTRAIT, UNKNOWN. \\\\\\\\ \\\\hline \\\\end{tabular}  \\n[MISSING_PAGE_EMPTY:19]\\n\", (126, 131))\n",
      "---------------------------\n",
      "('Source document: data/pdf_outputs/Apple_API-Ad-Repository/Ad Repository API.md\\nParagraph location: \\n\\tGet advertising-restrictions\\n\\nContent: \\n## Get advertising-restrictions  \\nUse this endpoint to return details of advertising-restrictions applied during a date range. Ads must have at least one advertising impression to be included in the response.  \\nGET [https://adrepository.apple.com/api/v1/restrictions](https://adrepository.apple.com/api/v1/restrictions)\\n', (164, 167))\n",
      "---------------------------\n",
      "('Source document: data/pdf_outputs/Apple_API-Ad-Repository/Ad Repository API.md\\nParagraph location: \\n\\tGet advertising-restrictions\\n\\tRequest example\\n\\nContent: \\n### Request example  \\nUse ql= URL encoded RSQL filter to allow special characters.  \\nGET -/api/v1/restrictions ?ql=  actionTaken==ACTION_ADVERTISING_REMOVED;  datePreset==LAST_90_DAYS  &offset=0  &limit=50\\n', (55, 58))\n",
      "---------------------------\n",
      "('Source document: data/pdf_outputs/Apple_API-Ad-Repository/Ad Repository API.md\\nParagraph location: \\n\\tGet advertising-restrictions\\n\\tRequest parameters\\n\\nContent: \\n### Request parameters  \\n\\\\begin{tabular}{|l|c|c|c|} \\\\hline \\\\multicolumn{1}{|c|}{Parameter} & \\\\multicolumn{1}{c|}{Type} & \\\\multicolumn{1}{c|}{Required} & \\\\multicolumn{1}{c|}{Description} \\\\\\\\ \\\\hline \\\\multirow{2}{*}{actionTaken} & \\\\multirow{2}{*}{string} & \\\\multirow{2}{*}{yes} & The action taken on the restriction, Values are: \\\\\\\\  & & & & ACTION_ACCOUNT_SUSPENDED \\\\\\\\  & & & ACTION_ADVERTISING_REMOVED \\\\\\\\ \\\\hline \\\\multirow{2}{*}{datePreset} & \\\\multirow{2}{*}{string} & \\\\multirow{2}{*}{no} & The date range of the request. Values are \\\\\\\\  & & & LAST_90_DAYS, LAST_180_DAYS, LAST_YEAR. The \\\\\\\\  & & & default is LAST_90_DAYS. \\\\\\\\ \\\\hline \\\\multirow{2}{*}{limit} & \\\\multirow{2}{*}{integer} & \\\\multirow{2}{*}{no} & The maximum number of entries to return per page. \\\\\\\\  & & & The default is 50. \\\\\\\\ \\\\hline \\\\multirow{2}{*}{offset} & \\\\multirow{2}{*}{integer} & \\\\multirow{2}{*}{no} & The offset negation that limits the number of \\\\\\\\  & & & returned records. The default is 0. \\\\\\\\ \\\\hline \\\\end{tabular}\\n', (59, 61))\n",
      "---------------------------\n",
      "('Source document: data/pdf_outputs/Apple_API-Ad-Repository/Ad Repository API.md\\nParagraph location: \\n\\tGet advertising-restrictions\\n\\tResponse example\\n\\nContent: \\n### Response example  \\nThe restriction object is returned.  \\n{  \"data\": [  {  \"actionDate\": \"2022-08-26\",  \"actionTaken\": \"ACTION_ADVERITISING_REMOVED\",  \"reason\": \"REASON_TOS_INCOMPATABILITY\",  \"basis\": \"Government order.\",  \"details\": \"Incompatible with Terms and Conditions.\",  \"firstImpressionDate\": \"2022-08-25\",  \"affectedplacements\": [  {  \"placement\": \"APPSTORE_TODAY_TAB\",  \"countriesOrRegions\": [  \"BE\",  \"FR\"  ]  }  ],  \"audienceRefinement\": {  \"ageTarget\": true,  \"genderTarget\": false,  \"locationTarget\": true,  \"customerTypeTarget\": false  }  }  ],  \"dataStartDate\": \"2022-01-01\",  \"dataEndDate\": \"2022-09-01\",  \"pagination\": {  \"totalResults\": 2,  \"startIndex\": 0,  \"itemsPerPage\": 50  }  }\\n', (120, 123))\n",
      "---------------------------\n",
      "('Source document: data/pdf_outputs/Apple_API-Ad-Repository/Ad Repository API.md\\nParagraph location: \\n\\tGet advertising-restrictions\\n\\tResponse properties\\n\\nContent: \\n### Response properties  \\n\\\\begin{tabular}{|p{142.3pt}|p{142.3pt}|p{142.3pt}|} \\\\hline\\n**Parameter** & **Type** & **Description** \\\\\\\\ \\\\hline actionDate & string & The date the restriction was enforced. \\\\\\\\ \\\\hline actionTaken & string & The action taken on the restriction, Values are: \\\\(\\\\text{ACTION\\\\_ACCount\\\\_SUSPENDED}\\\\) \\\\(\\\\text{ACTION\\\\_ADVERTISING\\\\_REMOVED}\\\\) \\\\\\\\ \\\\hline reason & string & The reason on the basis of which the advertising restriction was imposed. For example: \\\\(\\\\text{REASON\\\\_GOVERNMENT\\\\_ORDER}\\\\) \\\\(\\\\text{REASON\\\\_THIRD\\\\_PARTY\\\\_REPORT}\\\\) \\\\(\\\\text{REASON\\\\_POLICY\\\\_RESTRICION\\\\_AGE}\\\\) \\\\(\\\\text{REASON\\\\_POLICY\\\\_RESTRICITION}\\\\) \\\\(\\\\text{REASON\\\\_TOS\\\\_INCOMPATABILITY}\\\\) \\\\\\\\ \\\\hline basis & string & The basis of the restriction. For example, \"Incompatible with Terms and Conditions\". \\\\\\\\ \\\\hline details & string & Details of the restriction. For example, \"Removed due to governmental order\". \\\\\\\\ \\\\hline firstImpressionDate & string & The first date of the first recorded delivered impression for the subject advertising. \\\\\\\\ \\\\hline affectedPlacements & object & Metadata of the ad restriction\\'s App Store placement. \\\\\\\\ \\\\hline placement & string & The placement of the ad on the App Store. Values are \\\\(\\\\text{APPSTORE\\\\_SEARCH\\\\_RESULTS}\\\\), \\\\(\\\\text{APPSTORE\\\\_TODAY\\\\_TAB}\\\\), \\\\(\\\\text{APPSTORE\\\\_SEARCH\\\\_TAB}\\\\) \\\\\\\\ \\\\hline countriesOrRegions & string & Countries or regions in the European Union in which Apple-delivered advertising is available on the App Store, expressed in ISO 3166-1 alpha-2 format. For example: ES, SE, DE. \\\\\\\\ \\\\hline audienceRefinement & object & Audience refinement criteria used in an ad campaign. \\\\\\\\ \\\\hline ageTarget & boolean & Indicates if age parameters are used in the ad campaign. \\\\\\\\ \\\\hline genderTarget & boolean & Indicates if gender parameters are used in the ad campaign. \\\\\\\\ \\\\hline locationTarget & boolean & Indicates if country or region parameters are used in the ad campaign. \\\\\\\\ \\\\hline customerTypeTarget & boolean & Indicates app downloader type. Values are: \\\\(\\\\text{ALL\\\\_USERS}\\\\) \\\\(\\\\text{NEW\\\\_USERS}\\\\) \\\\(\\\\text{RETURNING\\\\_USERS}\\\\) \\\\(\\\\text{USERS\\\\_OF\\\\_MY\\\\_OTHER\\\\_APPS}\\\\) \\\\\\\\ \\\\hline dataStartDate & string & Returned data is delayed by 7 days. For example, if your datePreset is the \\\\(\\\\text{LAST\\\\_90\\\\_DAYS}\\\\), the system will fetch matching ads by factoring today (UTC) as the dataEndDate minus 7 days. The dataStartDate is reported as the dataEndDate minus 90 days. \\\\\\\\ \\\\hline \\\\end{tabular}  \\n[MISSING_PAGE_EMPTY:23]  \\n[MISSING_PAGE_EMPTY:25]\\n', (186, 191))\n",
      "---------------------------\n"
     ]
    }
   ],
   "source": [
    "splits = process_document(docs[0])\n",
    "for split in splits:\n",
    "    print(split)\n",
    "    print('---------------------------')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 190,
   "metadata": {},
   "outputs": [],
   "source": [
    "save_splits('data', 'test', splits) # test would be original folder name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 219,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Looking in folder ../data/platform-docs-versions/Pinterest_DSA-Home-Page\n",
      "Found 1\n",
      "Dumping in _jsons/platform-docs-versions/Pinterest_DSA-Home-Page...\n",
      "Looking in folder ../data/platform-docs-versions/Snapchat_Transparency-World\n",
      "Found 1\n",
      "Dumping in _jsons/platform-docs-versions/Snapchat_Transparency-World...\n",
      "Looking in folder ../data/platform-docs-versions/Google_Custom-Search-JSON-API\n",
      "Found 1\n",
      "Dumping in _jsons/platform-docs-versions/Google_Custom-Search-JSON-API...\n",
      "Looking in folder ../data/platform-docs-versions/Booking.com_AD-API\n",
      "Found 1\n",
      "Dumping in _jsons/platform-docs-versions/Booking.com_AD-API...\n",
      "Looking in folder ../data/platform-docs-versions/Facebook_Instagram-API\n",
      "Found 2\n",
      "Dumping in _jsons/platform-docs-versions/Facebook_Instagram-API...\n",
      "Dumping in _jsons/platform-docs-versions/Facebook_Instagram-API...\n",
      "Looking in folder ../data/platform-docs-versions/Booking.com_DSA-Home-Page\n",
      "Found 2\n",
      "Dumping in _jsons/platform-docs-versions/Booking.com_DSA-Home-Page...\n",
      "Dumping in _jsons/platform-docs-versions/Booking.com_DSA-Home-Page...\n",
      "Looking in folder ../data/platform-docs-versions/Facebook_Content-Library-API\n",
      "Found 1\n",
      "Dumping in _jsons/platform-docs-versions/Facebook_Content-Library-API...\n",
      "Looking in folder ../data/platform-docs-versions/Bing_DSA-Home-Page\n",
      "Found 2\n",
      "Dumping in _jsons/platform-docs-versions/Bing_DSA-Home-Page...\n",
      "Dumping in _jsons/platform-docs-versions/Bing_DSA-Home-Page...\n",
      "Looking in folder ../data/platform-docs-versions/X_Twitter-API-V2\n",
      "Found 9\n",
      "Dumping in _jsons/platform-docs-versions/X_Twitter-API-V2...\n",
      "Dumping in _jsons/platform-docs-versions/X_Twitter-API-V2...\n",
      "Dumping in _jsons/platform-docs-versions/X_Twitter-API-V2...\n",
      "Dumping in _jsons/platform-docs-versions/X_Twitter-API-V2...\n",
      "Dumping in _jsons/platform-docs-versions/X_Twitter-API-V2...\n",
      "Dumping in _jsons/platform-docs-versions/X_Twitter-API-V2...\n",
      "Dumping in _jsons/platform-docs-versions/X_Twitter-API-V2...\n",
      "Dumping in _jsons/platform-docs-versions/X_Twitter-API-V2...\n",
      "Dumping in _jsons/platform-docs-versions/X_Twitter-API-V2...\n",
      "Looking in folder ../data/platform-docs-versions/Pinterest_DSA-Report\n",
      "Found 1\n",
      "Dumping in _jsons/platform-docs-versions/Pinterest_DSA-Report...\n",
      "Looking in folder ../data/platform-docs-versions/CrowdTangle_API\n",
      "Found 2\n",
      "Dumping in _jsons/platform-docs-versions/CrowdTangle_API...\n",
      "Dumping in _jsons/platform-docs-versions/CrowdTangle_API...\n",
      "Looking in folder ../data/platform-docs-versions/Snapchat_Transparency\n",
      "Found 1\n",
      "Dumping in _jsons/platform-docs-versions/Snapchat_Transparency...\n",
      "Looking in folder ../data/platform-docs-versions/Facebook_Fort-Home-Page\n",
      "Found 1\n",
      "Dumping in _jsons/platform-docs-versions/Facebook_Fort-Home-Page...\n",
      "Looking in folder ../data/platform-docs-versions/Facebook_API-Threat-Exchange\n",
      "Found 2\n",
      "Dumping in _jsons/platform-docs-versions/Facebook_API-Threat-Exchange...\n",
      "Dumping in _jsons/platform-docs-versions/Facebook_API-Threat-Exchange...\n",
      "Looking in folder ../data/platform-docs-versions/Google_Transparency-Report\n",
      "Found 1\n",
      "Dumping in _jsons/platform-docs-versions/Google_Transparency-Report...\n",
      "Looking in folder ../data/platform-docs-versions/Facebook_Fort-Pages-API-Reference\n",
      "Found 1\n",
      "Dumping in _jsons/platform-docs-versions/Facebook_Fort-Pages-API-Reference...\n",
      "Looking in folder ../data/platform-docs-versions/Facebook_API-Ad\n",
      "Found 1\n",
      "Dumping in _jsons/platform-docs-versions/Facebook_API-Ad...\n",
      "Looking in folder ../data/platform-docs-versions/Apple_DSA-Home-Page\n",
      "Found 2\n",
      "Dumping in _jsons/platform-docs-versions/Apple_DSA-Home-Page...\n",
      "Dumping in _jsons/platform-docs-versions/Apple_DSA-Home-Page...\n",
      "Looking in folder ../data/platform-docs-versions/Wikipedia_API\n",
      "Found 1\n",
      "Dumping in _jsons/platform-docs-versions/Wikipedia_API...\n",
      "Looking in folder ../data/platform-docs-versions/X_DSA-Transparency-Report\n",
      "Found 1\n",
      "Dumping in _jsons/platform-docs-versions/X_DSA-Transparency-Report...\n",
      "Looking in folder ../data/platform-docs-versions/Facebook_Ad-Library-Tools\n",
      "Found 1\n",
      "Dumping in _jsons/platform-docs-versions/Facebook_Ad-Library-Tools...\n",
      "Looking in folder ../data/platform-docs-versions/Facebook_Community-Report\n",
      "Found 1\n",
      "Dumping in _jsons/platform-docs-versions/Facebook_Community-Report...\n",
      "Looking in folder ../data/platform-docs-versions/Booking.com_Transparency-Report\n",
      "Found 1\n",
      "Dumping in _jsons/platform-docs-versions/Booking.com_Transparency-Report...\n",
      "Looking in folder ../data/platform-docs-versions/Linkedin_Community-Report\n",
      "Found 1\n",
      "Dumping in _jsons/platform-docs-versions/Linkedin_Community-Report...\n",
      "Looking in folder ../data/platform-docs-versions/Linkedin_Ad-Library\n",
      "Found 1\n",
      "Dumping in _jsons/platform-docs-versions/Linkedin_Ad-Library...\n",
      "Looking in folder ../data/platform-docs-versions/Bing_Ad-Library-API\n",
      "Found 1\n",
      "Dumping in _jsons/platform-docs-versions/Bing_Ad-Library-API...\n",
      "Looking in folder ../data/platform-docs-versions/Linkedin_API\n",
      "Found 1\n",
      "Dumping in _jsons/platform-docs-versions/Linkedin_API...\n",
      "Looking in folder ../data/platform-docs-versions/Amazon_Transparency-Report\n",
      "Found 1\n",
      "Dumping in _jsons/platform-docs-versions/Amazon_Transparency-Report...\n",
      "Looking in folder ../data/platform-docs-versions/Facebook_Ad-Targeting-Dataset\n",
      "Found 4\n",
      "Dumping in _jsons/platform-docs-versions/Facebook_Ad-Targeting-Dataset...\n",
      "Dumping in _jsons/platform-docs-versions/Facebook_Ad-Targeting-Dataset...\n",
      "Dumping in _jsons/platform-docs-versions/Facebook_Ad-Targeting-Dataset...\n",
      "Dumping in _jsons/platform-docs-versions/Facebook_Ad-Targeting-Dataset...\n",
      "Looking in folder ../data/platform-docs-versions/TikTok_Transparency\n",
      "Found 5\n",
      "Dumping in _jsons/platform-docs-versions/TikTok_Transparency...\n",
      "Dumping in _jsons/platform-docs-versions/TikTok_Transparency...\n",
      "Dumping in _jsons/platform-docs-versions/TikTok_Transparency...\n",
      "Dumping in _jsons/platform-docs-versions/TikTok_Transparency...\n",
      "Dumping in _jsons/platform-docs-versions/TikTok_Transparency...\n",
      "Looking in folder ../data/platform-docs-versions/Facebook_Government-Report\n",
      "Found 1\n",
      "Dumping in _jsons/platform-docs-versions/Facebook_Government-Report...\n",
      "Looking in folder ../data/platform-docs-versions/Apple_DSA-Transparency-Report\n",
      "Found 2\n",
      "Dumping in _jsons/platform-docs-versions/Apple_DSA-Transparency-Report...\n",
      "Dumping in _jsons/platform-docs-versions/Apple_DSA-Transparency-Report...\n",
      "Looking in folder ../data/platform-docs-versions/X_Twitter-API-V1\n",
      "Found 6\n",
      "Dumping in _jsons/platform-docs-versions/X_Twitter-API-V1...\n",
      "Dumping in _jsons/platform-docs-versions/X_Twitter-API-V1...\n",
      "Dumping in _jsons/platform-docs-versions/X_Twitter-API-V1...\n",
      "Dumping in _jsons/platform-docs-versions/X_Twitter-API-V1...\n",
      "Dumping in _jsons/platform-docs-versions/X_Twitter-API-V1...\n",
      "Dumping in _jsons/platform-docs-versions/X_Twitter-API-V1...\n",
      "Looking in folder ../data/platform-docs-versions/Wikipedia_DSA-Statistics\n",
      "Found 1\n",
      "Dumping in _jsons/platform-docs-versions/Wikipedia_DSA-Statistics...\n",
      "Looking in folder ../data/platform-docs-versions/X_Twitter-API-Overview\n",
      "Found 4\n",
      "Dumping in _jsons/platform-docs-versions/X_Twitter-API-Overview...\n",
      "Dumping in _jsons/platform-docs-versions/X_Twitter-API-Overview...\n",
      "Dumping in _jsons/platform-docs-versions/X_Twitter-API-Overview...\n",
      "Dumping in _jsons/platform-docs-versions/X_Twitter-API-Overview...\n",
      "Looking in folder ../data/platform-docs-versions/TikTok_Research-API\n",
      "Found 2\n",
      "Dumping in _jsons/platform-docs-versions/TikTok_Research-API...\n",
      "Dumping in _jsons/platform-docs-versions/TikTok_Research-API...\n",
      "Looking in folder ../data/platform-docs-versions/Youtube_Data-API\n",
      "Found 1\n",
      "Dumping in _jsons/platform-docs-versions/Youtube_Data-API...\n",
      "Looking in folder ../data/platform-docs-versions/Facebook_Ranking-Transparency\n",
      "Found 1\n",
      "Dumping in _jsons/platform-docs-versions/Facebook_Ranking-Transparency...\n",
      "Looking in folder ../data/platform-docs-versions/TikTok_Example\n",
      "Found 1\n",
      "Dumping in _jsons/platform-docs-versions/TikTok_Example...\n",
      "Looking in folder ../data/platform-docs-versions/Facebook_Features-Reference\n",
      "Found 1\n",
      "Dumping in _jsons/platform-docs-versions/Facebook_Features-Reference...\n",
      "Looking in folder ../data/platform-docs-versions/Linkedin_Government-Requests\n",
      "Found 1\n",
      "Dumping in _jsons/platform-docs-versions/Linkedin_Government-Requests...\n",
      "Looking in folder ../data/platform-docs-versions/Apple_API-Ad-Repository\n",
      "Found 1\n",
      "Dumping in _jsons/platform-docs-versions/Apple_API-Ad-Repository...\n",
      "Looking in folder ../data/platform-docs-versions/Pinterest_API\n",
      "Found 1\n",
      "Dumping in _jsons/platform-docs-versions/Pinterest_API...\n",
      "Looking in folder ../data/platform-docs-versions/Facebook_Graph-API\n",
      "Found 57\n",
      "Dumping in _jsons/platform-docs-versions/Facebook_Graph-API...\n",
      "Dumping in _jsons/platform-docs-versions/Facebook_Graph-API...\n",
      "Dumping in _jsons/platform-docs-versions/Facebook_Graph-API...\n",
      "Dumping in _jsons/platform-docs-versions/Facebook_Graph-API...\n",
      "Dumping in _jsons/platform-docs-versions/Facebook_Graph-API...\n",
      "Dumping in _jsons/platform-docs-versions/Facebook_Graph-API...\n",
      "Dumping in _jsons/platform-docs-versions/Facebook_Graph-API...\n",
      "Dumping in _jsons/platform-docs-versions/Facebook_Graph-API...\n",
      "Dumping in _jsons/platform-docs-versions/Facebook_Graph-API...\n",
      "Dumping in _jsons/platform-docs-versions/Facebook_Graph-API...\n",
      "Dumping in _jsons/platform-docs-versions/Facebook_Graph-API...\n",
      "Dumping in _jsons/platform-docs-versions/Facebook_Graph-API...\n",
      "Dumping in _jsons/platform-docs-versions/Facebook_Graph-API...\n",
      "Dumping in _jsons/platform-docs-versions/Facebook_Graph-API...\n",
      "Dumping in _jsons/platform-docs-versions/Facebook_Graph-API...\n",
      "Dumping in _jsons/platform-docs-versions/Facebook_Graph-API...\n",
      "Dumping in _jsons/platform-docs-versions/Facebook_Graph-API...\n",
      "Dumping in _jsons/platform-docs-versions/Facebook_Graph-API...\n",
      "Dumping in _jsons/platform-docs-versions/Facebook_Graph-API...\n",
      "Dumping in _jsons/platform-docs-versions/Facebook_Graph-API...\n",
      "Dumping in _jsons/platform-docs-versions/Facebook_Graph-API...\n",
      "Dumping in _jsons/platform-docs-versions/Facebook_Graph-API...\n",
      "Dumping in _jsons/platform-docs-versions/Facebook_Graph-API...\n",
      "Dumping in _jsons/platform-docs-versions/Facebook_Graph-API...\n",
      "Dumping in _jsons/platform-docs-versions/Facebook_Graph-API...\n",
      "Dumping in _jsons/platform-docs-versions/Facebook_Graph-API...\n",
      "Dumping in _jsons/platform-docs-versions/Facebook_Graph-API...\n",
      "Dumping in _jsons/platform-docs-versions/Facebook_Graph-API...\n",
      "Dumping in _jsons/platform-docs-versions/Facebook_Graph-API...\n",
      "Dumping in _jsons/platform-docs-versions/Facebook_Graph-API...\n",
      "Dumping in _jsons/platform-docs-versions/Facebook_Graph-API...\n",
      "Dumping in _jsons/platform-docs-versions/Facebook_Graph-API...\n",
      "Dumping in _jsons/platform-docs-versions/Facebook_Graph-API...\n",
      "Dumping in _jsons/platform-docs-versions/Facebook_Graph-API...\n",
      "Dumping in _jsons/platform-docs-versions/Facebook_Graph-API...\n",
      "Dumping in _jsons/platform-docs-versions/Facebook_Graph-API...\n",
      "Dumping in _jsons/platform-docs-versions/Facebook_Graph-API...\n",
      "Dumping in _jsons/platform-docs-versions/Facebook_Graph-API...\n",
      "Dumping in _jsons/platform-docs-versions/Facebook_Graph-API...\n",
      "Dumping in _jsons/platform-docs-versions/Facebook_Graph-API...\n",
      "Dumping in _jsons/platform-docs-versions/Facebook_Graph-API...\n",
      "Dumping in _jsons/platform-docs-versions/Facebook_Graph-API...\n",
      "Dumping in _jsons/platform-docs-versions/Facebook_Graph-API...\n",
      "Dumping in _jsons/platform-docs-versions/Facebook_Graph-API...\n",
      "Dumping in _jsons/platform-docs-versions/Facebook_Graph-API...\n",
      "Dumping in _jsons/platform-docs-versions/Facebook_Graph-API...\n",
      "Dumping in _jsons/platform-docs-versions/Facebook_Graph-API...\n",
      "Dumping in _jsons/platform-docs-versions/Facebook_Graph-API...\n",
      "Dumping in _jsons/platform-docs-versions/Facebook_Graph-API...\n",
      "Dumping in _jsons/platform-docs-versions/Facebook_Graph-API...\n",
      "Dumping in _jsons/platform-docs-versions/Facebook_Graph-API...\n",
      "Dumping in _jsons/platform-docs-versions/Facebook_Graph-API...\n",
      "Dumping in _jsons/platform-docs-versions/Facebook_Graph-API...\n",
      "Dumping in _jsons/platform-docs-versions/Facebook_Graph-API...\n",
      "Dumping in _jsons/platform-docs-versions/Facebook_Graph-API...\n",
      "Dumping in _jsons/platform-docs-versions/Facebook_Graph-API...\n",
      "Dumping in _jsons/platform-docs-versions/Facebook_Graph-API...\n",
      "Looking in folder ../data/platform-docs-versions/TikTok_Commercial-Content-API\n",
      "Found 1\n",
      "Dumping in _jsons/platform-docs-versions/TikTok_Commercial-Content-API...\n",
      "Looking in folder ../data/platform-docs-versions/X_Twitter-API-Enterprise\n",
      "Found 8\n",
      "Dumping in _jsons/platform-docs-versions/X_Twitter-API-Enterprise...\n",
      "Dumping in _jsons/platform-docs-versions/X_Twitter-API-Enterprise...\n",
      "Dumping in _jsons/platform-docs-versions/X_Twitter-API-Enterprise...\n",
      "Dumping in _jsons/platform-docs-versions/X_Twitter-API-Enterprise...\n",
      "Dumping in _jsons/platform-docs-versions/X_Twitter-API-Enterprise...\n",
      "Dumping in _jsons/platform-docs-versions/X_Twitter-API-Enterprise...\n",
      "Dumping in _jsons/platform-docs-versions/X_Twitter-API-Enterprise...\n",
      "Dumping in _jsons/platform-docs-versions/X_Twitter-API-Enterprise...\n"
     ]
    }
   ],
   "source": [
    "\n",
    "folders = list_folders(VERSIONS_DIR)\n",
    "\n",
    "for folder in folders:\n",
    "    full_path = VERSIONS_DIR + folder\n",
    "    print(f\"Looking in folder {full_path}\")\n",
    "    docs = DirectoryLoader(Path(VERSIONS_DIR + folder), glob=\"[!.]*.md\", loader_cls=TextLoader).load()\n",
    "\n",
    "    print(f\"Found {len(docs)}\")\n",
    "    for doc in docs:\n",
    "        splits = process_document(doc)\n",
    "        save_splits(full_path.replace('../data', '_jsons'),  doc.metadata[\"source\"].split('/')[-1].replace('.md', '.json'), splits) # test would be original folder name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Looking in folder data/platform-docs-versions-english/Pinterest_DSA-Home-Page\n",
      "Found 1\n",
      "Dumping in data/_jsons/platform-docs-versions-english/Pinterest_DSA-Home-Page...\n",
      "Looking in folder data/platform-docs-versions-english/Snapchat_Transparency-World\n",
      "Found 1\n",
      "Dumping in data/_jsons/platform-docs-versions-english/Snapchat_Transparency-World...\n",
      "Looking in folder data/platform-docs-versions-english/Platforms\n",
      "Found 1\n",
      "Dumping in data/_jsons/platform-docs-versions-english/Platforms...\n",
      "Looking in folder data/platform-docs-versions-english/Google_Custom-Search-JSON-API\n",
      "Found 1\n",
      "Dumping in data/_jsons/platform-docs-versions-english/Google_Custom-Search-JSON-API...\n",
      "Looking in folder data/platform-docs-versions-english/Booking.com_AD-API\n",
      "Found 1\n",
      "Dumping in data/_jsons/platform-docs-versions-english/Booking.com_AD-API...\n",
      "Looking in folder data/platform-docs-versions-english/Facebook_Instagram-API\n",
      "Found 2\n",
      "Dumping in data/_jsons/platform-docs-versions-english/Facebook_Instagram-API...\n",
      "Dumping in data/_jsons/platform-docs-versions-english/Facebook_Instagram-API...\n",
      "Looking in folder data/platform-docs-versions-english/Booking.com_DSA-Home-Page\n",
      "Found 2\n",
      "Dumping in data/_jsons/platform-docs-versions-english/Booking.com_DSA-Home-Page...\n",
      "Dumping in data/_jsons/platform-docs-versions-english/Booking.com_DSA-Home-Page...\n",
      "Looking in folder data/platform-docs-versions-english/Facebook_Content-Library-API\n",
      "Found 1\n",
      "Dumping in data/_jsons/platform-docs-versions-english/Facebook_Content-Library-API...\n",
      "Looking in folder data/platform-docs-versions-english/Bing_DSA-Home-Page\n",
      "Found 2\n",
      "Dumping in data/_jsons/platform-docs-versions-english/Bing_DSA-Home-Page...\n",
      "Dumping in data/_jsons/platform-docs-versions-english/Bing_DSA-Home-Page...\n",
      "Looking in folder data/platform-docs-versions-english/X_Twitter-API-V2\n",
      "Found 9\n",
      "Dumping in data/_jsons/platform-docs-versions-english/X_Twitter-API-V2...\n",
      "Dumping in data/_jsons/platform-docs-versions-english/X_Twitter-API-V2...\n",
      "Dumping in data/_jsons/platform-docs-versions-english/X_Twitter-API-V2...\n",
      "Dumping in data/_jsons/platform-docs-versions-english/X_Twitter-API-V2...\n",
      "Dumping in data/_jsons/platform-docs-versions-english/X_Twitter-API-V2...\n",
      "Dumping in data/_jsons/platform-docs-versions-english/X_Twitter-API-V2...\n",
      "Dumping in data/_jsons/platform-docs-versions-english/X_Twitter-API-V2...\n",
      "Dumping in data/_jsons/platform-docs-versions-english/X_Twitter-API-V2...\n",
      "Dumping in data/_jsons/platform-docs-versions-english/X_Twitter-API-V2...\n",
      "Looking in folder data/platform-docs-versions-english/Pinterest_DSA-Report\n",
      "Found 1\n",
      "Dumping in data/_jsons/platform-docs-versions-english/Pinterest_DSA-Report...\n",
      "Looking in folder data/platform-docs-versions-english/CrowdTangle_API\n",
      "Found 2\n",
      "Dumping in data/_jsons/platform-docs-versions-english/CrowdTangle_API...\n",
      "Dumping in data/_jsons/platform-docs-versions-english/CrowdTangle_API...\n",
      "Looking in folder data/platform-docs-versions-english/Snapchat_Transparency\n",
      "Found 1\n",
      "Dumping in data/_jsons/platform-docs-versions-english/Snapchat_Transparency...\n",
      "Looking in folder data/platform-docs-versions-english/Facebook_Fort-Home-Page\n",
      "Found 1\n",
      "Dumping in data/_jsons/platform-docs-versions-english/Facebook_Fort-Home-Page...\n",
      "Looking in folder data/platform-docs-versions-english/Facebook_API-Threat-Exchange\n",
      "Found 2\n",
      "Dumping in data/_jsons/platform-docs-versions-english/Facebook_API-Threat-Exchange...\n",
      "Dumping in data/_jsons/platform-docs-versions-english/Facebook_API-Threat-Exchange...\n",
      "Looking in folder data/platform-docs-versions-english/Google_Transparency-Report\n",
      "Found 1\n",
      "Dumping in data/_jsons/platform-docs-versions-english/Google_Transparency-Report...\n",
      "Looking in folder data/platform-docs-versions-english/Facebook_API-Ad\n",
      "Found 1\n",
      "Dumping in data/_jsons/platform-docs-versions-english/Facebook_API-Ad...\n",
      "Looking in folder data/platform-docs-versions-english/Apple_DSA-Home-Page\n",
      "Found 2\n",
      "Dumping in data/_jsons/platform-docs-versions-english/Apple_DSA-Home-Page...\n",
      "Dumping in data/_jsons/platform-docs-versions-english/Apple_DSA-Home-Page...\n",
      "Looking in folder data/platform-docs-versions-english/Wikipedia_API\n",
      "Found 1\n",
      "Dumping in data/_jsons/platform-docs-versions-english/Wikipedia_API...\n",
      "Looking in folder data/platform-docs-versions-english/X_DSA-Transparency-Report\n",
      "Found 1\n",
      "Dumping in data/_jsons/platform-docs-versions-english/X_DSA-Transparency-Report...\n",
      "Looking in folder data/platform-docs-versions-english/Facebook_Ad-Library-Tools\n",
      "Found 1\n",
      "Dumping in data/_jsons/platform-docs-versions-english/Facebook_Ad-Library-Tools...\n",
      "Looking in folder data/platform-docs-versions-english/Facebook_Community-Report\n",
      "Found 1\n",
      "Dumping in data/_jsons/platform-docs-versions-english/Facebook_Community-Report...\n",
      "Looking in folder data/platform-docs-versions-english/Booking.com_Transparency-Report\n",
      "Found 1\n",
      "Dumping in data/_jsons/platform-docs-versions-english/Booking.com_Transparency-Report...\n",
      "Looking in folder data/platform-docs-versions-english/Linkedin_Community-Report\n",
      "Found 1\n",
      "Dumping in data/_jsons/platform-docs-versions-english/Linkedin_Community-Report...\n",
      "Looking in folder data/platform-docs-versions-english/Linkedin_Ad-Library\n",
      "Found 1\n",
      "Dumping in data/_jsons/platform-docs-versions-english/Linkedin_Ad-Library...\n",
      "Looking in folder data/platform-docs-versions-english/Bing_Ad-Library-API\n",
      "Found 1\n",
      "Dumping in data/_jsons/platform-docs-versions-english/Bing_Ad-Library-API...\n",
      "Looking in folder data/platform-docs-versions-english/Linkedin_API\n",
      "Found 1\n",
      "Dumping in data/_jsons/platform-docs-versions-english/Linkedin_API...\n",
      "Looking in folder data/platform-docs-versions-english/Amazon_Transparency-Report\n",
      "Found 1\n",
      "Dumping in data/_jsons/platform-docs-versions-english/Amazon_Transparency-Report...\n",
      "Looking in folder data/platform-docs-versions-english/Facebook_Ad-Targeting-Dataset\n",
      "Found 4\n",
      "Dumping in data/_jsons/platform-docs-versions-english/Facebook_Ad-Targeting-Dataset...\n",
      "Dumping in data/_jsons/platform-docs-versions-english/Facebook_Ad-Targeting-Dataset...\n",
      "Dumping in data/_jsons/platform-docs-versions-english/Facebook_Ad-Targeting-Dataset...\n",
      "Dumping in data/_jsons/platform-docs-versions-english/Facebook_Ad-Targeting-Dataset...\n",
      "Looking in folder data/platform-docs-versions-english/TikTok_Transparency\n",
      "Found 5\n",
      "Dumping in data/_jsons/platform-docs-versions-english/TikTok_Transparency...\n",
      "Dumping in data/_jsons/platform-docs-versions-english/TikTok_Transparency...\n",
      "Dumping in data/_jsons/platform-docs-versions-english/TikTok_Transparency...\n",
      "Dumping in data/_jsons/platform-docs-versions-english/TikTok_Transparency...\n",
      "Dumping in data/_jsons/platform-docs-versions-english/TikTok_Transparency...\n",
      "Looking in folder data/platform-docs-versions-english/Facebook_Government-Report\n",
      "Found 1\n",
      "Dumping in data/_jsons/platform-docs-versions-english/Facebook_Government-Report...\n",
      "Looking in folder data/platform-docs-versions-english/Apple_DSA-Transparency-Report\n",
      "Found 2\n",
      "found\n",
      "Dumping in data/_jsons/platform-docs-versions-english/Apple_DSA-Transparency-Report...\n",
      "Dumping in data/_jsons/platform-docs-versions-english/Apple_DSA-Transparency-Report...\n",
      "Looking in folder data/platform-docs-versions-english/X_Twitter-API-V1\n",
      "Found 6\n",
      "found\n",
      "found\n",
      "found\n",
      "Dumping in data/_jsons/platform-docs-versions-english/X_Twitter-API-V1...\n",
      "Dumping in data/_jsons/platform-docs-versions-english/X_Twitter-API-V1...\n",
      "found\n",
      "found\n",
      "found\n",
      "found\n",
      "found\n",
      "found\n",
      "found\n",
      "found\n",
      "found\n",
      "found\n",
      "found\n",
      "found\n",
      "found\n",
      "found\n",
      "found\n",
      "found\n",
      "found\n",
      "found\n",
      "found\n",
      "found\n",
      "found\n",
      "found\n",
      "found\n",
      "found\n",
      "found\n",
      "found\n",
      "found\n",
      "found\n",
      "found\n",
      "found\n",
      "found\n",
      "found\n",
      "found\n",
      "found\n",
      "found\n",
      "found\n",
      "found\n",
      "found\n",
      "found\n",
      "found\n",
      "found\n",
      "found\n",
      "found\n",
      "found\n",
      "found\n",
      "found\n",
      "found\n",
      "found\n",
      "found\n",
      "found\n",
      "found\n",
      "found\n",
      "found\n",
      "found\n",
      "found\n",
      "Dumping in data/_jsons/platform-docs-versions-english/X_Twitter-API-V1...\n",
      "found\n",
      "found\n",
      "found\n",
      "found\n",
      "found\n",
      "found\n",
      "Dumping in data/_jsons/platform-docs-versions-english/X_Twitter-API-V1...\n",
      "found\n",
      "found\n",
      "found\n",
      "found\n",
      "found\n",
      "found\n",
      "found\n",
      "found\n",
      "found\n",
      "found\n",
      "found\n",
      "found\n",
      "found\n",
      "found\n",
      "found\n",
      "found\n",
      "found\n",
      "found\n",
      "found\n",
      "found\n",
      "found\n",
      "found\n",
      "found\n",
      "found\n",
      "found\n",
      "found\n",
      "found\n",
      "found\n",
      "found\n",
      "Dumping in data/_jsons/platform-docs-versions-english/X_Twitter-API-V1...\n",
      "found\n",
      "found\n",
      "found\n",
      "found\n",
      "found\n",
      "found\n",
      "found\n",
      "found\n",
      "found\n",
      "found\n",
      "found\n",
      "found\n",
      "found\n",
      "found\n",
      "found\n",
      "found\n",
      "found\n",
      "found\n",
      "found\n",
      "Dumping in data/_jsons/platform-docs-versions-english/X_Twitter-API-V1...\n",
      "Looking in folder data/platform-docs-versions-english/Wikipedia_DSA-Statistics\n",
      "Found 1\n",
      "Dumping in data/_jsons/platform-docs-versions-english/Wikipedia_DSA-Statistics...\n",
      "Looking in folder data/platform-docs-versions-english/X_Twitter-API-Overview\n",
      "Found 4\n",
      "Dumping in data/_jsons/platform-docs-versions-english/X_Twitter-API-Overview...\n",
      "Dumping in data/_jsons/platform-docs-versions-english/X_Twitter-API-Overview...\n",
      "Dumping in data/_jsons/platform-docs-versions-english/X_Twitter-API-Overview...\n",
      "Dumping in data/_jsons/platform-docs-versions-english/X_Twitter-API-Overview...\n",
      "Looking in folder data/platform-docs-versions-english/TikTok_Research-API\n",
      "Found 2\n",
      "Dumping in data/_jsons/platform-docs-versions-english/TikTok_Research-API...\n",
      "Dumping in data/_jsons/platform-docs-versions-english/TikTok_Research-API...\n",
      "Looking in folder data/platform-docs-versions-english/Youtube_Data-API\n",
      "Found 1\n",
      "Dumping in data/_jsons/platform-docs-versions-english/Youtube_Data-API...\n",
      "Looking in folder data/platform-docs-versions-english/Facebook_Ranking-Transparency\n",
      "Found 1\n",
      "Dumping in data/_jsons/platform-docs-versions-english/Facebook_Ranking-Transparency...\n",
      "Looking in folder data/platform-docs-versions-english/TikTok_Example\n",
      "Found 1\n",
      "Dumping in data/_jsons/platform-docs-versions-english/TikTok_Example...\n",
      "Looking in folder data/platform-docs-versions-english/Facebook_Features-Reference\n",
      "Found 1\n",
      "Dumping in data/_jsons/platform-docs-versions-english/Facebook_Features-Reference...\n",
      "Looking in folder data/platform-docs-versions-english/Linkedin_Government-Requests\n",
      "Found 1\n",
      "Dumping in data/_jsons/platform-docs-versions-english/Linkedin_Government-Requests...\n",
      "Looking in folder data/platform-docs-versions-english/Apple_API-Ad-Repository\n",
      "Found 1\n",
      "Dumping in data/_jsons/platform-docs-versions-english/Apple_API-Ad-Repository...\n",
      "Looking in folder data/platform-docs-versions-english/Pinterest_API\n",
      "Found 1\n",
      "Dumping in data/_jsons/platform-docs-versions-english/Pinterest_API...\n",
      "Looking in folder data/platform-docs-versions-english/Facebook_Graph-API\n",
      "Found 57\n",
      "Dumping in data/_jsons/platform-docs-versions-english/Facebook_Graph-API...\n",
      "Dumping in data/_jsons/platform-docs-versions-english/Facebook_Graph-API...\n",
      "Dumping in data/_jsons/platform-docs-versions-english/Facebook_Graph-API...\n",
      "Dumping in data/_jsons/platform-docs-versions-english/Facebook_Graph-API...\n",
      "Dumping in data/_jsons/platform-docs-versions-english/Facebook_Graph-API...\n",
      "Dumping in data/_jsons/platform-docs-versions-english/Facebook_Graph-API...\n",
      "Dumping in data/_jsons/platform-docs-versions-english/Facebook_Graph-API...\n",
      "Dumping in data/_jsons/platform-docs-versions-english/Facebook_Graph-API...\n",
      "Dumping in data/_jsons/platform-docs-versions-english/Facebook_Graph-API...\n",
      "Dumping in data/_jsons/platform-docs-versions-english/Facebook_Graph-API...\n",
      "Dumping in data/_jsons/platform-docs-versions-english/Facebook_Graph-API...\n",
      "Dumping in data/_jsons/platform-docs-versions-english/Facebook_Graph-API...\n",
      "Dumping in data/_jsons/platform-docs-versions-english/Facebook_Graph-API...\n",
      "Dumping in data/_jsons/platform-docs-versions-english/Facebook_Graph-API...\n",
      "Dumping in data/_jsons/platform-docs-versions-english/Facebook_Graph-API...\n",
      "Dumping in data/_jsons/platform-docs-versions-english/Facebook_Graph-API...\n",
      "Dumping in data/_jsons/platform-docs-versions-english/Facebook_Graph-API...\n",
      "Dumping in data/_jsons/platform-docs-versions-english/Facebook_Graph-API...\n",
      "Dumping in data/_jsons/platform-docs-versions-english/Facebook_Graph-API...\n",
      "Dumping in data/_jsons/platform-docs-versions-english/Facebook_Graph-API...\n",
      "Dumping in data/_jsons/platform-docs-versions-english/Facebook_Graph-API...\n",
      "Dumping in data/_jsons/platform-docs-versions-english/Facebook_Graph-API...\n",
      "Dumping in data/_jsons/platform-docs-versions-english/Facebook_Graph-API...\n",
      "Dumping in data/_jsons/platform-docs-versions-english/Facebook_Graph-API...\n",
      "Dumping in data/_jsons/platform-docs-versions-english/Facebook_Graph-API...\n",
      "Dumping in data/_jsons/platform-docs-versions-english/Facebook_Graph-API...\n",
      "Dumping in data/_jsons/platform-docs-versions-english/Facebook_Graph-API...\n",
      "Dumping in data/_jsons/platform-docs-versions-english/Facebook_Graph-API...\n",
      "Dumping in data/_jsons/platform-docs-versions-english/Facebook_Graph-API...\n",
      "Dumping in data/_jsons/platform-docs-versions-english/Facebook_Graph-API...\n",
      "Dumping in data/_jsons/platform-docs-versions-english/Facebook_Graph-API...\n",
      "Dumping in data/_jsons/platform-docs-versions-english/Facebook_Graph-API...\n",
      "Dumping in data/_jsons/platform-docs-versions-english/Facebook_Graph-API...\n",
      "Dumping in data/_jsons/platform-docs-versions-english/Facebook_Graph-API...\n",
      "Dumping in data/_jsons/platform-docs-versions-english/Facebook_Graph-API...\n",
      "Dumping in data/_jsons/platform-docs-versions-english/Facebook_Graph-API...\n",
      "Dumping in data/_jsons/platform-docs-versions-english/Facebook_Graph-API...\n",
      "Dumping in data/_jsons/platform-docs-versions-english/Facebook_Graph-API...\n",
      "Dumping in data/_jsons/platform-docs-versions-english/Facebook_Graph-API...\n",
      "Dumping in data/_jsons/platform-docs-versions-english/Facebook_Graph-API...\n",
      "Dumping in data/_jsons/platform-docs-versions-english/Facebook_Graph-API...\n",
      "Dumping in data/_jsons/platform-docs-versions-english/Facebook_Graph-API...\n",
      "Dumping in data/_jsons/platform-docs-versions-english/Facebook_Graph-API...\n",
      "Dumping in data/_jsons/platform-docs-versions-english/Facebook_Graph-API...\n",
      "Dumping in data/_jsons/platform-docs-versions-english/Facebook_Graph-API...\n",
      "Dumping in data/_jsons/platform-docs-versions-english/Facebook_Graph-API...\n",
      "Dumping in data/_jsons/platform-docs-versions-english/Facebook_Graph-API...\n",
      "Dumping in data/_jsons/platform-docs-versions-english/Facebook_Graph-API...\n",
      "Dumping in data/_jsons/platform-docs-versions-english/Facebook_Graph-API...\n",
      "Dumping in data/_jsons/platform-docs-versions-english/Facebook_Graph-API...\n",
      "Dumping in data/_jsons/platform-docs-versions-english/Facebook_Graph-API...\n",
      "Dumping in data/_jsons/platform-docs-versions-english/Facebook_Graph-API...\n",
      "Dumping in data/_jsons/platform-docs-versions-english/Facebook_Graph-API...\n",
      "Dumping in data/_jsons/platform-docs-versions-english/Facebook_Graph-API...\n",
      "Dumping in data/_jsons/platform-docs-versions-english/Facebook_Graph-API...\n",
      "Dumping in data/_jsons/platform-docs-versions-english/Facebook_Graph-API...\n",
      "Dumping in data/_jsons/platform-docs-versions-english/Facebook_Graph-API...\n",
      "Looking in folder data/platform-docs-versions-english/TikTok_Commercial-Content-API\n",
      "Found 1\n",
      "Dumping in data/_jsons/platform-docs-versions-english/TikTok_Commercial-Content-API...\n",
      "Looking in folder data/platform-docs-versions-english/X_Twitter-API-Enterprise\n",
      "Found 8\n",
      "Dumping in data/_jsons/platform-docs-versions-english/X_Twitter-API-Enterprise...\n",
      "Dumping in data/_jsons/platform-docs-versions-english/X_Twitter-API-Enterprise...\n",
      "Dumping in data/_jsons/platform-docs-versions-english/X_Twitter-API-Enterprise...\n",
      "Dumping in data/_jsons/platform-docs-versions-english/X_Twitter-API-Enterprise...\n",
      "Dumping in data/_jsons/platform-docs-versions-english/X_Twitter-API-Enterprise...\n",
      "Dumping in data/_jsons/platform-docs-versions-english/X_Twitter-API-Enterprise...\n",
      "found\n",
      "found\n",
      "found\n",
      "found\n",
      "found\n",
      "found\n",
      "found\n",
      "found\n",
      "found\n",
      "found\n",
      "Dumping in data/_jsons/platform-docs-versions-english/X_Twitter-API-Enterprise...\n",
      "Dumping in data/_jsons/platform-docs-versions-english/X_Twitter-API-Enterprise...\n"
     ]
    }
   ],
   "source": [
    "from pathlib import Path\n",
    "import json\n",
    "\n",
    "VERSIONS_DIR = \"data/platform-docs-versions-english/\"\n",
    "\n",
    "from langchain_community.document_loaders import DirectoryLoader, TextLoader\n",
    "from langchain.text_splitter import MarkdownHeaderTextSplitter\n",
    "\n",
    "import os\n",
    "import re\n",
    "\n",
    "def list_folders(directory):\n",
    "    # Ensure the directory exists\n",
    "    if not os.path.isdir(directory):\n",
    "        print(f\"Error: '{directory}' is not a directory.\")\n",
    "        return\n",
    "\n",
    "    # List all items in the directory\n",
    "    items = os.listdir(directory)\n",
    "\n",
    "    # Filter out only the directories\n",
    "    return [item for item in items if os.path.isdir(os.path.join(directory, item))]\n",
    "\n",
    "\n",
    "def replace_alternate_headings(text):\n",
    "    lines = text.split('\\n')\n",
    "    new_lines = []\n",
    "    i = 0\n",
    "    while i < len(lines):\n",
    "        if i + 1 < len(lines) and lines[i].strip():\n",
    "            # '=' underline headings\n",
    "            if re.match(r'^=+$', lines[i + 1]):\n",
    "                new_lines.append(f'# {lines[i]}')\n",
    "                i += 2\n",
    "                continue\n",
    "            # '-' underline headings\n",
    "            elif re.match(r'^-+$', lines[i + 1]):\n",
    "                new_lines.append(f'## {lines[i]}')\n",
    "                i += 2\n",
    "                continue\n",
    "        new_lines.append(lines[i])\n",
    "        i += 1\n",
    "    return '\\n'.join(new_lines)\n",
    "\n",
    "\n",
    "def find_substring_line(content, substring):\n",
    "    all_lines = content.split('\\n')\n",
    "\n",
    "    substring_lines = substring.split('\\n')\n",
    "    for line_num, line in enumerate(all_lines):\n",
    "        if substring_lines[0].strip(' ') == line.strip(' '):\n",
    "            line_end = line_num + len(substring_lines)\n",
    "            return line_num, line_end\n",
    "    return None, None\n",
    "\n",
    "\n",
    "\n",
    "def process_document(document, merge_h3=False):\n",
    "    if merge_h3:\n",
    "        headers_to_split = [(\"#\", \"H1\"),(\"##\", \"H2\")]\n",
    "    \n",
    "    headers_to_split = [(\"#\", \"H1\"),(\"##\", \"H2\"), ('###', \"H3\")]\n",
    "    new_markdown = replace_alternate_headings(document.page_content)\n",
    "\n",
    "    markdown_splitter = MarkdownHeaderTextSplitter(\n",
    "        headers_to_split_on=headers_to_split, strip_headers=False\n",
    "    )\n",
    "    md_header_splits = markdown_splitter.split_text(new_markdown)\n",
    "    filtered_header_splits = []\n",
    "    threshold = 6000 # length of allowed threshold\n",
    "\n",
    "    metadata = item = None\n",
    "\n",
    "    for d in md_header_splits:\n",
    "        # skip if empty\n",
    "        if len(d.page_content) <= 50:\n",
    "            continue\n",
    "\n",
    "        # add to previous if short and have exactly the same metadata\n",
    "        if metadata == d.metadata.items() and item is not None and len(item) + d.page_content < threshold:\n",
    "            item = item + '\\n' + d.page_content\n",
    "            filtered_header_splits[-1][0] = item\n",
    "            continue\n",
    "        \n",
    "        # create a new split\n",
    "        source = \"Source document: \" + document.metadata['source']\n",
    "        metadata = d.metadata.items()\n",
    "        hierarchy = \"Paragraph location: \" + '\\n'\n",
    "        for k, v in metadata:\n",
    "            if k.startswith('H'):\n",
    "                hierarchy = hierarchy + f\"\\t{v}\\n\"\n",
    "\n",
    "        content = \"Content: \\n\" + d.page_content  + '\\n'\n",
    "        \n",
    "        item = f\"{source}\\n{hierarchy}\\n{content}\" \n",
    "\n",
    "\n",
    "\n",
    "        def extract_urls(text):\n",
    "            if \"Resource URL\" in text:\n",
    "                print('found')\n",
    "                # Regular expression pattern to match URLs\n",
    "                pattern = r'https?://\\S+'\n",
    "\n",
    "                # Find all matches using re.findall\n",
    "                urls = re.findall(pattern, text)\n",
    "\n",
    "                return urls[0]\n",
    "            \n",
    "            # Regular expression pattern to match URLs\n",
    "            pattern = r'https?://\\S+'\n",
    "\n",
    "            # Find all matches using re.findall\n",
    "            urls = re.findall(pattern, text)\n",
    "            \n",
    "            if len(urls):\n",
    "                return urls[0]\n",
    "            \n",
    "            return document.metadata['source'].split('/')[-2] + '/' + document.metadata['source'].split('/')[-1]\n",
    "        \n",
    "        url = extract_urls(new_markdown)\n",
    "\n",
    "        if new_markdown is not None and d.page_content is not None:\n",
    "            line_start, line_end = find_substring_line(new_markdown, d.page_content)\n",
    "            metadata=(line_start, line_end, url)\n",
    "    \n",
    "        filtered_header_splits.append((item, metadata))\n",
    "\n",
    "\n",
    "    return filtered_header_splits\n",
    "\n",
    "def save_splits(folder_name, file_name, splits):\n",
    "\n",
    "    data = {}\n",
    "    for idx, split in enumerate(splits):\n",
    "        data[idx] = {\n",
    "            'content': split[0],\n",
    "            'line_start': split[1][0],\n",
    "            'line_end': split[1][1],\n",
    "            'url': split[1][2],\n",
    "        }\n",
    "\n",
    "    if not os.path.exists(folder_name):\n",
    "        os.makedirs(folder_name)\n",
    "\n",
    "    with open(f\"{folder_name}/{file_name}\", \"w\") as f:\n",
    "        json.dump(data, f, indent=4)\n",
    "        print(f\"Dumping in {folder_name}...\")\n",
    "\n",
    "\n",
    "folders = list_folders(VERSIONS_DIR)\n",
    "\n",
    "for folder in folders:\n",
    "    full_path = VERSIONS_DIR + folder\n",
    "    print(f\"Looking in folder {full_path}\")\n",
    "    docs = DirectoryLoader(Path(VERSIONS_DIR + folder), glob=\"[!.]*.md\", loader_cls=TextLoader).load()\n",
    "\n",
    "    print(f\"Found {len(docs)}\")\n",
    "    for doc in docs:\n",
    "        splits = process_document(doc)\n",
    "        save_splits(full_path.replace('data', 'data/_jsons'),  doc.metadata[\"source\"].split('/')[-1].replace('.md', '.json'), splits) # test would be original folder name\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}