From 842f5f08c9fe63c1177b6acc1dd7647276e5bde2 Mon Sep 17 00:00:00 2001
From: David Else <12832280+David-Else@users.noreply.github.com>
Date: Sat, 1 Jun 2024 23:07:35 +0100
Subject: [PATCH] feat: add `get_web_page` (#11)

---
 sh/get_web_page.sh | 14 ++++++++++++++
 1 file changed, 14 insertions(+)
 create mode 100755 sh/get_web_page.sh

diff --git a/sh/get_web_page.sh b/sh/get_web_page.sh
new file mode 100755
index 0000000..65e6d2a
--- /dev/null
+++ b/sh/get_web_page.sh
@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+set -e
+
+# @describe Fetches the HTML content from a specified webpage URL and converts it to Markdown format.
+# Use it to answer user questions that require up-to-date content from web pages.
+# @meta require-tools curl,pandoc,sed
+# @option --url! The URL to scrape.
+
+main() {
+  # span and div tags are dropped from the HTML https://pandoc.org/MANUAL.html#raw-htmltex and sed removes any inline SVG images in image tags from the Markdown content.
+  curl -fsSL "$argc_url" | pandoc -f html-native_divs-native_spans -t gfm-raw_html | sed -E 's/!\[.*?\]\((data:image\/svg\+xml[^)]+)\)//g'
+}
+
+eval "$(argc --argc-eval "$0" "$@")"