diff --git a/README.md b/README.md
index 89da080..e2153a1 100644
--- a/README.md
+++ b/README.md
@@ -94,13 +94,14 @@ This would be the main function to use within chat-exporter.
**Optional Argument(s):**
`limit`: Integer value to set the limit (amount of messages) the chat exporter gathers when grabbing the history (default=unlimited).
-`tz_info`: String value of a [TZ Database name](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones#List) to set a custom timezone for the exported messages (default=UTC)
-`guild`: `discord.Guild` object which can be passed in to solve bugs for certain forks
-`military_time`: Boolean value to set a 24h format for times within your exported chat (default=False | 12h format)
-`fancy_times`: Boolean value which toggles the 'fancy times' (Today|Yesterday|Day)
-`before`: `datetime.datetime` object which allows to gather messages from before a certain date
-`after`: `datetime.datetime` object which allows to gather messages from after a certain date
-`bot`: `commands.Bot` object to gather members who are no longer in your guild.
+`tz_info`: String value of a [TZ Database name](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones#List) to set a custom timezone for the exported messages (default=UTC).
+`guild`: `discord.Guild` object which can be passed in to solve bugs for certain forks.
+`military_time`: Boolean value to set a 24h format for times within your exported chat (default=False | 12h format).
+`fancy_times`: Boolean value which toggles the 'fancy times' (Today|Yesterday|Day).
+`before`: `datetime.datetime` object which allows to gather messages from before a certain date.
+`after`: `datetime.datetime` object which allows to gather messages from after a certain date.
+`bot`: `commands.Bot` object to gather members who are no longer in your guild.
+`attachment_handler`: `chat_exporter.AttachmentHandler` object to export assets to in order to make them available after the `channel` got deleted.
**Return Argument:**
`transcript`: The HTML build-up for you to construct the HTML File with Discord.
@@ -149,6 +150,7 @@ This would be for people who want to filter what content to export.
`military_time`: Boolean value to set a 24h format for times within your exported chat (default=False | 12h format)
`fancy_times`: Boolean value which toggles the 'fancy times' (Today|Yesterday|Day)
`bot`: `commands.Bot` object to gather members who are no longer in your guild.
+`attachment_handler`: `chat_exporter.AttachmentHandler` object to export assets to in order to make them available after the `channel` got deleted.
**Return Argument:**
`transcript`: The HTML build-up for you to construct the HTML File with Discord.
@@ -183,6 +185,178 @@ async def purge(ctx: commands.Context, tz_info: str, military_time: bool):
```
+
+(back to top)
+
+---
+## Attachment Handler
+
+Due to Discords newly introduced restrictions on to their CDN, we have introduced an Attachment Handler. This handler
+will assist you with circumventing the 'broken' and 'dead-assets' which arise when former attachments hosted by Discord
+reach their expiration date.
+
+The `AttachmentHandler` serves as a template for you to implement your own asset handler. Below are two basic examples on
+how to use the `AttachmentHandler`. One using the example of storing files on a local webserver, with the other being
+an example of storing them on Discord *(the latter merely just being an example, this will still obviously run in to
+the expiration issue)*.
+
+If you do not specify an attachment handler, chat-exporter will continue to use the (proxy) URLs for the assets.
+
+Concept
+
+The concept of implementing such an AttachmentHandler is very easy. In the following a short general procedure is
+described to write your own AttachmentHandler fitting your storage solution. Here we will assume, that we store the
+attachments in a cloud storage.
+
+1. Subclassing
+Start by subclassing `chat_exporter.AttachmentHandler` and implement the `__init__` method if needed. This should look
+something like this:
+
+```python
+from chat_exporter import AttachmentHandler
+from cloud_wrapper import CloudClient
+
+
+class MyAttachmentHandler(AttachmentHandler):
+ def __init__(self, *args, **kwargs):
+ # Your initialization code here
+ # in your case we just create the cloud client
+ self.cloud_client = CloudClient()
+
+```
+
+2. Overwrite process_asset
+The `process_asset` method is the method that is called for each asset in the chat. Here we have to implement the
+upload logic and the generation of the asset url from the uploaded asset.
+
+```python
+import io
+import aiohttp
+from chat_exporter import AttachmentHandler
+from cloud_wrapper import CloudClient
+from discord import Attachment
+
+
+class MyAttachmentHandler(AttachmentHandler):
+ async def process_asset(self, attachment: Attachment):
+ # Your upload logic here, in our example we just upload the asset to the cloud
+
+ # first we need to authorize the client
+ await self.cloud_client.authorize()
+
+ # then we fetch the content of the attachment
+ async with aiohttp.ClientSession() as session:
+ async with session.get(attachment.url) as res:
+ if res.status != 200:
+ res.raise_for_status()
+ data = io.BytesIO(await res.read())
+ data.seek(0)
+
+ # and upload it to the cloud, back we get some sort of identifier for the uploaded file
+ asset_id = await self.cloud_client.upload(data)
+
+ # now we can generate the asset url from the identifier
+ asset_url = await self.cloud_client.get_share_url(asset_id, shared_with="everyone")
+
+ # and set the url attribute of the attachment to the generated url
+ attachment.url = asset_url
+ return attachment
+
+```
+
+Note
+1. The `process_asset` method should return the attachment object with the url attribute set to the generated url.
+2. The `process_asset` method should be an async method, as it is likely that you have to do some async operations
+ like fetching the content of the attachment or uploading it to the cloud.
+3. You are free to add other methods in your class, and call them from `process_asset` if you need to do some
+ operations before or after the upload of the asset. But the `process_asset` method is the only method that is
+called from chat-exporter.
+
+
+AttachmentToLocalFileHostHandler
+
+Assuming you have a file server running, which serves the content of the folder `/usr/share/assets/`
+under `https://example.com/assets/`, you can easily use the `AttachmentToLocalFileHostHandler` like this:
+```python
+import io
+import discord
+from discord.ext import commands
+import chat_exporter
+from chat_exporter import AttachmentToLocalFileHostHandler
+
+...
+
+# Establish the file handler
+file_handler = AttachmentToLocalFileHostHandler(
+ base_path="/usr/share/assets",
+ url_base="https://example.com/assets/",
+)
+
+@bot.command()
+async def save(ctx: commands.Context):
+ transcript = await chat_exporter.export(
+ ctx.channel,
+ attachment_handler=file_handler,
+ )
+
+ if transcript is None:
+ return
+
+ transcript_file = discord.File(
+ io.BytesIO(transcript.encode()),
+ filename=f"transcript-{ctx.channel.name}.html",
+ )
+
+ await ctx.send(file=transcript_file)
+
+```
+AttachmentToDiscordChannel
+
+Assuming you want to store your attachments in a discord channel, you can use the `AttachmentToDiscordChannel`.
+Please note that discord recent changes regarding content links will result in the attachments links being broken
+after 24 hours. While this is therefor not a recommended way to store your attachments, it should give you a good
+idea how to perform asynchronous storing of the attachments.
+
+```python
+import io
+import discord
+from discord.ext import commands
+import chat_exporter
+from chat_exporter import AttachmentToDiscordChannel
+
+...
+
+# Establish the file handler
+channel_handler = AttachmentToDiscordChannel(
+ channel=bot.get_channel(CHANNEL_ID),
+)
+
+@bot.command()
+async def save(ctx: commands.Context):
+ transcript = await chat_exporter.export(
+ ctx.channel,
+ attachment_handler=channel_handler,
+ )
+
+ if transcript is None:
+ return
+
+ transcript_file = discord.File(
+ io.BytesIO(transcript.encode()),
+ filename=f"transcript-{ctx.channel.name}.html",
+ )
+
+ await ctx.send(file=transcript_file)
+
+```
+
(back to top)
---
@@ -204,6 +378,7 @@ async def purge(ctx: commands.Context, tz_info: str, military_time: bool):
---
## Additional Functions
+
Link Function
Downloading exported chats can build up a bunch of unwanted files on your PC which can get annoying, additionally - not everyone wants to download content from Discord.
@@ -274,6 +449,8 @@ It simply makes a request to the given URL and echos (prints) the content for yo
\n'
+ html += '\n'
started = False
if indent % 2 == 0:
while indent < indent_stack[-1]:
@@ -114,25 +120,30 @@ def order_list_markdown_to_html(self):
html += f'- {content.strip()}
\n'
else:
- html += line
+ while len(indent_stack) > 1:
+ html += '
'
+ indent_stack.pop()
+ if not started:
+ html += '
'
+ started = True
+ html += line + '\n'
while len(indent_stack) > 1:
html += '\n'
indent_stack.pop()
- if not started:
- self.content = html
+ self.content = html
def parse_normal_markdown(self):
- # self.order_list_markdown_to_html()
+ self.order_list_markdown_to_html()
holder = (
[r"__(.*?)__", '%s'],
[r"\*\*(.*?)\*\*", '%s'],
[r"\*(.*?)\*", '%s'],
[r"~~(.*?)~~", '%s'],
- # [r"###\s(.*?)\n", '%s'],
- # [r"##\s(.*?)\n", '%s'],
- # [r"#\s(.*?)\n", '%s
'],
+ [r"^###\s(.*?)\n", '%s'],
+ [r"^##\s(.*?)\n", '%s'],
+ [r"^#\s(.*?)\n", '%s
'],
[r"\|\|(.*?)\|\|", ' %s'],
)
@@ -140,7 +151,7 @@ def parse_normal_markdown(self):
for x in holder:
p, r = x
- pattern = re.compile(p)
+ pattern = re.compile(p, re.M)
match = re.search(pattern, self.content)
while match is not None:
affected_text = match.group(1)
@@ -205,15 +216,16 @@ def parse_code_block_markdown(self, reference=False):
second_match = re.search(second_pattern, affected_text)
affected_text = re.sub(" ", " ", affected_text)
+ self.code_blocks_content.append(affected_text)
if not reference:
self.content = self.content.replace(
self.content[match.start():match.end()],
- '
%s
' % (language_class, affected_text)
+ '%s
' % (language_class, f'%s{len(self.code_blocks_content)}')
)
else:
self.content = self.content.replace(
self.content[match.start():match.end()],
- '%s' % affected_text
+ '%s' % f'%s{len(self.code_blocks_content)}'
)
match = re.search(pattern, self.content)
@@ -224,8 +236,9 @@ def parse_code_block_markdown(self, reference=False):
while match is not None:
affected_text = match.group(1)
affected_text = self.return_to_markdown(affected_text)
+ self.code_blocks_content.append(affected_text)
self.content = self.content.replace(self.content[match.start():match.end()],
- '%s' % affected_text)
+ '%s' % f'%s{len(self.code_blocks_content)}')
match = re.search(pattern, self.content)
# `code`
@@ -234,12 +247,17 @@ def parse_code_block_markdown(self, reference=False):
while match is not None:
affected_text = match.group(1)
affected_text = self.return_to_markdown(affected_text)
+ self.code_blocks_content.append(affected_text)
self.content = self.content.replace(self.content[match.start():match.end()],
- '%s' % affected_text)
+ '%s' % f'%s{len(self.code_blocks_content)}')
match = re.search(pattern, self.content)
self.content = re.sub(r"
", "\n", self.content)
+ def reverse_code_block_markdown(self):
+ for x in range(len(self.code_blocks_content)):
+ self.content = self.content.replace(f'%s{x + 1}', self.code_blocks_content[x])
+
def parse_embed_markdown(self):
# [Message](Link)
pattern = re.compile(r"\[(.+?)]\((.+?)\)")
diff --git a/chat_exporter/parse/mention.py b/chat_exporter/parse/mention.py
index cf4b86d..46441b4 100644
--- a/chat_exporter/parse/mention.py
+++ b/chat_exporter/parse/mention.py
@@ -3,9 +3,10 @@
import pytz
import datetime
+import time
from chat_exporter.ext.discord_import import discord
-
+from chat_exporter.parse.markdown import ParseMarkdown
bot: Optional[discord.Client] = None
@@ -19,6 +20,8 @@ def pass_bot(_bot):
class ParseMention:
REGEX_ROLES = r"<@&([0-9]+)>"
REGEX_ROLES_2 = r"<@&([0-9]+)>"
+ REGEX_EVERYONE = r"@(everyone)(?:[$\s\t\n\f\r\0]|$)"
+ REGEX_HERE = r"@(here)(?:[$\s\t\n\f\r\0]|$)"
REGEX_MEMBERS = r"<@!?([0-9]+)>"
REGEX_MEMBERS_2 = r"<@!?([0-9]+)>"
REGEX_CHANNELS = r"<#([0-9]+)>"
@@ -26,15 +29,16 @@ class ParseMention:
REGEX_EMOJIS = r"<a?(:[^\n:]+:)[0-9]+>"
REGEX_EMOJIS_2 = r""
REGEX_TIME_HOLDER = (
- [r"<t:([0-9]+):t>", "%H:%M"],
- [r"<t:([0-9]+):T>", "%T"],
- [r"<t:([0-9]+):d>", "%d/%m/%Y"],
- [r"<t:([0-9]+):D>", "%e %B %Y"],
- [r"<t:([0-9]+):f>", "%e %B %Y %H:%M"],
- [r"<t:([0-9]+):F>", "%A, %e %B %Y %H:%M"],
- [r"<t:([0-9]+):R>", "%e %B %Y %H:%M"],
- [r"<t:([0-9]+)>", "%e %B %Y %H:%M"]
+ [r"<t:([0-9]{1,13}):t>", "%H:%M"],
+ [r"<t:([0-9]{1,13}):T>", "%T"],
+ [r"<t:([0-9]{1,13}):d>", "%d/%m/%Y"],
+ [r"<t:([0-9]{1,13}):D>", "%e %B %Y"],
+ [r"<t:([0-9]{1,13}):f>", "%e %B %Y %H:%M"],
+ [r"<t:([0-9]{1,13}):F>", "%A, %e %B %Y %H:%M"],
+ [r"<t:([0-9]{1,13}):R>", "%e %B %Y %H:%M"],
+ [r"<t:([0-9]{1,13})>", "%e %B %Y %H:%M"]
)
+ REGEX_SLASH_COMMAND = r"<\/([\w]+ ?[\w]*):[0-9]+>"
ESCAPE_LT = "______lt______"
ESCAPE_GT = "______gt______"
@@ -43,8 +47,12 @@ class ParseMention:
def __init__(self, content, guild):
self.content = content
self.guild = guild
+ self.code_blocks_content = []
async def flow(self):
+ markdown = ParseMarkdown(self.content)
+ markdown.parse_code_block_markdown()
+ self.content = markdown.content
await self.escape_mentions()
await self.escape_mentions()
await self.unescape_mentions()
@@ -52,9 +60,13 @@ async def flow(self):
await self.member_mention()
await self.role_mention()
await self.time_mention()
-
+ await self.slash_command_mention()
+ markdown.content = self.content
+ markdown.reverse_code_block_markdown()
+ self.content = markdown.content
return self.content
+
async def escape_mentions(self):
for match in re.finditer("(%s|%s|%s|%s|%s|%s|%s|%s)"
% (self.REGEX_ROLES, self.REGEX_MEMBERS, self.REGEX_CHANNELS, self.REGEX_EMOJIS,
@@ -94,6 +106,16 @@ async def channel_mention(self):
match = re.search(regex, self.content)
async def role_mention(self):
+ holder = self.REGEX_EVERYONE, self.REGEX_HERE
+ for regex in holder:
+ match = re.search(regex, self.content)
+ while match is not None:
+ role_name = match.group(1)
+ replacement = '@%s' % (str(role_name), str(role_name))
+
+ self.content = self.content.replace(self.content[match.start():match.end()],
+ replacement)
+ match = re.search(regex, self.content)
holder = self.REGEX_ROLES, self.REGEX_ROLES_2
for regex in holder:
match = re.search(regex, self.content)
@@ -108,12 +130,22 @@ async def role_mention(self):
colour = "#dee0fc"
else:
colour = "#%02x%02x%02x" % (role.color.r, role.color.g, role.color.b)
- replacement = '@%s' \
- % (colour, role.name)
+ replacement = '@%s' % (colour, role.name)
self.content = self.content.replace(self.content[match.start():match.end()], replacement)
-
match = re.search(regex, self.content)
+ async def slash_command_mention(self):
+ match = re.search(self.REGEX_SLASH_COMMAND, self.content)
+ while match is not None:
+ slash_command_name = match.group(1)
+ replacement = (
+ '/%s'
+ % (slash_command_name, slash_command_name)
+ )
+ self.content = self.content.replace(self.content[match.start():match.end()], replacement)
+
+ match = re.search(self.REGEX_SLASH_COMMAND, self.content)
+
async def member_mention(self):
holder = self.REGEX_MEMBERS, self.REGEX_MEMBERS_2
for regex in holder:
@@ -150,9 +182,13 @@ async def time_mention(self):
regex, strf = p
match = re.search(regex, self.content)
while match is not None:
- time = datetime.datetime.fromtimestamp(int(match.group(1)), timezone)
- ui_time = time.strftime(strf)
- tooltip_time = time.strftime("%A, %e %B %Y at %H:%M")
+ timestamp = int(match.group(1)) - 1
+ time_stamp = time.gmtime(timestamp)
+ datetime_stamp = datetime.datetime(2010, *time_stamp[1:6], tzinfo=pytz.utc)
+ ui_time = datetime_stamp.strftime(strf)
+ ui_time = ui_time.replace(str(datetime_stamp.year), str(time_stamp[0]))
+ tooltip_time = datetime_stamp.strftime("%A, %e %B %Y at %H:%M")
+ tooltip_time = tooltip_time.replace(str(datetime_stamp.year), str(time_stamp[0]))
original = match.group().replace("<", "<").replace(">", ">")
replacement = (
f''