From cf87842acca73e1694b95dbde10a4c6e6b9da0ab Mon Sep 17 00:00:00 2001 From: Matthew Moss <56257224+mahtoid@users.noreply.github.com> Date: Tue, 11 Jul 2023 20:10:57 +0100 Subject: [PATCH] DiscordChatExporterPy 2.6.0 (#89) * Set default timezone just in case failed to setattr * Updated discord links * Font and Styling adjustments * Development branch solve (#83) * HTML Escape Embeds and get topic only if text channel * README.md update * Sped up references, updated README, updated twemoji link and more --------- Co-authored-by: mahtoid * Reference speed up and solves (#84) * HTML Escape Embeds and get topic only if text channel * README.md update * Sped up references, updated README, updated twemoji link and more * Reference speed up and solves --------- Co-authored-by: mahtoid * Solves for username update and threads * Thread improvements and thread add support (minus SVG) * Removed bottom padding on emoji--small * Headers and Order List support * Split imports --------- Co-authored-by: mahtoid --- .github/ISSUE_TEMPLATE/config.yml | 2 +- README.md | 5 +- chat_exporter/construct/assets/embed.py | 24 ++- chat_exporter/construct/message.py | 101 +++++++-- chat_exporter/construct/transcript.py | 11 +- chat_exporter/ext/discord_utils.py | 2 + chat_exporter/ext/discriminator.py | 4 + chat_exporter/ext/emoji_convert.py | 2 +- chat_exporter/ext/html_generator.py | 2 + chat_exporter/html/base.html | 203 ++++++++++-------- chat_exporter/html/message/interaction.html | 2 +- chat_exporter/html/message/message.html | 2 +- chat_exporter/html/message/pin.html | 2 +- chat_exporter/html/message/reference.html | 5 +- chat_exporter/html/message/thread_add.html | 18 ++ chat_exporter/html/message/thread_remove.html | 18 ++ chat_exporter/parse/markdown.py | 109 +++++++++- chat_exporter/parse/mention.py | 5 +- pyproject.toml | 2 +- 19 files changed, 384 insertions(+), 135 deletions(-) create mode 100644 chat_exporter/ext/discriminator.py create mode 100644 chat_exporter/html/message/thread_add.html create mode 100644 chat_exporter/html/message/thread_remove.html diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml index d19ed0e..732020e 100644 --- a/.github/ISSUE_TEMPLATE/config.yml +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -1,5 +1,5 @@ blank_issues_enabled: false contact_links: - name: Discord - url: https://discord.gg/2uhHBQDwcc + url: https://discord.mahto.id/ about: Find help within the Discord community \ No newline at end of file diff --git a/README.md b/README.md index 7e1ee9f..89da080 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@

Export Discord chats with your discord.py (or fork) bots!
- Join Discord + Join Discord · Report Bug · @@ -95,8 +95,11 @@ This would be the main function to use within chat-exporter. **Optional Argument(s):**
`limit`: Integer value to set the limit (amount of messages) the chat exporter gathers when grabbing the history (default=unlimited).
`tz_info`: String value of a [TZ Database name](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones#List) to set a custom timezone for the exported messages (default=UTC)
+`guild`: `discord.Guild` object which can be passed in to solve bugs for certain forks
`military_time`: Boolean value to set a 24h format for times within your exported chat (default=False | 12h format)
`fancy_times`: Boolean value which toggles the 'fancy times' (Today|Yesterday|Day)
+`before`: `datetime.datetime` object which allows to gather messages from before a certain date +`after`: `datetime.datetime` object which allows to gather messages from after a certain date `bot`: `commands.Bot` object to gather members who are no longer in your guild. **Return Argument:**
diff --git a/chat_exporter/construct/assets/embed.py b/chat_exporter/construct/assets/embed.py index af9663e..b8d7899 100644 --- a/chat_exporter/construct/assets/embed.py +++ b/chat_exporter/construct/assets/embed.py @@ -106,16 +106,19 @@ async def build_fields(self): ("FIELD_VALUE", field.value, PARSE_MODE_EMBED)]) async def build_author(self): - self.author = html.escape(self.embed.author.name) if self.embed.author.name != self.check_against else "" + self.author = html.escape(self.embed.author.name) if ( + self.embed.author and self.embed.author.name != self.check_against + ) else "" self.author = f'{self.author}' \ - if self.embed.author.url != self.check_against \ - else self.author + if ( + self.embed.author and self.embed.author.url != self.check_against + ) else self.author author_icon = await fill_out(self.guild, embed_author_icon, [ ("AUTHOR", self.author, PARSE_MODE_NONE), ("AUTHOR_ICON", self.embed.author.icon_url, PARSE_MODE_NONE) - ]) if self.embed.author.icon_url != self.check_against else "" + ]) if self.embed.author and self.embed.author.icon_url != self.check_against else "" if author_icon == "" and self.author != "": self.author = await fill_out(self.guild, embed_author, [("AUTHOR", self.author, PARSE_MODE_NONE)]) @@ -125,16 +128,21 @@ async def build_author(self): async def build_image(self): self.image = await fill_out(self.guild, embed_image, [ ("EMBED_IMAGE", str(self.embed.image.proxy_url), PARSE_MODE_NONE) - ]) if self.embed.image.url != self.check_against else "" + ]) if self.embed.image and self.embed.image.url != self.check_against else "" async def build_thumbnail(self): self.thumbnail = await fill_out(self.guild, embed_thumbnail, [ ("EMBED_THUMBNAIL", str(self.embed.thumbnail.url), PARSE_MODE_NONE)]) \ - if self.embed.thumbnail.url != self.check_against else "" + if self.embed.thumbnail and self.embed.thumbnail.url != self.check_against else "" async def build_footer(self): - self.footer = html.escape(self.embed.footer.text) if self.embed.footer.text != self.check_against else "" - footer_icon = self.embed.footer.icon_url if self.embed.footer.icon_url != self.check_against else None + self.footer = html.escape(self.embed.footer.text) if ( + self.embed.footer and self.embed.footer.text != self.check_against + ) else "" + + footer_icon = self.embed.footer.icon_url if ( + self.embed.footer and self.embed.footer.icon_url != self.check_against + ) else None if not self.footer: return diff --git a/chat_exporter/construct/message.py b/chat_exporter/construct/message.py index e675b8b..1642944 100644 --- a/chat_exporter/construct/message.py +++ b/chat_exporter/construct/message.py @@ -8,6 +8,7 @@ from chat_exporter.construct.assets import Attachment, Component, Embed, Reaction from chat_exporter.ext.discord_utils import DiscordUtils +from chat_exporter.ext.discriminator import discriminator from chat_exporter.ext.html_generator import ( fill_out, bot_tag, @@ -25,6 +26,8 @@ PARSE_MODE_NONE, PARSE_MODE_MARKDOWN, PARSE_MODE_REFERENCE, + message_thread_remove, + message_thread_add, ) @@ -37,7 +40,7 @@ def _gather_user_bot(author: discord.Member): def _set_edit_at(message_edited_at): - return f'(edited)' + return f'(edited)' class MessageConstruct: @@ -57,13 +60,15 @@ def __init__( pytz_timezone, military_time: bool, guild: discord.Guild, - meta_data: dict + meta_data: dict, + message_dict: dict ): self.message = message self.previous_message = previous_message self.pytz_timezone = pytz_timezone self.military_time = military_time self.guild = guild + self.message_dict = message_dict self.time_format = "%A, %e %B %Y %I:%M %p" if self.military_time: @@ -79,6 +84,10 @@ async def construct_message( await self.build_pin() elif discord.MessageType.thread_created == self.message.type: await self.build_thread() + elif discord.MessageType.recipient_remove == self.message.type: + await self.build_thread_remove() + elif discord.MessageType.recipient_add == self.message.type: + await self.build_thread_add() else: await self.build_message() return self.message_html, self.meta_data @@ -100,13 +109,21 @@ async def build_thread(self): await self.generate_message_divider(channel_audit=True) await self.build_thread_template() + async def build_thread_remove(self): + await self.generate_message_divider(channel_audit=True) + await self.build_remove() + + async def build_thread_add(self): + await self.generate_message_divider(channel_audit=True) + await self.build_add() + async def build_meta_data(self): user_id = self.message.author.id if user_id in self.meta_data: self.meta_data[user_id][4] += 1 else: - user_name_discriminator = self.message.author.name + "#" + self.message.author.discriminator + user_name_discriminator = await discriminator(self.message.author.name, self.message.author.discriminator) user_created_at = self.message.author.created_at user_bot = _gather_user_bot(self.message.author) user_avatar = ( @@ -142,13 +159,16 @@ async def build_reference(self): self.message.reference = "" return - try: - message: discord.Message = await self.message.channel.fetch_message(self.message.reference.message_id) - except (discord.NotFound, discord.HTTPException) as e: - self.message.reference = "" - if isinstance(e, discord.NotFound): - self.message.reference = message_reference_unknown - return + message: discord.Message = self.message_dict.get(self.message.reference.message_id) + + if not message: + try: + message: discord.Message = await self.message.channel.fetch_message(self.message.reference.message_id) + except (discord.NotFound, discord.HTTPException) as e: + self.message.reference = "" + if isinstance(e, discord.NotFound): + self.message.reference = message_reference_unknown + return is_bot = _gather_user_bot(message.author) user_colour = await self._gather_user_colour(message.author) @@ -173,7 +193,7 @@ async def build_reference(self): self.message.reference = await fill_out(self.guild, message_reference, [ ("AVATAR_URL", str(avatar_url), PARSE_MODE_NONE), ("BOT_TAG", is_bot, PARSE_MODE_NONE), - ("NAME_TAG", "%s#%s" % (message.author.name, message.author.discriminator), PARSE_MODE_NONE), + ("NAME_TAG", await discriminator(message.author.name, message.author.discriminator), PARSE_MODE_NONE), ("NAME", str(html.escape(message.author.display_name))), ("USER_COLOUR", user_colour, PARSE_MODE_NONE), ("CONTENT", message.content, PARSE_MODE_REFERENCE), @@ -195,7 +215,7 @@ async def build_interaction(self): self.message.interaction = await fill_out(self.guild, message_interaction, [ ("AVATAR_URL", str(avatar_url), PARSE_MODE_NONE), ("BOT_TAG", is_bot, PARSE_MODE_NONE), - ("NAME_TAG", "%s#%s" % (user.name, user.discriminator), PARSE_MODE_NONE), + ("NAME_TAG", await discriminator(user.name, user.discriminator), PARSE_MODE_NONE), ("NAME", str(html.escape(user.display_name))), ("USER_COLOUR", user_colour, PARSE_MODE_NONE), ("FILLER", "used ", PARSE_MODE_NONE), @@ -258,7 +278,8 @@ async def build_message_template(self): def _generate_message_divider_check(self): return bool( - self.previous_message is None or self.message.reference != "" or self.message.interaction != "" or + self.previous_message is None or self.message.reference != "" or + self.previous_message.type is not discord.MessageType.default or self.message.interaction != "" or self.previous_message.author.id != self.message.author.id or self.message.webhook_id is not None or self.message.created_at > (self.previous_message.created_at + timedelta(minutes=4)) ) @@ -269,6 +290,7 @@ async def generate_message_divider(self, channel_audit=False): self.message_html += await fill_out(self.guild, end_message, []) if channel_audit: + self.audit = True return followup_symbol = "" @@ -289,7 +311,7 @@ async def generate_message_divider(self, channel_audit=False): ("REFERENCE", self.message.reference if self.message.reference else self.message.interaction, PARSE_MODE_NONE), ("AVATAR_URL", str(avatar_url), PARSE_MODE_NONE), - ("NAME_TAG", "%s#%s" % (self.message.author.name, self.message.author.discriminator), PARSE_MODE_NONE), + ("NAME_TAG", await discriminator(self.message.author.name, self.message.author.discriminator), PARSE_MODE_NONE), ("USER_ID", str(self.message.author.id)), ("USER_COLOUR", await self._gather_user_colour(self.message.author)), ("USER_ICON", await self._gather_user_icon(self.message.author), PARSE_MODE_NONE), @@ -312,7 +334,7 @@ async def build_pin_template(self): ("PIN_URL", DiscordUtils.pinned_message_icon, PARSE_MODE_NONE), ("USER_COLOUR", await self._gather_user_colour(self.message.author)), ("NAME", str(html.escape(self.message.author.display_name))), - ("NAME_TAG", "%s#%s" % (self.message.author.name, self.message.author.discriminator), PARSE_MODE_NONE), + ("NAME_TAG", await discriminator(self.message.author.name, self.message.author.discriminator), PARSE_MODE_NONE), ("MESSAGE_ID", str(self.message.id), PARSE_MODE_NONE), ("REF_MESSAGE_ID", str(self.message.reference.message_id), PARSE_MODE_NONE) ]) @@ -324,7 +346,39 @@ async def build_thread_template(self): ("THREAD_NAME", self.message.content, PARSE_MODE_NONE), ("USER_COLOUR", await self._gather_user_colour(self.message.author)), ("NAME", str(html.escape(self.message.author.display_name))), - ("NAME_TAG", "%s#%s" % (self.message.author.name, self.message.author.discriminator), PARSE_MODE_NONE), + ("NAME_TAG", await discriminator(self.message.author.name, self.message.author.discriminator), PARSE_MODE_NONE), + ("MESSAGE_ID", str(self.message.id), PARSE_MODE_NONE), + ]) + + async def build_remove(self): + removed_member: discord.Member = self.message.mentions[0] + self.message_html += await fill_out(self.guild, message_thread_remove, [ + ("THREAD_URL", DiscordUtils.thread_remove_recipient, + PARSE_MODE_NONE), + ("USER_COLOUR", await self._gather_user_colour(self.message.author)), + ("NAME", str(html.escape(self.message.author.display_name))), + ("NAME_TAG", await discriminator(self.message.author.name, self.message.author.discriminator), + PARSE_MODE_NONE), + ("RECIPIENT_USER_COLOUR", await self._gather_user_colour(removed_member)), + ("RECIPIENT_NAME", str(html.escape(removed_member.display_name))), + ("RECIPIENT_NAME_TAG", await discriminator(removed_member.name, removed_member.discriminator), + PARSE_MODE_NONE), + ("MESSAGE_ID", str(self.message.id), PARSE_MODE_NONE), + ]) + + async def build_add(self): + removed_member: discord.Member = self.message.mentions[0] + self.message_html += await fill_out(self.guild, message_thread_add, [ + ("THREAD_URL", DiscordUtils.thread_add_recipient, + PARSE_MODE_NONE), + ("USER_COLOUR", await self._gather_user_colour(self.message.author)), + ("NAME", str(html.escape(self.message.author.display_name))), + ("NAME_TAG", await discriminator(self.message.author.name, self.message.author.discriminator), + PARSE_MODE_NONE), + ("RECIPIENT_USER_COLOUR", await self._gather_user_colour(removed_member)), + ("RECIPIENT_NAME", str(html.escape(removed_member.display_name))), + ("RECIPIENT_NAME_TAG", await discriminator(removed_member.name, removed_member.discriminator), + PARSE_MODE_NONE), ("MESSAGE_ID", str(self.message.id), PARSE_MODE_NONE), ]) @@ -385,6 +439,18 @@ async def gather_messages( meta_data: dict = {} previous_message: Optional[discord.Message] = None + message_dict = {message.id: message for message in messages} + + if "thread" in str(messages[0].channel.type) and messages[0].reference: + channel = guild.get_channel(messages[0].reference.channel_id) + + if not channel: + channel = await guild.fetch_channel(messages[0].reference.channel_id) + + message = await channel.fetch_message(messages[0].reference.message_id) + messages[0] = message + messages[0].reference = None + for message in messages: content_html, meta_data = await MessageConstruct( message, @@ -392,7 +458,8 @@ async def gather_messages( pytz_timezone, military_time, guild, - meta_data + meta_data, + message_dict, ).construct_message() message_html += content_html previous_message = message diff --git a/chat_exporter/construct/transcript.py b/chat_exporter/construct/transcript.py index 1cdbbb4..cab6ebf 100644 --- a/chat_exporter/construct/transcript.py +++ b/chat_exporter/construct/transcript.py @@ -2,6 +2,7 @@ import html import traceback +import re from typing import List, Optional import pytz @@ -81,10 +82,14 @@ async def export_transcript(self, message_html: str, meta_data: str): if meta_data[int(data)][5] else "Unknown" ) + pattern = r'^#\d{4}' + discrim = str(meta_data[int(data)][0][-5:]) + user = str(meta_data[int(data)][0]) + meta_data_html += await fill_out(self.channel.guild, meta_data_temp, [ ("USER_ID", str(data), PARSE_MODE_NONE), - ("USERNAME", str(meta_data[int(data)][0][:-5]), PARSE_MODE_NONE), - ("DISCRIMINATOR", str(meta_data[int(data)][0][-5:])), + ("USERNAME", user[:-5] if re.match(pattern, discrim) else user, PARSE_MODE_NONE), + ("DISCRIMINATOR", discrim if re.match(pattern, discrim) else ""), ("BOT", str(meta_data[int(data)][2]), PARSE_MODE_NONE), ("CREATED_AT", str(creation_time), PARSE_MODE_NONE), ("JOINED_AT", str(joined_time), PARSE_MODE_NONE), @@ -105,7 +110,7 @@ async def export_transcript(self, message_html: str, meta_data: str): channel_topic_html = "" if raw_channel_topic: channel_topic_html = await fill_out(self.channel.guild, channel_topic, [ - ("CHANNEL_TOPIC", raw_channel_topic) + ("CHANNEL_TOPIC", html.escape(raw_channel_topic)) ]) limit = "start" diff --git a/chat_exporter/ext/discord_utils.py b/chat_exporter/ext/discord_utils.py index 2e4e278..8b284a2 100644 --- a/chat_exporter/ext/discord_utils.py +++ b/chat_exporter/ext/discord_utils.py @@ -3,6 +3,8 @@ class DiscordUtils: default_avatar: str = 'https://cdn.jsdelivr.net/gh/mahtoid/DiscordUtils@master/discord-default.png' pinned_message_icon: str = 'https://cdn.jsdelivr.net/gh/mahtoid/DiscordUtils@master/discord-pinned.svg' thread_channel_icon: str = 'https://cdn.jsdelivr.net/gh/mahtoid/DiscordUtils@master/discord-thread.svg' + thread_remove_recipient: str = 'https://cdn.jsdelivr.net/gh/mahtoid/DiscordUtils@master/discord-thread-remove-recipient.svg' + thread_add_recipient: str = 'https://cdn.jsdelivr.net/gh/mahtoid/DiscordUtils@master/discord-thread-add-recipient.svg' file_attachment_audio: str = 'https://cdn.jsdelivr.net/gh/mahtoid/DiscordUtils@master/discord-audio.svg' file_attachment_acrobat: str = 'https://cdn.jsdelivr.net/gh/mahtoid/DiscordUtils@master/discord-acrobat.svg' file_attachment_webcode: str = 'https://cdn.jsdelivr.net/gh/mahtoid/DiscordUtils@master/discord-webcode.svg' diff --git a/chat_exporter/ext/discriminator.py b/chat_exporter/ext/discriminator.py new file mode 100644 index 0000000..be4eb06 --- /dev/null +++ b/chat_exporter/ext/discriminator.py @@ -0,0 +1,4 @@ +async def discriminator(user: str, discriminator: str): + if discriminator != "0": + return f"{user}#{discriminator}" + return user diff --git a/chat_exporter/ext/emoji_convert.py b/chat_exporter/ext/emoji_convert.py index 2dc84cb..2903733 100644 --- a/chat_exporter/ext/emoji_convert.py +++ b/chat_exporter/ext/emoji_convert.py @@ -37,7 +37,7 @@ from chat_exporter.ext.cache import cache -cdn_fmt = "https://cdnjs.cloudflare.com/ajax/libs/twemoji/14.0.2/72x72/{codepoint}.png" +cdn_fmt = "https://cdn.jsdelivr.net/gh/jdecked/twemoji@latest/assets/72x72/{codepoint}.png" @cache() diff --git a/chat_exporter/ext/html_generator.py b/chat_exporter/ext/html_generator.py index 98a4167..e28beb6 100644 --- a/chat_exporter/ext/html_generator.py +++ b/chat_exporter/ext/html_generator.py @@ -55,6 +55,8 @@ def read_file(filename): message_interaction = read_file(dir_path + "/html/message/interaction.html") message_pin = read_file(dir_path + "/html/message/pin.html") message_thread = read_file(dir_path + "/html/message/thread.html") +message_thread_remove = read_file(dir_path + "/html/message/thread_remove.html") +message_thread_add = read_file(dir_path + "/html/message/thread_add.html") message_reference_unknown = read_file(dir_path + "/html/message/reference_unknown.html") message_body = read_file(dir_path + "/html/message/message.html") end_message = read_file(dir_path + "/html/message/end.html") diff --git a/chat_exporter/html/base.html b/chat_exporter/html/base.html index ab9911f..368a2af 100644 --- a/chat_exporter/html/base.html +++ b/chat_exporter/html/base.html @@ -16,81 +16,33 @@