DiscordChatExporterPy 2.6.0 (#89)

* Set default timezone just in case failed to setattr

* Updated discord links

* Font and Styling adjustments

* Development branch solve (#83)

* HTML Escape Embeds and get topic only if text channel

* README.md update

* Sped up references, updated README, updated twemoji link and more

---------

Co-authored-by: mahtoid <[email protected]>

* Reference speed up and solves (#84)

* HTML Escape Embeds and get topic only if text channel

* README.md update

* Sped up references, updated README, updated twemoji link and more

* Reference speed up and solves

---------

Co-authored-by: mahtoid <[email protected]>

* Solves for username update and threads

* Thread improvements and thread add support (minus SVG)

* Removed bottom padding on emoji--small

* Headers and Order List support

* Split imports

---------

Co-authored-by: mahtoid <[email protected]>

.github/ISSUE_TEMPLATE/config.yml

@@ -1,5 +1,5 @@
 blank_issues_enabled: false
 contact_links:
   - name: Discord
-    url: https://discord.gg/2uhHBQDwcc
+    url: https://discord.mahto.id/
     about: Find help within the Discord community

README.md

@@ -13,7 +13,7 @@
   <p>
     Export Discord chats with your discord.py (or fork) bots!
     <br />
-    <a href="https://discord.gg/2uhHBQDwcc">Join Discord</a>
+    <a href="https://discord.mahto.id/">Join Discord</a>
     ·
     <a href="https://github.com/mahtoid/DiscordChatExporterPy/issues/new?assignees=&labels=bug&template=bug-report.yml">Report Bug</a>
     ·
@@ -95,8 +95,11 @@ This would be the main function to use within chat-exporter.
 **Optional Argument(s):**<br/>
 `limit`: Integer value to set the limit (amount of messages) the chat exporter gathers when grabbing the history (default=unlimited).<br/>
 `tz_info`: String value of a [TZ Database name](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones#List) to set a custom timezone for the exported messages (default=UTC)<br/>
+`guild`: `discord.Guild` object which can be passed in to solve bugs for certain forks<br/>
 `military_time`: Boolean value to set a 24h format for times within your exported chat (default=False | 12h format)<br/>
 `fancy_times`: Boolean value which toggles the 'fancy times' (Today|Yesterday|Day)<br/>
+`before`: `datetime.datetime` object which allows to gather messages from before a certain date
+`after`: `datetime.datetime` object which allows to gather messages from after a certain date
 `bot`: `commands.Bot` object to gather members who are no longer in your guild.
 
 **Return Argument:**<br/>

chat_exporter/construct/assets/embed.py

@@ -106,16 +106,19 @@ async def build_fields(self):
                     ("FIELD_VALUE", field.value, PARSE_MODE_EMBED)])
 
     async def build_author(self):
-        self.author = html.escape(self.embed.author.name) if self.embed.author.name != self.check_against else ""
+        self.author = html.escape(self.embed.author.name) if (
+                self.embed.author and self.embed.author.name != self.check_against
+        ) else ""
 
         self.author = f'<a class="chatlog__embed-author-name-link" href="{self.embed.author.url}">{self.author}</a>' \
-            if self.embed.author.url != self.check_against \
-            else self.author
+            if (
+                self.embed.author and self.embed.author.url != self.check_against
+            ) else self.author
 
         author_icon = await fill_out(self.guild, embed_author_icon, [
             ("AUTHOR", self.author, PARSE_MODE_NONE),
             ("AUTHOR_ICON", self.embed.author.icon_url, PARSE_MODE_NONE)
-        ]) if self.embed.author.icon_url != self.check_against else ""
+        ]) if self.embed.author and self.embed.author.icon_url != self.check_against else ""
 
         if author_icon == "" and self.author != "":
             self.author = await fill_out(self.guild, embed_author, [("AUTHOR", self.author, PARSE_MODE_NONE)])
@@ -125,16 +128,21 @@ async def build_author(self):
     async def build_image(self):
         self.image = await fill_out(self.guild, embed_image, [
             ("EMBED_IMAGE", str(self.embed.image.proxy_url), PARSE_MODE_NONE)
-        ]) if self.embed.image.url != self.check_against else ""
+        ]) if self.embed.image and self.embed.image.url != self.check_against else ""
 
     async def build_thumbnail(self):
         self.thumbnail = await fill_out(self.guild, embed_thumbnail, [
             ("EMBED_THUMBNAIL", str(self.embed.thumbnail.url), PARSE_MODE_NONE)]) \
-            if self.embed.thumbnail.url != self.check_against else ""
+            if self.embed.thumbnail and self.embed.thumbnail.url != self.check_against else ""
 
     async def build_footer(self):
-        self.footer = html.escape(self.embed.footer.text) if self.embed.footer.text != self.check_against else ""
-        footer_icon = self.embed.footer.icon_url if self.embed.footer.icon_url != self.check_against else None
+        self.footer = html.escape(self.embed.footer.text) if (
+                self.embed.footer and self.embed.footer.text != self.check_against
+        ) else ""
+
+        footer_icon = self.embed.footer.icon_url if (
+                self.embed.footer and self.embed.footer.icon_url != self.check_against
+        ) else None
 
         if not self.footer:
             return

chat_exporter/construct/message.py

@@ -8,6 +8,7 @@
 
 from chat_exporter.construct.assets import Attachment, Component, Embed, Reaction
 from chat_exporter.ext.discord_utils import DiscordUtils
+from chat_exporter.ext.discriminator import discriminator
 from chat_exporter.ext.html_generator import (
     fill_out,
     bot_tag,
@@ -25,6 +26,8 @@
     PARSE_MODE_NONE,
     PARSE_MODE_MARKDOWN,
     PARSE_MODE_REFERENCE,
+    message_thread_remove,
+    message_thread_add,
 )
 
 
@@ -37,7 +40,7 @@ def _gather_user_bot(author: discord.Member):
 
 
 def _set_edit_at(message_edited_at):
-    return f'<span class="chatlog__reference-edited-timestamp" title="{message_edited_at}">(edited)</span>'
+    return f'<span class="chatlog__reference-edited-timestamp" data-timestamp="{message_edited_at}">(edited)</span>'
 
 
 class MessageConstruct:
@@ -57,13 +60,15 @@ def __init__(
         pytz_timezone,
         military_time: bool,
         guild: discord.Guild,
-        meta_data: dict
+        meta_data: dict,
+        message_dict: dict
     ):
         self.message = message
         self.previous_message = previous_message
         self.pytz_timezone = pytz_timezone
         self.military_time = military_time
         self.guild = guild
+        self.message_dict = message_dict
 
         self.time_format = "%A, %e %B %Y %I:%M %p"
         if self.military_time:
@@ -79,6 +84,10 @@ async def construct_message(
             await self.build_pin()
         elif discord.MessageType.thread_created == self.message.type:
             await self.build_thread()
+        elif discord.MessageType.recipient_remove == self.message.type:
+            await self.build_thread_remove()
+        elif discord.MessageType.recipient_add == self.message.type:
+            await self.build_thread_add()
         else:
             await self.build_message()
         return self.message_html, self.meta_data
@@ -100,13 +109,21 @@ async def build_thread(self):
         await self.generate_message_divider(channel_audit=True)
         await self.build_thread_template()
 
+    async def build_thread_remove(self):
+        await self.generate_message_divider(channel_audit=True)
+        await self.build_remove()
+
+    async def build_thread_add(self):
+        await self.generate_message_divider(channel_audit=True)
+        await self.build_add()
+
     async def build_meta_data(self):
         user_id = self.message.author.id
 
         if user_id in self.meta_data:
             self.meta_data[user_id][4] += 1
         else:
-            user_name_discriminator = self.message.author.name + "#" + self.message.author.discriminator
+            user_name_discriminator = await discriminator(self.message.author.name, self.message.author.discriminator)
             user_created_at = self.message.author.created_at
             user_bot = _gather_user_bot(self.message.author)
             user_avatar = (
@@ -142,13 +159,16 @@ async def build_reference(self):
             self.message.reference = ""
             return
 
-        try:
-            message: discord.Message = await self.message.channel.fetch_message(self.message.reference.message_id)
-        except (discord.NotFound, discord.HTTPException) as e:
-            self.message.reference = ""
-            if isinstance(e, discord.NotFound):
-                self.message.reference = message_reference_unknown
-            return
+        message: discord.Message = self.message_dict.get(self.message.reference.message_id)
+
+        if not message:
+            try:
+                message: discord.Message = await self.message.channel.fetch_message(self.message.reference.message_id)
+            except (discord.NotFound, discord.HTTPException) as e:
+                self.message.reference = ""
+                if isinstance(e, discord.NotFound):
+                    self.message.reference = message_reference_unknown
+                return
 
         is_bot = _gather_user_bot(message.author)
         user_colour = await self._gather_user_colour(message.author)
@@ -173,7 +193,7 @@ async def build_reference(self):
         self.message.reference = await fill_out(self.guild, message_reference, [
             ("AVATAR_URL", str(avatar_url), PARSE_MODE_NONE),
             ("BOT_TAG", is_bot, PARSE_MODE_NONE),
-            ("NAME_TAG", "%s#%s" % (message.author.name, message.author.discriminator), PARSE_MODE_NONE),
+            ("NAME_TAG", await discriminator(message.author.name, message.author.discriminator), PARSE_MODE_NONE),
             ("NAME", str(html.escape(message.author.display_name))),
             ("USER_COLOUR", user_colour, PARSE_MODE_NONE),
             ("CONTENT", message.content, PARSE_MODE_REFERENCE),
@@ -195,7 +215,7 @@ async def build_interaction(self):
         self.message.interaction = await fill_out(self.guild, message_interaction, [
             ("AVATAR_URL", str(avatar_url), PARSE_MODE_NONE),
             ("BOT_TAG", is_bot, PARSE_MODE_NONE),
-            ("NAME_TAG", "%s#%s" % (user.name, user.discriminator), PARSE_MODE_NONE),
+            ("NAME_TAG", await discriminator(user.name, user.discriminator), PARSE_MODE_NONE),
             ("NAME", str(html.escape(user.display_name))),
             ("USER_COLOUR", user_colour, PARSE_MODE_NONE),
             ("FILLER", "used ", PARSE_MODE_NONE),
@@ -258,7 +278,8 @@ async def build_message_template(self):
 
     def _generate_message_divider_check(self):
         return bool(
-            self.previous_message is None or self.message.reference != "" or self.message.interaction != "" or
+            self.previous_message is None or self.message.reference != "" or
+            self.previous_message.type is not discord.MessageType.default or self.message.interaction != "" or
             self.previous_message.author.id != self.message.author.id or self.message.webhook_id is not None or
             self.message.created_at > (self.previous_message.created_at + timedelta(minutes=4))
         )
@@ -269,6 +290,7 @@ async def generate_message_divider(self, channel_audit=False):
                 self.message_html += await fill_out(self.guild, end_message, [])
 
             if channel_audit:
+                self.audit = True
                 return
 
             followup_symbol = ""
@@ -289,7 +311,7 @@ async def generate_message_divider(self, channel_audit=False):
                 ("REFERENCE", self.message.reference if self.message.reference else self.message.interaction,
                  PARSE_MODE_NONE),
                 ("AVATAR_URL", str(avatar_url), PARSE_MODE_NONE),
-                ("NAME_TAG", "%s#%s" % (self.message.author.name, self.message.author.discriminator), PARSE_MODE_NONE),
+                ("NAME_TAG", await discriminator(self.message.author.name, self.message.author.discriminator), PARSE_MODE_NONE),
                 ("USER_ID", str(self.message.author.id)),
                 ("USER_COLOUR", await self._gather_user_colour(self.message.author)),
                 ("USER_ICON", await self._gather_user_icon(self.message.author), PARSE_MODE_NONE),
@@ -312,7 +334,7 @@ async def build_pin_template(self):
             ("PIN_URL", DiscordUtils.pinned_message_icon, PARSE_MODE_NONE),
             ("USER_COLOUR", await self._gather_user_colour(self.message.author)),
             ("NAME", str(html.escape(self.message.author.display_name))),
-            ("NAME_TAG", "%s#%s" % (self.message.author.name, self.message.author.discriminator), PARSE_MODE_NONE),
+            ("NAME_TAG", await discriminator(self.message.author.name, self.message.author.discriminator), PARSE_MODE_NONE),
             ("MESSAGE_ID", str(self.message.id), PARSE_MODE_NONE),
             ("REF_MESSAGE_ID", str(self.message.reference.message_id), PARSE_MODE_NONE)
         ])
@@ -324,7 +346,39 @@ async def build_thread_template(self):
             ("THREAD_NAME", self.message.content, PARSE_MODE_NONE),
             ("USER_COLOUR", await self._gather_user_colour(self.message.author)),
             ("NAME", str(html.escape(self.message.author.display_name))),
-            ("NAME_TAG", "%s#%s" % (self.message.author.name, self.message.author.discriminator), PARSE_MODE_NONE),
+            ("NAME_TAG", await discriminator(self.message.author.name, self.message.author.discriminator), PARSE_MODE_NONE),
+            ("MESSAGE_ID", str(self.message.id), PARSE_MODE_NONE),
+        ])
+
+    async def build_remove(self):
+        removed_member: discord.Member = self.message.mentions[0]
+        self.message_html += await fill_out(self.guild, message_thread_remove, [
+            ("THREAD_URL", DiscordUtils.thread_remove_recipient,
+             PARSE_MODE_NONE),
+            ("USER_COLOUR", await self._gather_user_colour(self.message.author)),
+            ("NAME", str(html.escape(self.message.author.display_name))),
+            ("NAME_TAG", await discriminator(self.message.author.name, self.message.author.discriminator),
+             PARSE_MODE_NONE),
+            ("RECIPIENT_USER_COLOUR", await self._gather_user_colour(removed_member)),
+            ("RECIPIENT_NAME", str(html.escape(removed_member.display_name))),
+            ("RECIPIENT_NAME_TAG", await discriminator(removed_member.name, removed_member.discriminator),
+             PARSE_MODE_NONE),
+            ("MESSAGE_ID", str(self.message.id), PARSE_MODE_NONE),
+        ])
+
+    async def build_add(self):
+        removed_member: discord.Member = self.message.mentions[0]
+        self.message_html += await fill_out(self.guild, message_thread_add, [
+            ("THREAD_URL", DiscordUtils.thread_add_recipient,
+             PARSE_MODE_NONE),
+            ("USER_COLOUR", await self._gather_user_colour(self.message.author)),
+            ("NAME", str(html.escape(self.message.author.display_name))),
+            ("NAME_TAG", await discriminator(self.message.author.name, self.message.author.discriminator),
+             PARSE_MODE_NONE),
+            ("RECIPIENT_USER_COLOUR", await self._gather_user_colour(removed_member)),
+            ("RECIPIENT_NAME", str(html.escape(removed_member.display_name))),
+            ("RECIPIENT_NAME_TAG", await discriminator(removed_member.name, removed_member.discriminator),
+             PARSE_MODE_NONE),
             ("MESSAGE_ID", str(self.message.id), PARSE_MODE_NONE),
         ])
 
@@ -385,14 +439,27 @@ async def gather_messages(
     meta_data: dict = {}
     previous_message: Optional[discord.Message] = None
 
+    message_dict = {message.id: message for message in messages}
+
+    if "thread" in str(messages[0].channel.type) and messages[0].reference:
+        channel = guild.get_channel(messages[0].reference.channel_id)
+
+        if not channel:
+            channel = await guild.fetch_channel(messages[0].reference.channel_id)
+
+        message = await channel.fetch_message(messages[0].reference.message_id)
+        messages[0] = message
+        messages[0].reference = None
+
     for message in messages:
         content_html, meta_data = await MessageConstruct(
             message,
             previous_message,
             pytz_timezone,
             military_time,
             guild,
-            meta_data
+            meta_data,
+            message_dict,
         ).construct_message()
         message_html += content_html
         previous_message = message

chat_exporter/construct/transcript.py

@@ -2,6 +2,7 @@
 import html
 import traceback
 
+import re
 from typing import List, Optional
 
 import pytz
@@ -81,10 +82,14 @@ async def export_transcript(self, message_html: str, meta_data: str):
                 if meta_data[int(data)][5] else "Unknown"
             )
 
+            pattern = r'^#\d{4}'
+            discrim = str(meta_data[int(data)][0][-5:])
+            user = str(meta_data[int(data)][0])
+
             meta_data_html += await fill_out(self.channel.guild, meta_data_temp, [
                 ("USER_ID", str(data), PARSE_MODE_NONE),
-                ("USERNAME", str(meta_data[int(data)][0][:-5]), PARSE_MODE_NONE),
-                ("DISCRIMINATOR", str(meta_data[int(data)][0][-5:])),
+                ("USERNAME", user[:-5] if re.match(pattern, discrim) else user, PARSE_MODE_NONE),
+                ("DISCRIMINATOR", discrim if re.match(pattern, discrim) else ""),
                 ("BOT", str(meta_data[int(data)][2]), PARSE_MODE_NONE),
                 ("CREATED_AT", str(creation_time), PARSE_MODE_NONE),
                 ("JOINED_AT", str(joined_time), PARSE_MODE_NONE),
@@ -105,7 +110,7 @@ async def export_transcript(self, message_html: str, meta_data: str):
         channel_topic_html = ""
         if raw_channel_topic:
             channel_topic_html = await fill_out(self.channel.guild, channel_topic, [
-                ("CHANNEL_TOPIC", raw_channel_topic)
+                ("CHANNEL_TOPIC", html.escape(raw_channel_topic))
             ])
 
         limit = "start"

chat_exporter/ext/discord_utils.py

@@ -3,6 +3,8 @@ class DiscordUtils:
     default_avatar: str = 'https://cdn.jsdelivr.net/gh/mahtoid/DiscordUtils@master/discord-default.png'
     pinned_message_icon: str = 'https://cdn.jsdelivr.net/gh/mahtoid/DiscordUtils@master/discord-pinned.svg'
     thread_channel_icon: str = 'https://cdn.jsdelivr.net/gh/mahtoid/DiscordUtils@master/discord-thread.svg'
+    thread_remove_recipient: str = 'https://cdn.jsdelivr.net/gh/mahtoid/DiscordUtils@master/discord-thread-remove-recipient.svg'
+    thread_add_recipient: str = 'https://cdn.jsdelivr.net/gh/mahtoid/DiscordUtils@master/discord-thread-add-recipient.svg'
     file_attachment_audio: str = 'https://cdn.jsdelivr.net/gh/mahtoid/DiscordUtils@master/discord-audio.svg'
     file_attachment_acrobat: str = 'https://cdn.jsdelivr.net/gh/mahtoid/DiscordUtils@master/discord-acrobat.svg'
     file_attachment_webcode: str = 'https://cdn.jsdelivr.net/gh/mahtoid/DiscordUtils@master/discord-webcode.svg'

chat_exporter/ext/discriminator.py

@@ -0,0 +1,4 @@
+async def discriminator(user: str, discriminator: str):
+    if discriminator != "0":
+        return f"{user}#{discriminator}"
+    return user

chat_exporter/ext/emoji_convert.py

@@ -37,7 +37,7 @@
 from chat_exporter.ext.cache import cache
 
 
-cdn_fmt = "https://cdnjs.cloudflare.com/ajax/libs/twemoji/14.0.2/72x72/{codepoint}.png"
+cdn_fmt = "https://cdn.jsdelivr.net/gh/jdecked/twemoji@latest/assets/72x72/{codepoint}.png"
 
 
 @cache()

chat_exporter/ext/html_generator.py

@@ -55,6 +55,8 @@ def read_file(filename):
 message_interaction = read_file(dir_path + "/html/message/interaction.html")
 message_pin = read_file(dir_path + "/html/message/pin.html")
 message_thread = read_file(dir_path + "/html/message/thread.html")
+message_thread_remove = read_file(dir_path + "/html/message/thread_remove.html")
+message_thread_add = read_file(dir_path + "/html/message/thread_add.html")
 message_reference_unknown = read_file(dir_path + "/html/message/reference_unknown.html")
 message_body = read_file(dir_path + "/html/message/message.html")
 end_message = read_file(dir_path + "/html/message/end.html")