Feat/check max mail size

.coderabbit.yaml

@@ -73,7 +73,3 @@ reviews:
 #  code_guidelines:
 #    filePatterns:
 #      - ".github/AGENT.md"
-
-checks:
-  docstring_coverage:
-    enabled: false

composer.json

@@ -78,7 +78,8 @@
         "webklex/php-imap": "^6.2",
         "ext-imap": "*",
         "tatevikgr/rss-feed": "dev-main",
-        "ext-pdo": "*"
+        "ext-pdo": "*",
+        "ezyang/htmlpurifier": "^4.19"
     },
     "require-dev": {
         "phpunit/phpunit": "^9.5",

config/PhpCodeSniffer/ruleset.xml

@@ -30,7 +30,9 @@
 
     <!-- Commenting -->
     <rule ref="Generic.Commenting.Fixme"/>
-    <rule ref="Generic.Commenting.Todo"/>
+    <rule ref="Generic.Commenting.Todo">
+        <severity>0</severity>
+    </rule>
     <rule ref="PEAR.Commenting.InlineComment"/>
     <rule ref="Squiz.Commenting.DocCommentAlignment"/>
     <rule ref="Squiz.Commenting.EmptyCatchComment"/>

config/parameters.yml.dist

@@ -87,3 +87,5 @@ parameters:
     env(MAILQUEUE_THROTTLE): '5'
     messaging.max_process_time: '%%env(MESSAGING_MAX_PROCESS_TIME)%%'
     env(MESSAGING_MAX_PROCESS_TIME): '600'
+    messaging.max_mail_size: '%%env(MAX_MAILSIZE)%%'
+    env(MAX_MAILSIZE): '209715200'

config/services.yml

@@ -50,3 +50,21 @@ services:
     lazy: true
     tags:
       - { name: 'doctrine.dbal.schema_filter', connection: 'default' }
+
+  HTMLPurifier_Config:
+    class: HTMLPurifier_Config
+    factory: [ 'HTMLPurifier_Config', 'createDefault' ]
+    calls:
+      - [ set, [ 'Cache.SerializerPath', '%kernel.cache_dir%/htmlpurifier' ] ]
+      - [ set, [ 'HTML.ForbiddenElements', [ 'script', 'style' ] ] ]
+      - [ set, [ 'CSS.Disable', true ] ]
+      - [ set, [ 'URI.DisableJavaScript', true ] ]
+      - [ set, [ 'URI.DisableDataURI', true ] ]
+      - [ set, [ 'HTML.Doctype', 'HTML5' ] ]
+      - [ set, [ 'HTML.Allowed', 'p,br,b,strong,i,em,u,a[href|title],ul,ol,li,blockquote,img[src|alt|title],span,div'] ]
+
+  HTMLPurifier:
+    class: HTMLPurifier
+    arguments:
+      - '@HTMLPurifier_Config'
+

config/services/managers.yml

@@ -110,3 +110,7 @@ services:
   PhpList\Core\Domain\Messaging\Service\Manager\SendProcessManager:
     autowire: true
     autoconfigure: true
+
+  PhpList\Core\Domain\Messaging\Service\Manager\MessageDataManager:
+    autowire: true
+    autoconfigure: true

config/services/messenger.yml

@@ -29,6 +29,11 @@ services:
         autoconfigure: true
         tags: [ 'messenger.message_handler' ]
 
+    PhpList\Core\Domain\Messaging\MessageHandler\CampaignProcessorMessageHandler:
+        autowire: true
+        arguments:
+            $maxMailSize: '%messaging.max_mail_size%'
+
     PhpList\Core\Domain\Subscription\MessageHandler\DynamicTableMessageHandler:
         autowire: true
         autoconfigure: true

config/services/repositories.yml

@@ -140,3 +140,8 @@ services:
     parent: PhpList\Core\Domain\Common\Repository\AbstractRepository
     arguments:
       - PhpList\Core\Domain\Messaging\Model\SendProcess
+
+  PhpList\Core\Domain\Messaging\Repository\MessageDataRepository:
+    parent: PhpList\Core\Domain\Common\Repository\AbstractRepository
+    arguments:
+      - PhpList\Core\Domain\Messaging\Model\MessageData

src/Domain/Analytics/Service/LinkTrackService.php

@@ -9,6 +9,7 @@
 use PhpList\Core\Domain\Analytics\Model\LinkTrack;
 use PhpList\Core\Domain\Analytics\Repository\LinkTrackRepository;
 use PhpList\Core\Domain\Messaging\Model\Message;
+use PhpList\Core\Domain\Messaging\Model\Message\MessageContent;
 
 class LinkTrackService
 {
@@ -38,15 +39,12 @@ public function isExtractAndSaveLinksApplicable(): bool
      * @return LinkTrack[] The saved LinkTrack entities
      * @throws MissingMessageIdException
      */
-    public function extractAndSaveLinks(Message $message, int $userId): array
+    public function extractAndSaveLinks(MessageContent $content, int $userId, ?int $messageId = null): array
     {
         if (!$this->isExtractAndSaveLinksApplicable()) {
             return [];
         }
 
-        $content = $message->getContent();
-        $messageId = $message->getId();
-
         if ($messageId === null) {
             throw new MissingMessageIdException();
         }

src/Domain/Identity/Service/PermissionChecker.php

@@ -70,6 +70,8 @@ private function resolveRelatedEntity(DomainModel $resource, string $relatedClas
         }
 
         if ($resource instanceof Message && $relatedClass === SubscriberList::class) {
+            // todo: check which one is correct
+            // return $resource->getListMessages()->map(fn(ListMessage $lm) => $lm->getList())->toArray();
             return $resource->getListMessages()->map(fn($lm) => $lm->getSubscriberList())->toArray();
         }
 

src/Domain/Messaging/Exception/MessageSizeLimitExceededException.php

@@ -0,0 +1,31 @@
+<?php
+
+declare(strict_types=1);
+
+namespace PhpList\Core\Domain\Messaging\Exception;
+
+use RuntimeException;
+
+class MessageSizeLimitExceededException extends RuntimeException
+{
+    public function __construct(
+        private readonly int $actualSize,
+        private readonly int $maxSize
+    ) {
+        parent::__construct(sprintf(
+            'Message too large (%d bytes exceeds limit of %d bytes)',
+            $actualSize,
+            $maxSize
+        ));
+    }
+
+    public function getActualSize(): int
+    {
+        return $this->actualSize;
+    }
+
+    public function getMaxSize(): int
+    {
+        return $this->maxSize;
+    }
+}

src/Domain/Messaging/MessageHandler/CampaignProcessorMessageHandler.php

@@ -5,6 +5,7 @@
 namespace PhpList\Core\Domain\Messaging\MessageHandler;
 
 use Doctrine\ORM\EntityManagerInterface;
+use PhpList\Core\Domain\Messaging\Exception\MessageSizeLimitExceededException;
 use PhpList\Core\Domain\Messaging\Message\CampaignProcessorMessage;
 use PhpList\Core\Domain\Messaging\Message\SyncCampaignProcessorMessage;
 use PhpList\Core\Domain\Messaging\Model\Message;
@@ -14,14 +15,19 @@
 use PhpList\Core\Domain\Messaging\Repository\MessageRepository;
 use PhpList\Core\Domain\Messaging\Repository\UserMessageRepository;
 use PhpList\Core\Domain\Messaging\Service\Handler\RequeueHandler;
+use PhpList\Core\Domain\Messaging\Service\Manager\MessageDataManager;
 use PhpList\Core\Domain\Messaging\Service\MaxProcessTimeLimiter;
+use PhpList\Core\Domain\Messaging\Service\MessagePrecacheService;
 use PhpList\Core\Domain\Messaging\Service\MessageProcessingPreparator;
 use PhpList\Core\Domain\Messaging\Service\RateLimitedCampaignMailer;
+use PhpList\Core\Domain\Configuration\Service\Manager\EventLogManager;
 use PhpList\Core\Domain\Subscription\Model\Subscriber;
 use PhpList\Core\Domain\Subscription\Service\Manager\SubscriberHistoryManager;
 use PhpList\Core\Domain\Subscription\Service\Provider\SubscriberProvider;
 use Psr\Log\LoggerInterface;
+use Psr\SimpleCache\CacheInterface;
 use Symfony\Component\Messenger\Attribute\AsMessageHandler;
+use Symfony\Component\Mime\Email;
 use Symfony\Contracts\Translation\TranslatorInterface;
 use Throwable;
 
@@ -32,42 +38,27 @@
 #[AsMessageHandler]
 class CampaignProcessorMessageHandler
 {
-    private RateLimitedCampaignMailer $mailer;
-    private EntityManagerInterface $entityManager;
-    private SubscriberProvider $subscriberProvider;
-    private MessageProcessingPreparator $messagePreparator;
-    private LoggerInterface $logger;
-    private UserMessageRepository $userMessageRepository;
-    private MaxProcessTimeLimiter $timeLimiter;
-    private RequeueHandler $requeueHandler;
-    private TranslatorInterface $translator;
-    private SubscriberHistoryManager $subscriberHistoryManager;
-    private MessageRepository $messageRepository;
+    private ?int $maxMailSize;
 
     public function __construct(
-        RateLimitedCampaignMailer $mailer,
-        EntityManagerInterface $entityManager,
-        SubscriberProvider $subscriberProvider,
-        MessageProcessingPreparator $messagePreparator,
-        LoggerInterface $logger,
-        UserMessageRepository $userMessageRepository,
-        MaxProcessTimeLimiter $timeLimiter,
-        RequeueHandler $requeueHandler,
-        TranslatorInterface $translator,
-        SubscriberHistoryManager $subscriberHistoryManager,
-        MessageRepository $messageRepository,
+        private readonly RateLimitedCampaignMailer $mailer,
+        private readonly EntityManagerInterface $entityManager,
+        private readonly SubscriberProvider $subscriberProvider,
+        private readonly MessageProcessingPreparator $messagePreparator,
+        private readonly LoggerInterface $logger,
+        private readonly CacheInterface $cache,
+        private readonly UserMessageRepository $userMessageRepository,
+        private readonly MaxProcessTimeLimiter $timeLimiter,
+        private readonly RequeueHandler $requeueHandler,
+        private readonly TranslatorInterface $translator,
+        private readonly SubscriberHistoryManager $subscriberHistoryManager,
+        private readonly MessageRepository $messageRepository,
+        private readonly EventLogManager $eventLogManager,
+        private readonly MessageDataManager $messageDataManager,
+        private readonly MessagePrecacheService $precacheService,
+        ?int $maxMailSize = null,
     ) {
-        $this->mailer = $mailer;
-        $this->entityManager = $entityManager;
-        $this->subscriberProvider = $subscriberProvider;
-        $this->messagePreparator = $messagePreparator;
-        $this->logger = $logger;
-        $this->userMessageRepository = $userMessageRepository;
-        $this->timeLimiter = $timeLimiter;
-        $this->requeueHandler = $requeueHandler;
-        $this->translator = $translator;
-        $this->subscriberHistoryManager = $subscriberHistoryManager;
-        $this->messageRepository = $messageRepository;
+        $this->maxMailSize = $maxMailSize ?? 0;
     }
 
     public function __invoke(CampaignProcessorMessage|SyncCampaignProcessorMessage $message): void
@@ -82,6 +73,8 @@ public function __invoke(CampaignProcessorMessage|SyncCampaignProcessorMessage $
             return;
         }
 
+        $messageContent = $this->precacheService->getOrCacheBaseMessageContent($campaign);
+
         $this->updateMessageStatus($campaign, MessageStatus::Prepared);
         $subscribers = $this->subscriberProvider->getSubscribersForMessage($campaign);
 
@@ -111,7 +104,7 @@ public function __invoke(CampaignProcessorMessage|SyncCampaignProcessorMessage $
                 continue;
             }
 
-            $this->handleEmailSending($campaign, $subscriber, $userMessage);
+            $this->handleEmailSending($campaign, $subscriber, $userMessage, $messageContent);
         }
 
         if ($stoppedEarly && $this->requeueHandler->handle($campaign)) {
@@ -142,7 +135,7 @@ private function updateUserMessageStatus(UserMessage $userMessage, UserMessageSt
         $this->entityManager->flush();
     }
 
-    private function handleInvalidEmail(UserMessage $userMessage, Subscriber $subscriber, mixed $campaign): void
+    private function handleInvalidEmail(UserMessage $userMessage, Subscriber $subscriber, Message $campaign): void
     {
         $this->updateUserMessageStatus($userMessage, UserMessageStatus::InvalidEmailAddress);
         $this->unconfirmSubscriber($subscriber);
@@ -159,14 +152,25 @@ private function handleInvalidEmail(UserMessage $userMessage, Subscriber $subscr
         );
     }
 
-    private function handleEmailSending(mixed $campaign, Subscriber $subscriber, UserMessage $userMessage): void
-    {
-        $processed = $this->messagePreparator->processMessageLinks($campaign, $subscriber->getId());
+    private function handleEmailSending(
+        Message $campaign,
+        Subscriber $subscriber,
+        UserMessage $userMessage,
+        Message\MessageContent $precachedContent,
+    ): void {
+        $processed = $this->messagePreparator->processMessageLinks($campaign->getId(), $precachedContent, $subscriber);
 
         try {
-            $email = $this->mailer->composeEmail($processed, $subscriber);
+            $email = $this->mailer->composeEmail($campaign, $subscriber, $processed);
             $this->mailer->send($email);
+            $this->checkMessageSizeOrSuspendCampaign($campaign, $email, $subscriber->hasHtmlEmail());
+            $this->updateUserMessageStatus($userMessage, UserMessageStatus::Sent);
+        } catch (MessageSizeLimitExceededException $e) {
+            // stop after the first message if size is exceeded
+            $this->updateMessageStatus($campaign, MessageStatus::Suspended);
             $this->updateUserMessageStatus($userMessage, UserMessageStatus::Sent);
+
+            throw $e;
         } catch (Throwable $e) {
             $this->updateUserMessageStatus($userMessage, UserMessageStatus::NotSent);
             $this->logger->error($e->getMessage(), [
@@ -178,4 +182,57 @@ private function handleEmailSending(mixed $campaign, Subscriber $subscriber, Use
             ]));
         }
     }
+
+    private function checkMessageSizeOrSuspendCampaign(
+        Message $campaign,
+        Email $email,
+        bool $hasHtmlEmail
+    ): void {
+        if ($this->maxMailSize <= 0) {
+            return;
+        }
+        $sizeName = $hasHtmlEmail ? 'htmlsize' : 'textsize';
+        $cacheKey = sprintf('messaging.size.%d.%s', $campaign->getId(), $sizeName);
+        if (!$this->cache->has($cacheKey)) {
+            $size = $this->calculateEmailSize($email);
+            $this->messageDataManager->setMessageData($campaign, $sizeName, $size);
+            $this->cache->set($cacheKey, $size);
+        }
+
+        $size = $this->cache->get($cacheKey);
+        if ($size <= $this->maxMailSize) {
+            return;
+        }
+
+        $this->logger->warning(sprintf(
+            'Message too large (%d is over %d), suspending campaign %d',
+            $size,
+            $this->maxMailSize,
+            $campaign->getId()
+        ));
+
+        $this->eventLogManager->log('send', sprintf(
+            'Message too large (%d is over %d), suspending',
+            $size,
+            $this->maxMailSize
+        ));
+
+        $this->eventLogManager->log('send', sprintf(
+            'Campaign %d suspended. Message too large',
+            $campaign->getId()
+        ));
+
+        throw new MessageSizeLimitExceededException($size, $this->maxMailSize);
+    }
+
+    private function calculateEmailSize(Email $email): int
+    {
+        $size = 0;
+
+        foreach ($email->toIterable() as $line) {
+            $size += strlen($line);
+        }
+
+        return $size;
+    }
 }

src/Domain/Messaging/Repository/MessageDataRepository.php

@@ -7,8 +7,14 @@
 use PhpList\Core\Domain\Common\Repository\AbstractRepository;
 use PhpList\Core\Domain\Common\Repository\CursorPaginationTrait;
 use PhpList\Core\Domain\Common\Repository\Interfaces\PaginatableRepositoryInterface;
+use PhpList\Core\Domain\Messaging\Model\MessageData;
 
 class MessageDataRepository extends AbstractRepository implements PaginatableRepositoryInterface
 {
     use CursorPaginationTrait;
+
+    public function findByIdAndName(int $messageId, string $name): ?MessageData
+    {
+        return $this->findOneBy(['id' => $messageId, 'name' => $name]);
+    }
 }