add new websites, update code with new crawling logic

Author: Mindaugas Kasparavicius

.gitignore

@@ -4,4 +4,6 @@ composer.lock
 storage/logs
 storage/cache
 .idea
-vendor
+vendor
+node_modules
+yarn.lock

README.md

@@ -10,21 +10,12 @@ Note: tested on Ubuntu 20.04 only.
 2. Run `sudo apt install sox`
 3. Run `sudo apt install libsox-fmt-mp3`
 4. Run `sudo apt-get install sendmail`
-5. Install required dependencies by running command `composer install`
-6. Copy configuration file by running command `cp .env.example .env`
-7. Set your desired configuration presets for .env file
-8. Start web scan by running `php ./app app:scan:web `
-
-## Architecture
-
-1. app.php (script entry, bootstraps all application code and starts ScanWebCommand)
-2. ScanWebCommand (initiates scan)
-   1. Manager (iterates over website list and initiates crawling)
-      1. Websites[] (goes over website and fetches stock data)
-   2. Mailer
-   3. Logger
-   4. Notification sound
-   
+5. Run `yarn install`
+6. Install required dependencies by running command `composer install`
+7. Copy configuration file by running command `cp .env.example .env`
+8. Set your desired configuration presets for .env file
+9. Start web scan by running `php ./app app:scan:web `
+
 ## Dependencies
 
 1. PHP 7.2
@@ -42,6 +33,8 @@ Note: tested on Ubuntu 20.04 only.
 1. skytech.lt
 2. Varle.lt
 3. kilobaitas.lt
+4. Topocentras.lt
+5. Kaina24.lt
 
 ## How to add more stores support
 

app.php

@@ -6,7 +6,7 @@
 use Symfony\Component\Console\Application;
 use Symfony\Component\Dotenv\Dotenv;
 use App\Container\ContainerBindings;
-use App\Console\ScanWebCommand;
+use App\Console\CrawlCommand;
 
 $dotenv = new Dotenv();
 $dotenv->load(__DIR__.'/.env');
@@ -15,5 +15,5 @@
 $bindings->bind();
 
 $application = new Application();
-$application->add($bindings->getContainer()->get(ScanWebCommand::class));
+$application->add($bindings->getContainer()->get(CrawlCommand::class));
 $application->run();

app/Console/ScanWebCommand.php renamed to app/Console/CrawlCommand.php

@@ -5,18 +5,18 @@
 
 use App\Services\Logger\Logger;
 use App\Services\Mail\Mailer;
-use App\Services\ScanManager\Manager;
-use App\Services\ScanManager\ScanResultInterface;
+use App\Services\CrawlerManager\CrawlerManager;
 use App\Services\Websites\Data\Product;
 use Carbon\Carbon;
+use Psr\Http\Message\UriInterface;
 use Symfony\Component\Console\Command\Command;
 use Symfony\Component\Console\Input\InputInterface;
 use Symfony\Component\Console\Output\OutputInterface;
 use Throwable;
 use Monolog\Logger as MonoLog;
 use Twig\Environment;
 
-class ScanWebCommand extends Command
+class CrawlCommand extends Command
 {
     protected static $defaultName = 'app:scan:web';
 
@@ -26,9 +26,9 @@ class ScanWebCommand extends Command
     private $twig;
     private $recipient;
 
-    private $scanIterations = 0;
+    private $iterations = 0;
 
-    public function __construct(Manager $manager, Logger $logger, Mailer $mailer, Environment $twig, string $emailRecipient)
+    public function __construct(CrawlerManager $manager, Logger $logger, Mailer $mailer, Environment $twig, string $emailRecipient)
     {
         parent::__construct();
         $this->manager = $manager;
@@ -38,24 +38,24 @@ public function __construct(Manager $manager, Logger $logger, Mailer $mailer, En
         $this->recipient = $emailRecipient;
     }
 
-
     protected function execute(InputInterface $input, OutputInterface $output): int
     {
         try {
-            $output->writeln(sprintf('%s: Scanning. Iteration #%s.', Carbon::now()->toDateTimeString(),  ++$this->scanIterations));
+            $output->writeln(sprintf('%s: Crawling. Iteration #%s.', Carbon::now()->toDateTimeString(),  ++$this->iterations));
+
+            $result = [];
+            $this->manager->crawl(
+                function (UriInterface $url, array $products) use (&$result, $output) {
+                    $output->writeln(sprintf('%s: Parsed website %s.', Carbon::now()->toDateTimeString(),  $url->getHost()));
+                    $result = array_merge($result, $products);
+                }
+            );
 
-            $result = $this->manager->scan();
-            if ($result->getProductCount() !== 0) {
-                $output->writeln(sprintf('%s: %s Products found.', Carbon::now()->toDateTimeString(), $result->getProductCount()));
-                $this->parseResult($result);
-                $this->playSound();
-            } else {
-                $output->writeln(sprintf('%s: No updates found yet.', Carbon::now()->toDateTimeString()));
-            }
+            !empty($result)
+                ? $this->notifyAboutProducts($result)
+                : $output->writeln(sprintf('%s: No results found.', Carbon::now()->toDateTimeString()));
 
-            $rescanTime = $this->manager->getRescanTimeSeconds();
-            $output->writeln(sprintf('%s: Sleeping for %s seconds.', Carbon::now()->toDateTimeString(), $rescanTime));
-            sleep($rescanTime);
+            sleep(10);
 
             return $this->execute($input, $output);
         } catch (Throwable $e) {
@@ -66,9 +66,10 @@ protected function execute(InputInterface $input, OutputInterface $output): int
         }
     }
 
-    private function parseResult(ScanResultInterface $result): void
+    private function notifyAboutProducts(array $products): void
     {
-        $products = $result->getProducts();
+        //$this->playSound();
+
         $html = $this->twig->render('index.html', ['products' => $products]);
 
         $this->mailer->send(

app/Container/ContainerBindings.php

@@ -3,16 +3,21 @@
 
 namespace App\Container;
 
-use App\Console\ScanWebCommand;
+use App\Console\CrawlCommand;
 use App\Services\Cache\ProductCache;
+use App\Services\Crawler\CrawlerObserver;
 use App\Services\Logger\Logger;
 use App\Services\Mail\Mailer;
-use App\Services\ScanManager\Manager;
+use App\Services\CrawlerManager\CrawlerManager;
+use App\Services\Websites\Kaina24Website;
+use App\Services\Websites\KilobaitasWebsite;
+use App\Services\Websites\SkytechWebsite;
+use App\Services\Websites\TopoCentrasWebsite;
+use App\Services\Websites\VarleWebsite;
 use Symfony\Component\DependencyInjection\ContainerBuilder;
 use Symfony\Component\DependencyInjection\TaggedContainerInterface;
 use Twig\Environment;
 use Twig\Loader\FilesystemLoader;
-use FilesystemIterator;
 
 class ContainerBindings
 {
@@ -33,59 +38,42 @@ public function bind(): void
         $this->container->register(Logger::class, Logger::class);
         $this->container->register(Mailer::class, Mailer::class);
         $this->container->register(ProductCache::class, ProductCache::class);
+        $this->container
+            ->register(CrawlerObserver::class, CrawlerObserver::class)
+            ->addArgument($this->container->get(ProductCache::class));
+
 
         $loader = new FilesystemLoader(getcwd() . '/views/email');
         $twig = new Environment($loader, ['cache' => getcwd() . '/storage/cache']);
         $this->container->set(get_class($twig), $twig);
 
-        $this->registerManagerWebsites();
-
-        $this->container
-            ->register(ScanWebCommand::class, ScanWebCommand::class)
-            ->addArgument($this->container->get(Manager::getContainerId()))
-            ->addArgument($this->container->get(Logger::class))
-            ->addArgument($this->container->get(Mailer::class))
-            ->addArgument($this->container->get(get_class($twig)))
-            ->addArgument($_ENV['NOTIFICATION_EMAIL'])
-        ;
-    }
-
-    private function registerManagerWebsites(): void
-    {
         $manager = $this->container
-            ->register(Manager::getContainerId(), Manager::class)
+            ->register(CrawlerManager::class, CrawlerManager::class)
             ->addArgument($this->container->get(ProductCache::class))
-            ->addArgument($_ENV['MANAGER_RESCAN_TIME_SECONDS'] ?? 60)
-        ;
-
-        $params = array_filter(explode(',', $_ENV['SCAN_KEYWORDS'] ?? ','));
-        $dir = getcwd() . '/app/Services/Websites';
+            ->addArgument($this->container->get(CrawlerObserver::class))
+            ->addArgument($this->container->get(Logger::class));
 
-        $iterator = new FilesystemIterator($dir);
-        foreach ($iterator as $file) {
-            $filename = $file->getFileName();
-            if ($file->getExtension() !== 'php') {
-                continue;
-            }
-            if (strpos($filename, 'Abstract') !== false) {
-                continue;
-            }
-            if (strpos($filename, 'Interface') !== false) {
-                continue;
-            }
-            if (strpos($filename, 'Website') === false) {
-                continue;
-            }
+        $keywords = array_filter(explode(',', $_ENV['SCAN_KEYWORDS'] ?? ','));
 
-            $fileParts = explode('.', $filename);
-            $objPath = 'App\Services\Websites\\' . $fileParts[0];
+        $websites[] = $this->container->register(VarleWebsite::class, VarleWebsite::class);
+        $websites[] = $this->container->register(KilobaitasWebsite::class, KilobaitasWebsite::class);
+        $websites[] = $this->container->register(SkytechWebsite::class, SkytechWebsite::class);
+        $websites[] = $this->container->register(TopoCentrasWebsite::class, TopoCentrasWebsite::class);
+        $websites[] = $this->container->register(Kaina24Website::class, Kaina24Website::class);
 
-            $website = $this->container->register($objPath, $objPath);
-            foreach ($params as $param) {
-                $website->addMethodCall('addKeyword', [$param]);
-            }
 
-            $manager->addMethodCall('addWebsite', [$this->container->get($objPath)]);
+        foreach ($websites as $website) {
+            $website->addMethodCall('setKeywords', [$keywords]);
         }
+
+        $manager->addMethodCall('setWebsites', [$websites]);
+
+        $this->container
+            ->register(CrawlCommand::class, CrawlCommand::class)
+            ->addArgument($this->container->get(CrawlerManager::class))
+            ->addArgument($this->container->get(Logger::class))
+            ->addArgument($this->container->get(Mailer::class))
+            ->addArgument($this->container->get(get_class($twig)))
+            ->addArgument($_ENV['NOTIFICATION_EMAIL']);
     }
 }

app/Services/Cache/AbstractCache.php

@@ -0,0 +1,29 @@
+<?php
+declare(strict_types=1);
+
+namespace App\Services\Cache;
+
+use Symfony\Component\Cache\Adapter\FilesystemAdapter;
+
+abstract class AbstractCache
+{
+    protected $cache;
+
+    public function __construct()
+    {
+        $this->cache = new FilesystemAdapter($this->getCacheNameSpace(), 1800, $this->getCacheDir());
+    }
+
+    abstract public function getCacheNameSpace(): string;
+
+    protected function getCacheDir(): string
+    {
+        return getcwd() . '/storage/cache';
+    }
+
+    public function clear(): AbstractCache
+    {
+        $this->cache->clear();
+        return $this;
+    }
+}

app/Services/Cache/ProductCache.php

@@ -4,39 +4,28 @@
 namespace App\Services\Cache;
 
 use App\Services\Websites\Data\Product;
-use Symfony\Component\Cache\Adapter\FilesystemAdapter;
 use Psr\Cache\InvalidArgumentException;
 
-class ProductCache
+class ProductCache extends AbstractCache
 {
-    private $cache;
-
-    public function __construct()
+    public function getCacheNameSpace(): string
     {
-        $this->cache = new FilesystemAdapter('products', 1800, $this->getCacheDir());
+        return 'products';
     }
 
     public function getCacheKeyForProduct(Product $product): string
     {
         return md5($product->getUrl());
     }
 
-    public function getCacheDir(): string
-    {
-        return getcwd() . '/storage/cache';
-    }
-
     /**
      * @throws InvalidArgumentException
      */
     public function cacheProduct(Product $product): void
     {
-        $this->cache->get(
-            $this->getCacheKeyForProduct($product),
-            function () use ($product) {
-                return $product;
-            }
-        );
+        $item = $this->cache->getItem($this->getCacheKeyForProduct($product));
+        $item->set($product);
+        $this->cache->save($item);
     }
 
     /**

app/Services/Crawler/AbstractCrawler.php

@@ -0,0 +1,34 @@
+<?php
+declare(strict_types=1);
+
+namespace App\Services\Crawler;
+
+use App\Services\Cache\ProductCache;
+use GuzzleHttp\Exception\RequestException;
+use Psr\Http\Message\ResponseInterface;
+use Psr\Http\Message\UriInterface;
+use Spatie\Crawler\CrawlObserver;
+
+abstract class AbstractCrawler extends CrawlObserver
+{
+    protected $cache;
+    protected $crawledCallback;
+
+    public function __construct(ProductCache $cache)
+    {
+        $this->cache = $cache;
+    }
+
+    public function setCrawledCallback(callable $callback): AbstractCrawler
+    {
+        $this->crawledCallback = $callback;
+        return $this;
+    }
+
+    abstract public function crawled(UriInterface $url, ResponseInterface $response, ?UriInterface $foundOnUrl = null);
+
+    public function crawlFailed(UriInterface $url, RequestException $requestException, ?UriInterface $foundOnUrl = null)
+    {
+        throw $requestException;
+    }
+}