static build using crawler

composer.json

@@ -18,6 +18,7 @@
     "require": {
         "php": "^8.1",
         "illuminate/contracts": "^9.0",
+        "spatie/crawler": "^7.1",
         "spatie/laravel-package-tools": "^1.13.0",
         "voku/html-min": "^4.5"
     },

config/static.php

@@ -11,17 +11,26 @@
      */
     'fallback_cache' => 'memcached',
 
-    /**
-     * Clear static files before warming up static cache.
-     * When disabled, the cache is warmed up rather by updating and overwriting files instead of starting without an existing cache.
-     */
-    'clear_before_warm_up' => false,
+    'build' => [
+        /**
+         * Clear static files before building static cache.
+         * When disabled, the cache is warmed up rather by updating and overwriting files instead of starting without an existing cache.
+         */
+        'clear_before_start' => false,
 
-    /**
-     * HTTP Header that is being sent to web server by warm up command, to recognize and pass through static cache and
-     * hit the Laravel application on the server.
-     */
-    'warm_up_http_header' => 'X-Laravel-Static',
+        /**
+         * Concurrency for crawling to warm up static cache.
+         */
+        'concurrency' => 5,
+
+        /**
+         * HTTP header that can be used to bypass the cache. Useful for updating the cache without needing to clear it first,
+         * or to monitor the performance of your application.
+         */
+        'bypass_header' => [
+            'X-Laravel-Static' => 'off',
+        ],
+    ],
 
     /**
      * Different caches per domain.
@@ -37,7 +46,7 @@
     /**
      * Define if you want to save the static cache after response has been sent to browser.
      */
-    'on_termination' => false,
+    'on_termination' => true,
 
     /**
      * This setting prevents saving the same static cache file twice (with and without trailing slash) using a 302 redirect.

src/Commands/StaticBuildCommand.php

@@ -0,0 +1,55 @@
+<?php
+
+namespace Vormkracht10\LaravelStatic\Commands;
+
+use GuzzleHttp\RequestOptions;
+use Illuminate\Config\Repository;
+use Illuminate\Console\Command;
+use Spatie\Crawler\Crawler;
+use Spatie\Crawler\CrawlProfiles\CrawlInternalUrls;
+use Vormkracht10\LaravelStatic\Crawler\StaticCrawlObserver;
+use Vormkracht10\LaravelStatic\LaravelStatic;
+
+class StaticBuildCommand extends Command
+{
+    public $signature = 'static:build';
+
+    public $description = 'Build Static version';
+
+    protected Repository $config;
+
+    protected LaravelStatic $static;
+
+    public function __construct(Repository $config, LaravelStatic $static)
+    {
+        parent::__construct();
+
+        $this->config = $config;
+        $this->static = $static;
+    }
+
+    public function handle(): void
+    {
+        if ($this->config->get('static.build.clear_before_start')) {
+            $this->call(StaticClearCommand::class);
+        }
+
+        $bypassHeader = $this->config->get('static.build.bypass_header');
+
+        Crawler::create([
+            RequestOptions::VERIFY => ! app()->environment('local'),
+            RequestOptions::ALLOW_REDIRECTS => true,
+            RequestOptions::HEADERS => [
+                array_key_first($bypassHeader) => array_shift($bypassHeader),
+                'User-Agent' => 'LaravelStatic/1.0',
+            ],
+        ])
+        ->setCrawlObserver(new StaticCrawlObserver)
+        ->setCrawlProfile(new CrawlInternalUrls($this->config->get('app.url')))
+        ->acceptNofollowLinks()
+        ->setConcurrency($this->config->get('static.build.concurrency'))
+        ->setDefaultScheme('https')
+        // ->setParseableMimeTypes(['text/html', 'text/plain'])
+        ->startCrawling($this->config->get('app.url'));
+    }
+}

src/Commands/StaticClearCommand.php

@@ -2,6 +2,7 @@
 
 namespace Vormkracht10\LaravelStatic\Commands;
 
+use Illuminate\Config\Repository;
 use Illuminate\Console\Command;
 use Vormkracht10\LaravelStatic\LaravelStatic;
 
@@ -11,8 +12,22 @@ class StaticClearCommand extends Command
 
     public $description = 'Clear static cached files';
 
-    public function handle(LaravelStatic $static): void
+    protected Repository $config;
+
+    protected LaravelStatic $static;
+
+    public function __construct(Repository $config, LaravelStatic $static)
     {
-        $static->clear();
+        parent::__construct();
+
+        $this->config = $config;
+        $this->static = $static;
+    }
+
+    public function handle(): void
+    {
+        $this->static->clear();
+
+        $this->info('✔ Static cache cleared!');
     }
 }

src/Console/Terminal.php

@@ -0,0 +1,28 @@
+<?php
+
+namespace Vormkracht10\LaravelStatic\Console;
+
+use Illuminate\Console\Command;
+
+class Terminal extends Command
+{
+    protected $name = 'NONEXISTENT';
+
+    protected $hidden = true;
+
+    public $outputSymfony;
+
+    public $outputStyle;
+
+    public function __construct($argInput = '')
+    {
+        parent::__construct();
+
+        $this->input = new \Symfony\Component\Console\Input\StringInput($argInput);
+
+        $this->outputSymfony = new \Symfony\Component\Console\Output\ConsoleOutput();
+        $this->outputStyle = new \Illuminate\Console\OutputStyle($this->input, $this->outputSymfony);
+
+        $this->output = $this->outputStyle;
+    }
+}

src/Crawler/StaticCrawlObserver.php

@@ -0,0 +1,59 @@
+<?php
+
+namespace Vormkracht10\LaravelStatic\Crawler;
+
+use GuzzleHttp\Exception\RequestException;
+use Psr\Http\Message\ResponseInterface;
+use Psr\Http\Message\UriInterface;
+use Spatie\Crawler\CrawlObservers\CrawlObserver;
+
+class StaticCrawlObserver extends CrawlObserver
+{
+    /**
+     * Called when the crawler will crawl the url.
+     *
+     * @param  \Psr\Http\Message\UriInterface  $url
+     */
+    public function willCrawl(UriInterface $url): void
+    {
+        // ...
+    }
+
+    /**
+     * Called when the crawler has crawled the given url successfully.
+     *
+     * @param  \Psr\Http\Message\UriInterface  $url
+     * @param  \Psr\Http\Message\ResponseInterface  $response
+     * @param  \Psr\Http\Message\UriInterface|null  $foundOnUrl
+     */
+    public function crawled(
+        UriInterface $url,
+        ResponseInterface $response,
+        ?UriInterface $foundOnUrl = null
+    ): void {
+        console()->info('✔ '.$url);
+    }
+
+    /**
+     * Called when the crawler had a problem crawling the given url.
+     *
+     * @param  \Psr\Http\Message\UriInterface  $url
+     * @param  \GuzzleHttp\Exception\RequestException  $requestException
+     * @param  \Psr\Http\Message\UriInterface|null  $foundOnUrl
+     */
+    public function crawlFailed(
+        UriInterface $url,
+        RequestException $requestException,
+        ?UriInterface $foundOnUrl = null
+    ): void {
+        console()->error('✘ '.$url);
+    }
+
+    /**
+     * Called when the crawl has ended.
+     */
+    public function finishedCrawling(): void
+    {
+        console()->info('✔ Static build completed');
+    }
+}

src/LaravelStaticServiceProvider.php

@@ -4,6 +4,7 @@
 
 use Spatie\LaravelPackageTools\Package;
 use Spatie\LaravelPackageTools\PackageServiceProvider;
+use Vormkracht10\LaravelStatic\Commands\StaticBuildCommand;
 use Vormkracht10\LaravelStatic\Commands\StaticClearCommand;
 
 class LaravelStaticServiceProvider extends PackageServiceProvider
@@ -13,6 +14,9 @@ public function configurePackage(Package $package): void
         $package
             ->name('laravel-static')
             ->hasConfigFile()
-            ->hasCommand(StaticClearCommand::class);
+            ->hasCommands([
+                StaticClearCommand::class,
+                StaticBuildCommand::class,
+            ]);
     }
 }

src/Middleware/StaticResponse.php

@@ -16,10 +16,13 @@ class StaticResponse
 
     protected Filesystem $files;
 
+    protected array $bypassHeader;
+
     public function __construct(Repository $config, Filesystem $files)
     {
         $this->config = $config;
         $this->files = $files;
+        $this->bypassHeader = $this->config->get('static.build.bypass_header');
     }
 
     /**
@@ -93,7 +96,8 @@ protected function shouldBeStatic(Request $request, Response $response): bool
     {
         return
             $request->isMethod('GET') &&
-            $response->getStatusCode() == 200;
+            $response->getStatusCode() == 200 &&
+            $request->header(array_key_first($this->bypassHeader)) != array_shift($this->bypassHeader);
     }
 
     /**