feat: import (#302)

* init import * wip * street_address => address * cleanup * wip * wip * refactor: separte import commands * import activity domains * wip * import activity domains * wip * wip * consolidate routes * wip * import projects * import articles * wip * confirm run import in production * cleanup * wip * [wip]fix project import by status --------- Co-authored-by: Lupu Gheorghe <[email protected]>
code4romania · Nov 10, 2023 · bb7d10c · bb7d10c
1 parent 0602bba
commit bb7d10c
Show file tree

Hide file tree

Showing 74 changed files with 1,593 additions and 184 deletions.
diff --git a/.gitignore b/.gitignore
@@ -30,3 +30,4 @@ docker-compose.override.yml
 phpunit.xml
 _ide_helper_models.php
 _ide_helper.php
+/.scannerwork/
diff --git a/app/Concerns/Publishable.php b/app/Concerns/Publishable.php
@@ -0,0 +1,87 @@
+<?php
+
+declare(strict_types=1);
+
+namespace App\Concerns;
+
+use Carbon\Carbon;
+use Illuminate\Database\Eloquent\Builder;
+
+trait Publishable
+{
+    public function initializePublishable(): void
+    {
+        $this->fillable[] = 'published_at';
+
+        $this->casts['published_at'] = 'datetime';
+    }
+
+    public static function bootPublishable(): void
+    {
+        static::addGlobalScope('published', function (Builder $query) {
+            $query->onlyPublished();
+        });
+    }
+
+    public function scopeWithDrafted(Builder $query): Builder
+    {
+        return $query->withoutGlobalScope('published');
+    }
+
+    public function scopeOnlyDrafted(Builder $query): Builder
+    {
+        return $query
+            ->withDrafted()
+            ->whereNull('published_at');
+    }
+
+    public function scopeOnlyScheduled(Builder $query): Builder
+    {
+        return $query
+            ->withDrafted()
+            ->whereNotNull('published_at')
+            ->where('published_at', '>', Carbon::now());
+    }
+
+    public function scopeOnlyPublished(Builder $query): Builder
+    {
+        return $query
+            ->whereNotNull('published_at')
+            ->where('published_at', '<=', Carbon::now());
+    }
+
+    public function isDraft(): bool
+    {
+        return \is_null($this->published_at);
+    }
+
+    public function isPublished(): bool
+    {
+        return ! $this->isDraft() && $this->published_at->isPast();
+    }
+
+    public function isScheduled(): bool
+    {
+        return ! $this->isDraft() && $this->published_at->isFuture();
+    }
+
+    /**
+     * Determine the publish status of the model instance.
+     *
+     * @return string
+     */
+    public function status(): string
+    {
+        if ($this->isDraft()) {
+            return 'draft';
+        }
+
+        if ($this->isPublished()) {
+            return 'published';
+        }
+
+        if ($this->isScheduled()) {
+            return 'scheduled';
+        }
+    }
+}
diff --git a/app/Console/Commands/Import/Command.php b/app/Console/Commands/Import/Command.php
@@ -0,0 +1,138 @@
+<?php
+
+declare(strict_types=1);
+
+namespace App\Console\Commands\Import;
+
+use Carbon\Carbon;
+use Illuminate\Console\Command as BaseCommand;
+use Illuminate\Console\ConfirmableTrait;
+use Illuminate\Database\Connection;
+use Illuminate\Database\Eloquent\Model;
+use Illuminate\Support\Collection;
+use Illuminate\Support\Facades\Cache;
+use Illuminate\Support\Facades\DB;
+use Symfony\Component\Console\Helper\ProgressBar;
+
+abstract class Command extends BaseCommand
+{
+    use ConfirmableTrait;
+
+    protected readonly Connection $db;
+
+    protected ?ProgressBar $progressBar = null;
+
+    protected int $errorsCount = 0;
+
+    public function __construct()
+    {
+        parent::__construct();
+
+        $this->db = DB::connection('import');
+    }
+
+    public function createProgressBar(string $message, int $max): void
+    {
+        $this->progressBar = $this->output->createProgressBar($max);
+        $this->progressBar->setFormat("\n<options=bold>%message%</>\n[%bar%] %current%/%max%\n");
+        $this->progressBar->setMessage('⏳ ' . $message);
+        $this->progressBar->setMessage('', 'status');
+        $this->progressBar->setBarWidth(48);
+        $this->progressBar->setBarCharacter('<comment>=</>');
+        $this->progressBar->setEmptyBarCharacter('<fg=gray>-</>');
+        $this->progressBar->setProgressCharacter('<comment>></>');
+        $this->progressBar->start();
+    }
+
+    public function finishProgressBar(string $message): void
+    {
+        if ($this->hasErrors()) {
+            $this->progressBar->setMessage('🚨 <fg=red>' . $message . ' with ' . $this->errorsCount . ' errors</>');
+        } else {
+            $this->progressBar->setMessage('✅ <info>' . $message . '</>');
+        }
+
+        $this->progressBar->finish();
+        $this->resetErrors();
+    }
+
+    public function logError(string $message, array $context = []): void
+    {
+        logger()->error($message, $context);
+
+        $this->errorsCount++;
+    }
+
+    public function hasErrors(): bool
+    {
+        return $this->errorsCount > 0;
+    }
+
+    public function resetErrors(): void
+    {
+        $this->errorsCount = 0;
+    }
+
+    public function getRejectedOrganizations(): Collection
+    {
+        return Cache::driver('array')
+            ->rememberForever(
+                'import-rejected-organizations',
+                fn () => $this->db
+                    ->table('dbo.ONGs')
+                    ->orderBy('dbo.ONGs.Id')
+                    ->select([
+                        'dbo.ONGs.Id',
+                        'dbo.ONGs.CIF',
+                        'dbo.ONGs.ONGStatusId',
+                        'ProjectsCount' => $this->db
+                            ->table('dbo.ONGProjects')
+                            ->whereColumn('dbo.ONGs.Id', 'dbo.ONGProjects.ONGId')
+                            ->selectRaw('count(*)'),
+                    ])
+                    ->get()
+                    ->groupBy('CIF')
+                    ->reject(fn (Collection $collection) => $collection->count() < 2)
+                    ->flatMap(
+                        fn (Collection $collection) => $collection
+                            ->sortBy([
+                                ['ONGStatusId', 'asc'],
+                                ['ProjectsCount', 'desc'],
+                            ])
+                            ->skip(1)
+                    )
+                    ->pluck('Id')
+            );
+    }
+
+    public function addFilesToCollection(Model $model, int|array|null $fileIds, string $collection = 'default'): void
+    {
+        $this->db
+            ->table('dbo.Files')
+            ->join('dbo.FilesData', 'dbo.FilesData.Id', 'dbo.Files.Id')
+            ->whereIn(
+                'dbo.Files.Id',
+                collect($fileIds)
+                    ->filter()
+                    ->all()
+            )
+            ->get()
+            ->each(function (object $file) use ($model, $collection) {
+                $filename = rtrim($file->FileName, '.') . '.' . ltrim($file->FileExtension, '.');
+
+                $model->addMediaFromString($file->Data)
+                    ->usingFileName($filename)
+                    ->usingName($filename)
+                    ->toMediaCollection($collection);
+            });
+    }
+
+    public function parseDate(?string $input): ?Carbon
+    {
+        if ($input === null) {
+            return null;
+        }
+
+        return Carbon::createFromFormat('M d Y H:i:s:A', $input);
+    }
+}
diff --git a/app/Console/Commands/Import/ImportArticlesCommand.php b/app/Console/Commands/Import/ImportArticlesCommand.php
@@ -0,0 +1,143 @@
+<?php
+
+declare(strict_types=1);
+
+namespace App\Console\Commands\Import;
+
+use App\Models\Article;
+use App\Models\ArticleCategory;
+use App\Services\Sanitize;
+use Carbon\Carbon;
+use Illuminate\Support\Collection;
+use Throwable;
+
+class ImportArticlesCommand extends Command
+{
+    /**
+     * The name and signature of the console command.
+     *
+     * @var string
+     */
+    protected $signature = 'app:import:articles
+        {--chunk=100 : The number of records to process at a time}
+        {--skip-files : Skip importing files}
+        {--force : Force the operation to run when in production}';
+
+    /**
+     * The console command description.
+     *
+     * @var string
+     */
+    protected $description = 'Import articles from the old database.';
+
+    /**
+     * Execute the console command.
+     */
+    public function handle(): int
+    {
+        if (! $this->confirmToProceed()) {
+            return static::FAILURE;
+        }
+
+        $this->importArticleCategories();
+        $this->importArticles();
+
+        return static::SUCCESS;
+    }
+
+    protected function importArticleCategories(): void
+    {
+        $articleCategories = $this->db
+            ->table('lkp.ArticleCategories')
+            ->orderBy('lkp.ArticleCategories.Id')
+            ->get();
+
+        $this->createProgressBar('Importing article categories...', $articleCategories->count());
+
+        foreach ($this->progressBar->iterate($articleCategories) as $row) {
+            ArticleCategory::forceCreate([
+                'id' => (int) $row->Id,
+                'name' => Sanitize::text($row->Name),
+                'slug' => Sanitize::slug($row->Name),
+            ]);
+        }
+
+        $this->finishProgressBar('Imported article categories');
+    }
+
+    protected function importArticles(): void
+    {
+        $query = $this->db
+            ->table('dbo.Articles')
+            ->addSelect([
+                'dbo.Articles.*',
+                'MainImageId' => $this->db
+                    ->table('dbo.ArticleImages')
+                    ->select('ImageId')
+                    ->whereColumn('dbo.ArticleImages.ArticleId', 'dbo.Articles.Id')
+                    ->where('dbo.ArticleImages.IsMainImage', 1),
+                'GalleryImageIds' => $this->db
+                    ->table('dbo.ArticleImages')
+                    ->selectRaw("STRING_AGG(ImageId,',')")
+                    ->whereColumn('dbo.ArticleImages.ArticleId', 'dbo.Articles.Id')
+                    ->where('dbo.ArticleImages.IsMainImage', 0),
+                'AttachmentIds' => $this->db
+                    ->table('dbo.ArticleAttachments')
+                    ->selectRaw("STRING_AGG(GenericFileId,',')")
+                    ->whereColumn('dbo.ArticleAttachments.ArticleId', 'dbo.Articles.Id'),
+            ])
+            ->orderBy('dbo.Articles.Id');
+
+        $this->createProgressBar(
+            $this->option('skip-files')
+                ? 'Importing articles [skip-files]...'
+                : 'Importing articles...',
+            $query->count()
+        );
+
+        $query->chunk((int) $this->option('chunk'), function (Collection $items) {
+            $items->each(function (object $row) {
+                $created_at = Carbon::parse($row->CreationDate);
+
+                try {
+                    $article = Article::forceCreate([
+                        'id' => (int) $row->Id,
+                        'title' => Sanitize::text($row->Title),
+                        'slug' => Sanitize::text($row->DynamicUrl),
+                        'author' => Sanitize::text($row->Author),
+                        'content' => $row->Content,
+                        'article_category_id' => (int) $row->ArticleCategoryId,
+                        'created_at' => $created_at,
+                        'updated_at' => $created_at,
+                        'published_at' => $this->parseDate($row->PublishDate),
+                    ]);
+
+                    if (! $this->option('skip-files')) {
+                        // Add main image
+                        $this->addFilesToCollection($article, $row->MainImageId, 'preview');
+
+                        // Add gallery images
+                        if ($row->GalleryImageIds) {
+                            $this->addFilesToCollection($article, explode(',', $row->GalleryImageIds), 'gallery');
+                        }
+
+                        // Add attachments
+                        if ($row->AttachmentIds) {
+                            $this->addFilesToCollection($article, explode(',', $row->AttachmentIds));
+                        }
+                    }
+                } catch (Throwable $th) {
+                    $this->logError('Error importing article #' . $row->Id, [$th->getMessage()]);
+                }
+
+                $this->progressBar->advance();
+            });
+        });
+
+        $this->finishProgressBar(
+            $this->option('skip-files')
+            ? 'Imported article [skip-files]'
+            : 'Imported article'
+        );
+    }
+}