.env.example

@@ -1,23 +1,39 @@
+APP_NAME=Crawler
 APP_ENV=local
-APP_DEBUG=true
 APP_KEY=
+APP_DEBUG=true
+APP_URL=http://crawler.localhost
+
+LOG_CHANNEL=stack
 
+DB_CONNECTION=mysql
 DB_HOST=127.0.0.1
-DB_DATABASE=php-crawler
+DB_PORT=3306
+DB_DATABASE=crawler
 DB_USERNAME=root
-DB_PASSWORD=root
+DB_PASSWORD=
 
+BROADCAST_DRIVER=log
 CACHE_DRIVER=file
+QUEUE_CONNECTION=sync
 SESSION_DRIVER=file
-QUEUE_DRIVER=sync
+SESSION_LIFETIME=120
 
 REDIS_HOST=127.0.0.1
 REDIS_PASSWORD=null
 REDIS_PORT=6379
 
 MAIL_DRIVER=smtp
-MAIL_HOST=mailtrap.io
+MAIL_HOST=smtp.mailtrap.io
 MAIL_PORT=2525
 MAIL_USERNAME=null
 MAIL_PASSWORD=null
 MAIL_ENCRYPTION=null
+
+PUSHER_APP_ID=
+PUSHER_APP_KEY=
+PUSHER_APP_SECRET=
+PUSHER_APP_CLUSTER=mt1
+
+MIX_PUSHER_APP_KEY="${PUSHER_APP_KEY}"
+MIX_PUSHER_APP_CLUSTER="${PUSHER_APP_CLUSTER}"

.env.travis

@@ -0,0 +1,10 @@
+APP_NAME=Crawler
+APP_ENV=testing
+APP_KEY=
+
+DB_CONNECTION=sqlite
+CACHE_DRIVER=array
+SESSION_DRIVER=array
+QUEUE_DRIVER=sync
+DB_CONNECTION=sqlite
+DB_DATABASE=":memory:"

.gitattributes

@@ -1,3 +1,5 @@
 * text=auto
 *.css linguist-vendored
-*.less linguist-vendored
+*.scss linguist-vendored
+*.js linguist-vendored
+CHANGELOG.md export-ignore

.gitignore

@@ -1,8 +1,15 @@
-/vendor
 /node_modules
+/public/hot
 /public/storage
-Homestead.yaml
+/storage/*.key
+/vendor
+/.idea
+/.vagrant
 Homestead.json
-
-.DS_Store
-.idea/
+Homestead.yaml
+npm-debug.log
+yarn-error.log
+.env
+/.phpstorm.meta.php
+/_ide_helper.php
+/_ide_helper_models.php

.travis.yml

@@ -0,0 +1,11 @@
+language: php
+php:
+  - 7.2
+before_script:
+  - cp .env.travis .env
+  - composer self-update
+  - composer install --no-interaction
+  - php artisan key:generate
+  - php artisan crawler:build
+script:
+  - ./vendor/bin/phpunit

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2018 hedii
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -1,75 +1,83 @@
 # php-crawler
-A crawler written in php with laravel that find email addresses on the internets.
-Given an entry point url, the crawler will search for emails in all the urls available for this entry point domain name.
-The emails are downloadable in a text file at any time.
-Several users can start searching for emails without viewing the other users' searches (searches are related to a user).
+
+***This repository has not been updated since a long time and I am not using it anymore. Please feel free to fork it and do whatever you want with it.***
+
+[![Build Status](https://travis-ci.org/hedii/php-crawler.svg?branch=master)](https://travis-ci.org/hedii/php-crawler)
+
+A crawler application with a php backend using Laravel, and a js frontend using vuejs, that finds email addresses on the internets.
+
+Given an entry point url, the crawler will search for emails in all the urls available from this entry point domain name.
+The emails are downloadable as a text file.
+
+Multiple users can start searching for emails without being able see the other users' searches (searches are related to a single user).
+
+## Screenshots
+
+![Screenshot1](https://user-images.githubusercontent.com/5358048/35198632-c9afcdf2-fef1-11e7-91ec-900b1ff25b62.png)
+
+![Screenshot2](https://user-images.githubusercontent.com/5358048/35198634-cddecb44-fef1-11e7-836b-5dbe16a1bb2a.png)
+
+![Screenshot3](https://user-images.githubusercontent.com/5358048/35198637-d015a180-fef1-11e7-93a0-9eb72e1c1281.png)
+
+![Screenshot4](https://user-images.githubusercontent.com/5358048/35198638-d20ea1a8-fef1-11e7-8ecf-df3cf2c4c8e1.png)
+
+## Server requirements
+
+- PHP >= 7.2.0
+- OpenSSL PHP Extension
+- PDO PHP Extension
+- Mbstring PHP Extension
+- Tokenizer PHP Extension
+- XML PHP Extension
 
 ## Installation
-- Create a mysql database (default name: `php-crawler`)
-- Install the repo with [composer](https://getcomposer.org/):
-```bash
-composer create-project hedii/php-crawler php-crawler
-cd php-crawler
-```
-- Install [npm](https://docs.npmjs.com/getting-started/what-is-npm) dependencies (optional):
+
+- Create a mysql database (default name: `crawler`)
+- Install the project with [composer](https://getcomposer.org/):
 ```bash
-npm install
+composer create-project hedii/php-crawler crawler
+cd crawler
 ```
 - Open the `.env` file, check the database credentials, and modify it if needed:
 ```
+DB_CONNECTION=mysql
 DB_HOST=127.0.0.1
-DB_DATABASE=php-crawler
+DB_PORT=3306
+DB_DATABASE=crawler
 DB_USERNAME=root
-DB_PASSWORD=root
+DB_PASSWORD=your_password_here
+```
+- In the `.env` file, set the application url:
+```
+APP_URL=http://crawler.localhost
 ```
-- Build the app
+- Build the crawler application
 ```bash
 php artisan crawler:build
 ```
-- Point your webserver to the public directory: `php-crawler/public`
+- Point your web server's document / web root to be the public directory: `/some/path/crawler/public`. The index.php in this directory serves as the front controller for all HTTP requests entering your application. [See Laravel documentation](https://laravel.com/docs/master/installation). I highly recommend using [Laravel Valet](https://laravel.com/docs/master/valet) if you are using a Mac. Otherwise, check [Laravel Homestead](https://laravel.com/docs/master/homestead).
 - Done
 
 ## Usage
+
 - Navigate to your php-crawler website
 - Register a new account
 - Create a new search
 - Create more searches
-- Download the found resources
-
-## Troubleshooting
-
-#### Blank space in path
-On some systems, if there is any blank space in the path to the crawler public directory, the crawler app won't work.
-Remove any space in folders that are part of the crawler path.
-
-#### MAMP server
-If you are running the crawler on a MAMP server, edit `config/database.php` and add a unix socket conf:
-```php
-'mysql' => [
-    'driver'    => 'mysql',
-    'host'      => env('DB_HOST', 'localhost'),
-    'database'  => env('DB_DATABASE', 'forge'),
-    'username'  => env('DB_USERNAME', 'forge'),
-    'password'  => env('DB_PASSWORD', ''),
-    'charset'   => 'utf8',
-    'collation' => 'utf8_unicode_ci',
-    'prefix'    => '',
-    'strict'    => false,
-    'engine'    => null,
-
-    'unix_socket' => '/Applications/MAMP/tmp/mysql/mysql.sock', // add this line
-],
+- Download the found emails
+
+## Testing
+
+```
+composer test
 ```
 
+## Contributing
 
-## Todo
-- write php tests
-- write js tests
-- Crawl for other things than emails
-- ...
+**All** contributions are welcome :)
 
+Please write some tests if you are adding or modifying features.
 
-## Screenshots
-![](https://cloud.githubusercontent.com/assets/5358048/13635825/9a75ec3c-e5fe-11e5-84b8-f4d42973bbda.png)
+## License
 
-![](https://cloud.githubusercontent.com/assets/5358048/13635826/9a76ebfa-e5fe-11e5-9bc8-5e770cccfd7a.png)
+php-crawler is open-sourced software licensed under the [MIT license](https://opensource.org/licenses/MIT).

app/Console/Commands/BuildCommand.php

@@ -0,0 +1,60 @@
+<?php
+
+namespace App\Console\Commands;
+
+use Illuminate\Console\Command;
+
+class BuildCommand extends Command
+{
+    /**
+     * The name and signature of the console command.
+     *
+     * @var string
+     */
+    protected $signature = 'crawler:build';
+
+    /**
+     * The console command description.
+     *
+     * @var string
+     */
+    protected $description = 'Build the crawler app';
+
+    /**
+     * Execute the console command.
+     *
+     * @return mixed
+     */
+    public function handle(): bool
+    {
+        $this->comment('Running fresh application migrations');
+
+        $returnCode = $this->call('migrate:fresh');
+
+        if ($returnCode !== 0) {
+            $this->error('Unable run the fresh application migrations');
+
+            return false;
+        } else {
+            $this->info('Application fresh migrations ok');
+            $this->line('---');
+        }
+
+        $this->comment('Installing Laravel Passport');
+
+        $returnCode = $this->callSilent('passport:install');
+
+        if ($returnCode !== 0) {
+            $this->error('Unable to install Laravel Passport');
+
+            return false;
+        } else {
+            $this->info('Laravel Passport installed');
+            $this->line('---');
+        }
+
+        $this->info('Crawler app ready!');
+
+        return true;
+    }
+}

app/Console/Commands/CrawlCommand.php

@@ -0,0 +1,42 @@
+<?php
+
+namespace App\Console\Commands;
+
+use App\Search;
+use App\Services\Crawler\Crawler;
+use Illuminate\Console\Command;
+
+class CrawlCommand extends Command
+{
+    /**
+     * The name and signature of the console command.
+     *
+     * @var string
+     */
+    protected $signature = 'crawler:crawl {search_id : A search id}';
+
+    /**
+     * The console command description.
+     *
+     * @var string
+     */
+    protected $description = 'Start a new crawler on a given search';
+
+    /**
+     * Execute the console command.
+     *
+     * @return bool
+     */
+    public function handle(): bool
+    {
+        if (! $search = Search::find($this->argument('search_id'))) {
+            $this->error("No search with the id `{$this->argument('search_id')}`");
+
+            return false;
+        }
+
+        $crawler = new Crawler($search);
+
+        return $crawler->run();
+    }
+}