Docs and safer dump (#2)

* some docs and process-mysqldump safer

* stuff for v0.2 & docs

* fix tests

* install docker on build

* store process-mysqldump locally

* use $GITHUB_TOKEN from travis

* only deploy docs on master

.gitignore

@@ -1,2 +1,3 @@
 /vendor/
 /tests/report/
+/site/

.markdownlint.json

@@ -0,0 +1,8 @@
+{
+    "default": true,
+    "MD013": {
+        "line_length": 120,
+        "code_blocks": false,
+        "tables": false
+    }
+}

.travis.yml

@@ -24,10 +24,30 @@ matrix:
 before_script:
   - composer config platform.php $(php -r "echo PHP_VERSION;")
   - travis_retry composer update --no-interaction --prefer-dist $PREFER_LOWEST
+  - sudo apt-get -y -o Dpkg::Options::="--force-confnew" install docker-ce
 
 script:
   - vendor/bin/phpcs -p --warning-severity=0 src/ tests/
   - vendor/bin/phpunit --coverage-clover=./tests/report/coverage.clover
+  - make lint-md
 
 after_script:
   - test -f ./tests/report/coverage.clover && (wget https://scrutinizer-ci.com/ocular.phar; php ocular.phar code-coverage:upload --format=php-clover ./tests/report/coverage.clover)
+
+stages:
+  - test
+  - name: deploy
+    if: branch = master
+jobs:
+  include:
+  - stage: deploy
+    script: make docs-build
+    deploy:
+      provider: pages
+      local-dir: site
+      skip-cleanup: true
+      keep-history: true
+      verbose: true
+      github-token: $GITHUB_TOKEN
+      on:
+        branch: master

Dockerfile

@@ -1,4 +1,17 @@
-FROM composer AS build
+FROM alpine as build-c
+
+WORKDIR /app
+COPY dump-parser /app
+
+RUN set +xe \
+    && apk add -u --no-cache --virtual .build-deps \
+        gcc \
+        musl-dev \
+    && gcc -O2 -Wall -pedantic process-mysqldump.c -o process-mysqldump \
+    && chmod +x process-mysqldump \
+    && apk del .build-deps
+
+FROM composer AS build-php
 
 WORKDIR /app
 COPY src /app/src
@@ -14,9 +27,10 @@ RUN set +xe \
         mariadb-client
 
 WORKDIR /app
-COPY --from=build /app/src /app/src
-COPY --from=build /app/vendor /app/vendor
+COPY --from=build-php /app/src /app/src
+COPY --from=build-php /app/vendor /app/vendor
 COPY bin /app/bin
+COPY --from=build-c /app/process-mysqldump /bin/process-mysqldump
 
 ARG BUILD_DATE
 ARG VCS_REF

Makefile

@@ -45,12 +45,18 @@ build-docker: ## Build the docker image
 test: ## Run the unit and integration testsuites.
 test: lint test-unit
 
-lint: ## Run phpcs against the code.
+lint: ## Check file syntax
+lint: lint-php lint-md
+
+lint-php: ## Run phpcs against the code.
 	${DOCKER_RUN} vendor/bin/phpcs -p --warning-severity=0 src/ tests/
 
 lint-fix: ## Run phpcsf and fix possible lint errors.
 	${DOCKER_RUN} vendor/bin/phpcbf -p src/ tests/
 
+lint-md: ## Check the markdown files
+	${DOCKER} run --rm -v $$(pwd):/data:cached gouvinb/docker-markdownlint -v *.md docs/*.md docs/*/*.md
+
 test-unit: ## Run the unit testsuite.
 	${DOCKER_RUN} vendor/bin/phpunit --testsuite unit
 
@@ -76,6 +82,18 @@ test-coverage-html: ## Run all tests and output coverage to html.
 test-coverage-clover: ## Run all tests and output clover coverage to file.
 	${DOCKER_RUN} phpdbg7 -qrr vendor/bin/phpunit --coverage-clover=./tests/report/coverage.clover
 
+# Documentation
+
+docs-test: ## Run docs test server
+	docker run --rm -it -p 8000:8000 -v $$(pwd):/docs squidfunk/mkdocs-material
+
+docs-build: ## Build the mkdocs documentation
+	docker run --rm -it -v $$(pwd):/docs squidfunk/mkdocs-material build
+
+docs-deploy: ## Deploy the mkdocs documentation
+	docker run --rm -it -v ~/.ssh:/root/.ssh -v $$(pwd):/docs -e GITHUB_TOKEN squidfunk/mkdocs-material gh-deploy
+
+
 # Help
 
 help: ## Show this help message.

README.md

@@ -5,13 +5,13 @@
 [![Build Status](https://img.shields.io/travis/graze/sprout/master.svg?style=flat-square)](https://travis-ci.org/graze/sprout)
 [![Coverage Status](https://img.shields.io/scrutinizer/coverage/g/graze/sprout.svg?style=flat-square)](https://scrutinizer-ci.com/g/graze/sprout/code-structure)
 [![Quality Score](https://img.shields.io/scrutinizer/g/graze/sprout.svg?style=flat-square)](https://scrutinizer-ci.com/g/graze/sprout)
-[![Total Downloads](https://img.shields.io/packagist/dt/graze/sprout.svg?style=flat-square)](https://packagist.org/packages/graze/sprout)
 [![PHP Version](https://img.shields.io/packagist/php-v/graze/sprout.svg?style=flat-square)](https://php.net)
 [![Docker Image Size](https://img.shields.io/microbadger/image-size/graze/sprout.svg?style=flat-square)](https://hub.docker.com/r/graze/sprout/)
+[![Docker Pulls](https://img.shields.io/docker/pulls/graze/sprout.svg?style=flat-square)](https://hub.docker.com/r/graze/sprout/)
 
 Sprout is a tool to help Dump, Truncate and Seed development data into your databases.
 
-![](https://78.media.tumblr.com/534425eb11706448af8ce5838629f76d/tumblr_inline_n9t8gdzC7p1qzjzhu.gif)
+![fun image](https://78.media.tumblr.com/534425eb11706448af8ce5838629f76d/tumblr_inline_n9t8gdzC7p1qzjzhu.gif)
 
 1. Seed sql data from local files
 1. Dump data from mysql tables
@@ -34,17 +34,35 @@ docker run -v [volumes] --rm graze/sprout [command]
 
 ## Usage
 
+### File Structure
+
+Sprout will use the following file structure by default, you can change the root and each group's path in the
+configuration file.
+
+```text
+- /seed
+  - group1
+    - schema1
+      - table1.sql
+      - table2.sql
+    - schema2
+      - table3.sql
+  - group1
+    - schema3
+      - table4.sql
+```
+
 ### Quick Start
 
 ```bash
 # Dump all tables you are interested in
-sprout dump --config=config/sprout.yml --group=core a_schema:table_1,table_2 ...
+sprout dump --config=config/sprout.yml --group=static a_schema:table_1,table_2 ...
 
 # Store the data in your repository of choice
-git add /seed/data/*
+git add /seed/static/*
 
-# Seed the data from your seed data
-sprout seed --config=config/sprout.yml --group=core
+# Seed the data from your local files
+sprout seed --config=config/sprout.yml --group=static
 ```
 
 ### Seeding
@@ -87,6 +105,48 @@ sprout dump --config=config/sprout.yml --group=core
 sprout dump --config=config/sprout.yml --group=core the_schema:country
 ```
 
+### Configuration
+
+The configuration file follows the following standards.
+
+By default sprout looks for a `config/sprout.yml` file, you can specify a different file
+using `--config=path/to/file.yml`.
+
+```yaml
+defaults:
+  group: core
+  # default path
+  path: /seed
+  # number of simultaneous processors to run at a time (default: 10)
+  simultaneousProcesses: 10
+
+# ability to specify custom paths for groups
+groups:
+  core:
+    path: /custom/path/to/group
+
+schemas:
+  # name of the schema to reference
+  <name>:
+    # [optional] the actual name of the schema in the database. If not specified, <name> from above will be used
+    schema: 'schema'
+
+    # Connection details - this is just an example, you may want to specify
+    # different properties, e.g. if connecting to a remote server. You are
+    # advised to refer to the 'pdo' documentation for further details.
+    connection:
+      user: 'morphism'
+      password: 'morphism'
+      # driver for the database connection, currently only: `mysql` is supported
+      driver: 'mysql'
+      # [optional] name of the database
+      dbName: 'schema'
+      # database on a remote host
+      host: 'db'
+      # [optional] port to use, by default: 3306
+      port: 3306
+```
+
 ## Testing
 
 ```bash
@@ -109,4 +169,4 @@ instead of using the issue tracker.
 
 ## License
 
-The MIT License (MIT). Please see [License File](LICENSE.md) for more information.
+The MIT License (MIT). Please see [License File](LICENSE) for more information.

composer.json

@@ -56,5 +56,9 @@
       "Graze\\Sprout\\Test\\Unit\\": "tests/unit",
       "Graze\\Sprout\\Test\\Integration\\": "tests/integration"
     }
-  }
+  },
+  "bin": [
+    "./bin/sprout",
+    "./bin/process-mysqldump"
+  ]
 }

dev.md

@@ -19,8 +19,8 @@ This is to spec out what we would require?
 - Be as quick as possible (use as many tricks to speed up seeding as possible)
 - Handle at least mysql
 - Group seeding into types (core, operational, testing)
-    - You could make a custom group for testing a particular problem ?
-- When using a .sql file, the table must be truncated before seeding 
+  - You could make a custom group for testing a particular problem ?
+- When using a .sql file, the table must be truncated before seeding
 
 ### Extended requirements
 
@@ -32,8 +32,8 @@ This is to spec out what we would require?
 
 - You have a series of grouped seed data (e.g. core, sample, test)
 - You can apply each of the seed data independently
-    - For example, you can 'apply' each test data seed at the beginning of each test run (or maybe even each test)
-    - test data should be idempotent (can be created and removed without modifying other test data)
+  - For example, you can 'apply' each test data seed at the beginning of each test run (or maybe even each test)
+  - test data should be idempotent (can be created and removed without modifying other test data)
 
 ## Commands
 
@@ -92,7 +92,7 @@ default:
   # default path
   path: /seed
 
-# ability to specify custom paths for groups 
+# ability to specify custom paths for groups
 groups:
   core:
     path: /custom/path/to/stuff
@@ -101,7 +101,7 @@ schemas:
 
   # name of the schema in the database
   <name>:
-  
+
     # Connection details - this is just an example, you may want to specify
     # different properties, e.g. if connecting to a remote server. You are
     # advised to refer to the 'pdo' documentation for further details.
@@ -139,11 +139,11 @@ schemas:
 1. if doing this from within php, how can we parallelise -> spawn workers?
 1. when you have multiple groups, how to do you handle truncation?
 1. When truncating should you truncate all tables in a schema, irrespective of the seed data?
-1. When dealing with projects that are not php, why would use 
+1. When dealing with projects that are not php, why would use
 
 ## Future / Scope
 
 1. Should this be limited to sql dumping/restoring only?
-    1. Alternative is to use php/yaml/json files too. 
+    1. Alternative is to use php/yaml/json files too.
 1. Should this handle stored procedures?
 1. How should this handle triggers / complex table setup? (prep tables, disable triggers, indexes, etc)

docs/commands/chop.md

@@ -0,0 +1,87 @@
+# Truncating Data (Chop)
+
+The chop command has the following structure:
+
+```bash
+sprout chop [--config=<path>] [--group=<group>] [<schema>[:<table>,...]] ...
+```
+
+This will truncate the contents of all the specified tables.
+
+!!! note
+    By default when you [seed](seed.md) data it will truncate the tables it finds first, so there is no need to do this
+    manually.
+
+## Configuration file
+
+The optional `--config` option allows you specify the configuration file to use. By default it will look for a file
+called `config/sprout.yml`. The location is relative to the working directory. Within docker this is `/app`.
+
+## Schema and Table configuration
+
+All commands make use of the same [Schema and Tables](../schemas_tables.md) parsing.
+
+The `[<schema>[:<table>,...]] ...` part of the command line allows you to specify none or some schemas, each schema with
+a set of tables or not.
+
+If no schema is defined, all the schemas and tables that are on the filesystem in a group will be truncated.
+If no tables are defined for a schema, all tables on the filesystem will be truncated.
+
+### Chopping all the data
+
+You can truncate all the tables that exist locally if you do not specify and schemas or tables.
+
+```bash
+sprout chop
+```
+
+This will truncate all the data in the default group. See [groups](#groups) for more information on how the groups work.
+
+### Truncating all the tables in a schema
+
+```bash
+sprout chop schema1
+```
+
+This will chop (truncate) all the tables that exist on the filesystem in the schema: `schema1`.
+
+You can truncate multiple schemas too:
+
+```bash
+sprout chop schema1 schema2
+```
+
+### Chopping specific tables
+
+If you only want to truncate a set of specific tables you can specify them as a comma separated list
+after the schema they apply to.
+
+```bash
+sprout chop schema1:table1,table2
+```
+
+You can also specify multiple schemas, each with their own set of tables
+
+```bash
+sprout chop schema1:table1,table2 schema2:table3
+```
+
+## Groups
+
+The optional `--group` option allows you to specify which group to read the seed data from. If this is not
+supplied it will use the default value as defined in the [configuration file](../setup/configuration.md).
+
+See [Chopping individual groups](../groups.md#you-can-truncate-from-a-group-as-well) for more information on how to
+truncate data in groups.
+
+You can see all of the data in a group by calling:
+
+```bash
+sprout chop --group=testing
+```
+
+Or you can limit it to a set of schemas and tables:
+
+```bash
+sprout chop --group=testing schema1 schema2:table1,table2
+```