diff --git a/.gitignore b/.gitignore index 89d3d2b..b666d52 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ /vendor/ /tests/report/ +/site/ diff --git a/.markdownlint.json b/.markdownlint.json new file mode 100644 index 0000000..4aedcfb --- /dev/null +++ b/.markdownlint.json @@ -0,0 +1,8 @@ +{ + "default": true, + "MD013": { + "line_length": 120, + "code_blocks": false, + "tables": false + } +} diff --git a/.travis.yml b/.travis.yml index 18b7987..52b883f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -24,10 +24,30 @@ matrix: before_script: - composer config platform.php $(php -r "echo PHP_VERSION;") - travis_retry composer update --no-interaction --prefer-dist $PREFER_LOWEST + - sudo apt-get -y -o Dpkg::Options::="--force-confnew" install docker-ce script: - vendor/bin/phpcs -p --warning-severity=0 src/ tests/ - vendor/bin/phpunit --coverage-clover=./tests/report/coverage.clover + - make lint-md after_script: - test -f ./tests/report/coverage.clover && (wget https://scrutinizer-ci.com/ocular.phar; php ocular.phar code-coverage:upload --format=php-clover ./tests/report/coverage.clover) + +stages: + - test + - name: deploy + if: branch = master +jobs: + include: + - stage: deploy + script: make docs-build + deploy: + provider: pages + local-dir: site + skip-cleanup: true + keep-history: true + verbose: true + github-token: $GITHUB_TOKEN + on: + branch: master diff --git a/Dockerfile b/Dockerfile index 654f84d..7607bb7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,17 @@ -FROM composer AS build +FROM alpine as build-c + +WORKDIR /app +COPY dump-parser /app + +RUN set +xe \ + && apk add -u --no-cache --virtual .build-deps \ + gcc \ + musl-dev \ + && gcc -O2 -Wall -pedantic process-mysqldump.c -o process-mysqldump \ + && chmod +x process-mysqldump \ + && apk del .build-deps + +FROM composer AS build-php WORKDIR /app COPY src /app/src @@ -14,9 +27,10 @@ RUN set +xe \ mariadb-client WORKDIR /app -COPY --from=build /app/src /app/src -COPY --from=build /app/vendor /app/vendor +COPY --from=build-php /app/src /app/src +COPY --from=build-php /app/vendor /app/vendor COPY bin /app/bin +COPY --from=build-c /app/process-mysqldump /bin/process-mysqldump ARG BUILD_DATE ARG VCS_REF diff --git a/Makefile b/Makefile index 9a528d9..115e83c 100644 --- a/Makefile +++ b/Makefile @@ -45,12 +45,18 @@ build-docker: ## Build the docker image test: ## Run the unit and integration testsuites. test: lint test-unit -lint: ## Run phpcs against the code. +lint: ## Check file syntax +lint: lint-php lint-md + +lint-php: ## Run phpcs against the code. ${DOCKER_RUN} vendor/bin/phpcs -p --warning-severity=0 src/ tests/ lint-fix: ## Run phpcsf and fix possible lint errors. ${DOCKER_RUN} vendor/bin/phpcbf -p src/ tests/ +lint-md: ## Check the markdown files + ${DOCKER} run --rm -v $$(pwd):/data:cached gouvinb/docker-markdownlint -v *.md docs/*.md docs/*/*.md + test-unit: ## Run the unit testsuite. ${DOCKER_RUN} vendor/bin/phpunit --testsuite unit @@ -76,6 +82,18 @@ test-coverage-html: ## Run all tests and output coverage to html. test-coverage-clover: ## Run all tests and output clover coverage to file. ${DOCKER_RUN} phpdbg7 -qrr vendor/bin/phpunit --coverage-clover=./tests/report/coverage.clover +# Documentation + +docs-test: ## Run docs test server + docker run --rm -it -p 8000:8000 -v $$(pwd):/docs squidfunk/mkdocs-material + +docs-build: ## Build the mkdocs documentation + docker run --rm -it -v $$(pwd):/docs squidfunk/mkdocs-material build + +docs-deploy: ## Deploy the mkdocs documentation + docker run --rm -it -v ~/.ssh:/root/.ssh -v $$(pwd):/docs -e GITHUB_TOKEN squidfunk/mkdocs-material gh-deploy + + # Help help: ## Show this help message. diff --git a/README.md b/README.md index d0d646e..67b0181 100644 --- a/README.md +++ b/README.md @@ -5,13 +5,13 @@ [![Build Status](https://img.shields.io/travis/graze/sprout/master.svg?style=flat-square)](https://travis-ci.org/graze/sprout) [![Coverage Status](https://img.shields.io/scrutinizer/coverage/g/graze/sprout.svg?style=flat-square)](https://scrutinizer-ci.com/g/graze/sprout/code-structure) [![Quality Score](https://img.shields.io/scrutinizer/g/graze/sprout.svg?style=flat-square)](https://scrutinizer-ci.com/g/graze/sprout) -[![Total Downloads](https://img.shields.io/packagist/dt/graze/sprout.svg?style=flat-square)](https://packagist.org/packages/graze/sprout) [![PHP Version](https://img.shields.io/packagist/php-v/graze/sprout.svg?style=flat-square)](https://php.net) [![Docker Image Size](https://img.shields.io/microbadger/image-size/graze/sprout.svg?style=flat-square)](https://hub.docker.com/r/graze/sprout/) +[![Docker Pulls](https://img.shields.io/docker/pulls/graze/sprout.svg?style=flat-square)](https://hub.docker.com/r/graze/sprout/) Sprout is a tool to help Dump, Truncate and Seed development data into your databases. -![](https://78.media.tumblr.com/534425eb11706448af8ce5838629f76d/tumblr_inline_n9t8gdzC7p1qzjzhu.gif) +![fun image](https://78.media.tumblr.com/534425eb11706448af8ce5838629f76d/tumblr_inline_n9t8gdzC7p1qzjzhu.gif) 1. Seed sql data from local files 1. Dump data from mysql tables @@ -34,17 +34,35 @@ docker run -v [volumes] --rm graze/sprout [command] ## Usage +### File Structure + +Sprout will use the following file structure by default, you can change the root and each group's path in the +configuration file. + +```text +- /seed + - group1 + - schema1 + - table1.sql + - table2.sql + - schema2 + - table3.sql + - group1 + - schema3 + - table4.sql +``` + ### Quick Start ```bash # Dump all tables you are interested in -sprout dump --config=config/sprout.yml --group=core a_schema:table_1,table_2 ... +sprout dump --config=config/sprout.yml --group=static a_schema:table_1,table_2 ... # Store the data in your repository of choice -git add /seed/data/* +git add /seed/static/* -# Seed the data from your seed data -sprout seed --config=config/sprout.yml --group=core +# Seed the data from your local files +sprout seed --config=config/sprout.yml --group=static ``` ### Seeding @@ -87,6 +105,48 @@ sprout dump --config=config/sprout.yml --group=core sprout dump --config=config/sprout.yml --group=core the_schema:country ``` +### Configuration + +The configuration file follows the following standards. + +By default sprout looks for a `config/sprout.yml` file, you can specify a different file +using `--config=path/to/file.yml`. + +```yaml +defaults: + group: core + # default path + path: /seed + # number of simultaneous processors to run at a time (default: 10) + simultaneousProcesses: 10 + +# ability to specify custom paths for groups +groups: + core: + path: /custom/path/to/group + +schemas: + # name of the schema to reference + : + # [optional] the actual name of the schema in the database. If not specified, from above will be used + schema: 'schema' + + # Connection details - this is just an example, you may want to specify + # different properties, e.g. if connecting to a remote server. You are + # advised to refer to the 'pdo' documentation for further details. + connection: + user: 'morphism' + password: 'morphism' + # driver for the database connection, currently only: `mysql` is supported + driver: 'mysql' + # [optional] name of the database + dbName: 'schema' + # database on a remote host + host: 'db' + # [optional] port to use, by default: 3306 + port: 3306 +``` + ## Testing ```bash @@ -109,4 +169,4 @@ instead of using the issue tracker. ## License -The MIT License (MIT). Please see [License File](LICENSE.md) for more information. +The MIT License (MIT). Please see [License File](LICENSE) for more information. diff --git a/bin/process-mysqldump b/bin/process-mysqldump new file mode 100755 index 0000000..36dbbe2 Binary files /dev/null and b/bin/process-mysqldump differ diff --git a/composer.json b/composer.json index 4d5a2a2..a958a9a 100644 --- a/composer.json +++ b/composer.json @@ -56,5 +56,9 @@ "Graze\\Sprout\\Test\\Unit\\": "tests/unit", "Graze\\Sprout\\Test\\Integration\\": "tests/integration" } - } + }, + "bin": [ + "./bin/sprout", + "./bin/process-mysqldump" + ] } diff --git a/dev.md b/dev.md index 546148d..c40f3ca 100644 --- a/dev.md +++ b/dev.md @@ -19,8 +19,8 @@ This is to spec out what we would require? - Be as quick as possible (use as many tricks to speed up seeding as possible) - Handle at least mysql - Group seeding into types (core, operational, testing) - - You could make a custom group for testing a particular problem ? -- When using a .sql file, the table must be truncated before seeding + - You could make a custom group for testing a particular problem ? +- When using a .sql file, the table must be truncated before seeding ### Extended requirements @@ -32,8 +32,8 @@ This is to spec out what we would require? - You have a series of grouped seed data (e.g. core, sample, test) - You can apply each of the seed data independently - - For example, you can 'apply' each test data seed at the beginning of each test run (or maybe even each test) - - test data should be idempotent (can be created and removed without modifying other test data) + - For example, you can 'apply' each test data seed at the beginning of each test run (or maybe even each test) + - test data should be idempotent (can be created and removed without modifying other test data) ## Commands @@ -92,7 +92,7 @@ default: # default path path: /seed -# ability to specify custom paths for groups +# ability to specify custom paths for groups groups: core: path: /custom/path/to/stuff @@ -101,7 +101,7 @@ schemas: # name of the schema in the database : - + # Connection details - this is just an example, you may want to specify # different properties, e.g. if connecting to a remote server. You are # advised to refer to the 'pdo' documentation for further details. @@ -139,11 +139,11 @@ schemas: 1. if doing this from within php, how can we parallelise -> spawn workers? 1. when you have multiple groups, how to do you handle truncation? 1. When truncating should you truncate all tables in a schema, irrespective of the seed data? -1. When dealing with projects that are not php, why would use +1. When dealing with projects that are not php, why would use ## Future / Scope 1. Should this be limited to sql dumping/restoring only? - 1. Alternative is to use php/yaml/json files too. + 1. Alternative is to use php/yaml/json files too. 1. Should this handle stored procedures? 1. How should this handle triggers / complex table setup? (prep tables, disable triggers, indexes, etc) diff --git a/docs/commands/chop.md b/docs/commands/chop.md new file mode 100644 index 0000000..09c49a3 --- /dev/null +++ b/docs/commands/chop.md @@ -0,0 +1,87 @@ +# Truncating Data (Chop) + +The chop command has the following structure: + +```bash +sprout chop [--config=] [--group=] [[:,...]] ... +``` + +This will truncate the contents of all the specified tables. + +!!! note + By default when you [seed](seed.md) data it will truncate the tables it finds first, so there is no need to do this + manually. + +## Configuration file + +The optional `--config` option allows you specify the configuration file to use. By default it will look for a file +called `config/sprout.yml`. The location is relative to the working directory. Within docker this is `/app`. + +## Schema and Table configuration + +All commands make use of the same [Schema and Tables](../schemas_tables.md) parsing. + +The `[[:
,...]] ...` part of the command line allows you to specify none or some schemas, each schema with +a set of tables or not. + +If no schema is defined, all the schemas and tables that are on the filesystem in a group will be truncated. +If no tables are defined for a schema, all tables on the filesystem will be truncated. + +### Chopping all the data + +You can truncate all the tables that exist locally if you do not specify and schemas or tables. + +```bash +sprout chop +``` + +This will truncate all the data in the default group. See [groups](#groups) for more information on how the groups work. + +### Truncating all the tables in a schema + +```bash +sprout chop schema1 +``` + +This will chop (truncate) all the tables that exist on the filesystem in the schema: `schema1`. + +You can truncate multiple schemas too: + +```bash +sprout chop schema1 schema2 +``` + +### Chopping specific tables + +If you only want to truncate a set of specific tables you can specify them as a comma separated list +after the schema they apply to. + +```bash +sprout chop schema1:table1,table2 +``` + +You can also specify multiple schemas, each with their own set of tables + +```bash +sprout chop schema1:table1,table2 schema2:table3 +``` + +## Groups + +The optional `--group` option allows you to specify which group to read the seed data from. If this is not +supplied it will use the default value as defined in the [configuration file](../setup/configuration.md). + +See [Chopping individual groups](../groups.md#you-can-truncate-from-a-group-as-well) for more information on how to +truncate data in groups. + +You can see all of the data in a group by calling: + +```bash +sprout chop --group=testing +``` + +Or you can limit it to a set of schemas and tables: + +```bash +sprout chop --group=testing schema1 schema2:table1,table2 +``` diff --git a/docs/commands/dump.md b/docs/commands/dump.md new file mode 100644 index 0000000..aa1c5b5 --- /dev/null +++ b/docs/commands/dump.md @@ -0,0 +1,82 @@ +# Dumping your data + +The dump command has the following structure: + +```bash +sprout dump [--config=] [--group=] [[:
,...]] ... +``` + +This will dump the contents of a table to a related file in the specified group as a collection of sql insert +statements. + +## Output File + +This is an example of the output from the dump command: + +```sql +INSERT INTO `country` +(`id`, `country_code`, `country_code_iso2`, `country_code_iso3`, `name`, `decimal_point`, `thousands_separator`, `added`, `updated`, `deleted`) VALUES +(1,'AF','AF','AFG','Afghanistan','.',',','2012-09-20 12:43:09','2015-10-07 11:05:58',NULL), +(2,'AX','AX','ALA','Aland Islands','.',',','2012-09-20 12:43:09','2015-10-07 11:05:58',NULL), +(3,'AL','AL','ALB','Albania','.',',','2012-09-20 12:43:09','2015-10-07 11:05:58',NULL), +(4,'DZ','DZ','DZA','Algeria','.',',','2012-09-20 12:43:09','2015-10-07 11:05:58',NULL); +``` + +It puts each row on a separate line to help with storing the data in your code repository and seeing what has changed +over time. + +## Configuration file + +The optional `--config` option allows you specify the configuration file to use. By default it will look for a file +called `config/sprout.yml`. The location is relative to the working directory. Within docker this is `/app`. + +## Schema and Table configuration + +All commands make use of the same [Schema and Tables](../schemas_tables.md) parsing. + +The `[[:
,...]] ...` part of the command line allows you to specify none or some schemas, each schema with +a set of tables or not. + +If no schema is defined, all the schemas and tables in a group will be dumped. +If no tables are defined for a schema, all tables previously dumped will be used. + +### Dumping all the files from a schema + +```bash +sprout dump schema1 +``` + +This will over-write all the existing dumps from `schema1` for the default group. + +You can dump multiple schemas: + +```bash +sprout dump schema1 schema2 +``` + +### Dumping specific tables + +If you only want to dump a set of specific tables (most likely scenario) you can specify them as a comma separated list +after the schema they apply to. + +```bash +sprout dump schema1:table1,table2 +``` + +You can also specify multiple schemas, each with their own set of tables + +```bash +sprout dump schema1:table1,table2 schema2:table3 +``` + +## Groups + +The optional `--group` option allows you to specify which group the following dump will belong to. If this is not +supplied it will use the default value as defined in the [configuration file](setup/configuration.md). + +See [Creating grouped seed data](../groups.md#creating-grouped-seed-data) for more information on how to dump data into +groups. + +```bash +sprout dump --group=testing schema1 schema2:table1,table2 +``` diff --git a/docs/commands/seed.md b/docs/commands/seed.md new file mode 100644 index 0000000..4d3dead --- /dev/null +++ b/docs/commands/seed.md @@ -0,0 +1,98 @@ +# Seeding Data + +The seed command has the following structure: + +```bash +sprout seed [--config=] [--no-chop] [--group=] [[:
,...]] ... +``` + +This will dump the contents of a table to a related file in the specified group as a collection of sql insert +statements. + +!!! note + Currently only `.sql` files are supported when seeding data. + +## Configuration file + +The optional `--config` option allows you specify the configuration file to use. By default it will look for a file +called `config/sprout.yml`. The location is relative to the working directory. Within docker this is `/app`. + +## Prevent truncation + +By default the seed command will truncate all the relevant tables. To prevent this you can specify the `--no-chop` +option. + +## Schema and Table configuration + +All commands make use of the same [Schema and Tables](../schemas_tables.md) parsing. + +The `[[:
,...]] ...` part of the command line allows you to specify none or some schemas, each schema with +a set of tables or not. + +If no schema is defined, all the schemas and tables in a group will be seeded. +If no tables are defined for a schema, all tables on the filesystem will be seeded. + +### Seeding all the data + +You can seed all the current data if you do not specify and schemas or tables. + +```bash +sprout seed +``` + +This will seed all the data in the default group. See [groups](#groups) for more information on how the groups work. + +### Seeding all the files in a schema + +```bash +sprout seed schema1 +``` + +This will chop (truncate) and seed all the tables that exist on the filesystem in the schema: `schema1`. + +If you do not wish to truncate the tables first, you can use: + +```bash +sprout seed --no-chop schema1 +``` + +You can seed multiple schemas too: + +```bash +sprout seed schema1 schema2 +``` + +### Seeding specific tables + +If you only want to seed a set of specific tables you can specify them as a comma separated list +after the schema they apply to. + +```bash +sprout seed schema1:table1,table2 +``` + +You can also specify multiple schemas, each with their own set of tables + +```bash +sprout seed schema1:table1,table2 schema2:table3 +``` + +## Groups + +The optional `--group` option allows you to specify which group to read the seed data from. If this is not +supplied it will use the default value as defined in the [configuration file](../setup/configuration.md). + +See [Seeding individual groups](../groups.md#seeding-individual-groups) for more information on how to seed data in +groups. + +You can see all of the data in a group by calling: + +```bash +sprout seed --group=testing +``` + +Or you can limit it to a set of schemas and tables: + +```bash +sprout seed --group=testing schema1 schema2:table1,table2 +``` diff --git a/docs/groups.md b/docs/groups.md new file mode 100644 index 0000000..7739d08 --- /dev/null +++ b/docs/groups.md @@ -0,0 +1,91 @@ +# Grouping Seed Data + +Not all seed data is made equal. + +For example you can have: + +- Static data: stuff that will not change whatever, types, etc +- Operational data: a minimum subset of data to make the application work +- Testing data: sample data that you can use to test the application with (accounts, products, etc) + +Sprout allows you to group your seed data into arbitrary categories. + +## Creating grouped seed data + +!!! note + Currently you must create the group/schema directory before running this. + +You can dump data from sprout into a grouped collection using the `--group=` option on all of the commands + +```bash +sprout dump --group=static schema:table1,table2 schema2:table3 ... +``` + +This will write the data in `table1` and `table2` from `schema` and `table3` from `schema2` into to `static` group +directory. Resulting in the data structure: + +```text +- /seed + - static + - schema + - table1.sql + - table2.sql + - schema2 + - table3.sql +``` + +If you do not specify a set of schemas or tables it will dump all the data from all the schemas and tables previously +dumped. + +```bash +sprout dump --group=static +``` + +### Dumping to a different group + +You can then dump your operational or testing data tables using a different group. + +```bash +sprout dump --group=operational schema:table4,table5 +``` + +This will result in the data structure: + +```text +- /seed + - operational + - schema + - table4.sql + - table5.sql + - static + - schema + - table1.sql + - table2.sql + - schema2 + - table3.sql +``` + +## Seeding individual groups + +You can then just seed an individual group using the same `--groups=` option. + +```bash +sprout seed --group=static +``` + +If you want to only seed a subset of the schemas, or tables you can still specify them in the command line. + +```bash +sprout seed --group=static schema1 + +sprout seed --group=static schema1:table1 schema2 +``` + +## You can truncate from a group as well + +!!! warning + If the same schema and table are in multiple groups, it will truncate the entire table + +```bash +sprout chop --group=static +``` diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 0000000..c4018c2 --- /dev/null +++ b/docs/index.md @@ -0,0 +1,56 @@ +# Sprout 🌱 + +Sprout is a tool to help you Populate your databases with seed data. + +It's core method is by using .sql files that can be committed and diffed by your code repository. + +It can: + +1. [Seed](commands/seed.md) sql data from local files +1. [Dump](commands/dump.md) data from mysql tables +1. [Chop (truncate)](commands/chop.md) data in mysql tables +1. Performs actions in parallel +1. Handle multiple [groups](groups.md) of seed data (for example, `static`, `core`, `testing`) + +## Quick Start + +### Getting Sprout + +You can get sprout in the following ways: + +1. [composer](setup/composer.md) to grab the application using PHP's composer +1. [docker](setup/docker.md) to run sprout without installing it locally + +### Configuration + +You will need a [configuration](setup/configuration.md) file to tell sprout how to talk to your databases. + +### Populate your seed data + +You can [group](groups.md) your seed data depending on its purpose. For example: `static`, `operational` and `development`. + +Ensure your database is populated with your seed data and run the following command: + +```bash +sprout dump --config=/path/to/config.yml --group=group1 schema:table1,table2,... schema2:table3,... ... +``` + +This will create a set of `.sql` files locally containing the data in your database. + +!!! warning + Currently you need to create the directories before running this command (or let it fail and tell you what you need + to create) + +### Seed your data + +You can now seed your data using the local files. + +```bash +sprout seed --config=/path/to/config.yml --group=group1 +``` + +If you do not need to truncate your tables first, use the `--no-chop` option. + +```bash +sprout seed --config=/path/to/config.yml --no-chop --group=group1 +``` diff --git a/docs/schemas_tables.md b/docs/schemas_tables.md new file mode 100644 index 0000000..262d1cd --- /dev/null +++ b/docs/schemas_tables.md @@ -0,0 +1,114 @@ +# Schemas and Tables + +All of the commands take a standardised set of schemas and tables that is used to build up what tables the action should +be performed on. + +The definition looks like: + +```text +[[:
,...]] ... +``` + +This means that it supports multiple optional schemas. Each schema supports an optional list of one or more tables. + +## Not specifying anything + +If you do not specify anything for this option. + +1. Sprout will look at the current schemas defined in your configuration file. +1. Filter out any schemas that do not have a directory in the current group. +1. Find all the tables for each schema. + +### Schema scanning example + +Given the directory structure: + +```text +/seed +- core + - schema1 + - table1 + - table2 + - schema2 + - table3 +- extra + - schema1 + - table4 +``` + +An empty schema `sprout ` will produce the equivalent of: + +```text +schema1:table1,table2 schema2:table3 +``` + +If you specified `sprout --group=extra` it would produce the equivalent to: + +```text +schema1:table4 +``` + +## Just specifying the schema + +You can specify one or more schemas. This will look at all the tables that are defined for only the specified schemas. + +The schema values specified should be the names of the schemas defined in the configuration file. +This does not necessary need to match the name of the actual schema. + +Given the directory structure: + +```text +/seed +- core + - schema1 + - table1 + - table2 + - schema2 + - table3 + - schema3 + - table5 +- extra + - schema1 + - table4 +``` + +The command: + +```bash +sprout schema1 schema2 +``` + +Will be the equivalent of: + +```bash +sprout schema1:table1,table2 schema2:table3 +``` + +### Specifying a schema that does not exist locally + +If you specify a schema (without any tables) that does not exist in the file structure, it will be ignored. + +```bash +sprout --group=extra schema1 schema2 +``` + +Would be the equivalent to: + +```bash +sprout --group=extra schema1:table4 +``` + +## Schemas with Tables + +You are able to specify a schema with a collection of tables. This will not look at the filesystem to see if they exist +or not, which allows you to dump schemas and tables that do not exist yet. + +```bash +sprout schema1:table1,table2 +``` + +Any number of schemas and table combinations can be used + +```bash +sprout schema1:table1,table2 schema2:table3 schema3 +``` diff --git a/docs/setup/composer.md b/docs/setup/composer.md new file mode 100644 index 0000000..7282594 --- /dev/null +++ b/docs/setup/composer.md @@ -0,0 +1,33 @@ +# Running sprout with docker + +Sprout can be run locally in PHP by including it from composer. + +Add it to your project using: + +```bash +composer require graze/sprout +``` + +You can now run it using: + +```bash +./vendor/bin/sprout [command] +``` + +## Versions + +You can specify which version you require through composer: + +```bash +composer require graze/sprout:^0.1 +``` + +or in the `composer.json` file: + +```json +{ + "require": { + "graze/sprout": "^0.1" + } +} +``` diff --git a/docs/setup/configuration.md b/docs/setup/configuration.md new file mode 100644 index 0000000..aca65af --- /dev/null +++ b/docs/setup/configuration.md @@ -0,0 +1,86 @@ +# Sprout Configuration File + +The configuration file follows the following standards. + +By default sprout looks for a `config/sprout.yml` file, you can specify a different file +using `--config=path/to/file.yml`. This file is relative to the current working directory. + +## Complete Configuration File Example + +```yaml +# [optional] default properties to use +defaults: + + # [optional, string, default: `core`] The default group to use if `--group` is not specified + group: core + + # [optional, string, default: `/seed`] The root path of all the seed data + path: /seed + + # [optional, int, default: `10`] Maximum number of simultaneous processes to run at a time + simultaneousProcesses: 10 + +# [optional] The groups root collection allows you to specify custom paths for an individual group +groups: + + # The name of the group (used when specifying `--group=` + core: + + # [optional, string] A custom path to the group if it does not follow the `/root/group/` hierarchy. This is absolute + # or relative to the working directory + path: /custom/path/to/group + +# [required, min: 1] Schemas specify each schema you wish to seed in the database and their connection information +schemas: + + # name of the schema to reference. There must be at least 1 schema in the configuration file + schema1: + + # [optional, string] the actual name of the schema in the database. If not specified, the schema name from + # above will be used + schema: 'schema' + + # [optional, string] A custom directory name for this schema, only required if it is different from the `schema` value + dirName: directory + + # [required] Connection details + connection: &default_connection + + # [required, string] Username of the database connection + user: 'morphism' + + # [required, string] Password of the database connection + password: 'morphism' + + # [required] driver for the database connection, currently only: `mysql` is supported + driver: 'mysql' + + # [required, string] database on a remote host + host: 'db' + + # [optional, string] name of the database. If not specified `` will be used + dbName: 'schema' + + # [optional, int, default: `3306`] The port to connect on + port: 3306 + + # A second schema, + schema2: + connection: + # you can use yaml anchors to reduce duplicate data + <<: *default_connection +``` + +## Minimal Configuration File + +This is a minimal configuration file with a single schema and connection + +```yaml +schemas: + first_schema: + connection: &connection + host: db + driver: mysql + user: root + password: rootpassword +``` diff --git a/docs/setup/docker.md b/docs/setup/docker.md new file mode 100644 index 0000000..9345343 --- /dev/null +++ b/docs/setup/docker.md @@ -0,0 +1,62 @@ +# Running sprout with docker + +There is a docker image: `graze/sprout` on [DockerHub](https://hub.docker.com/r/graze/sprout) that can be used to run +sprout. + +When specifying paths in the configuration file, remember they are relative to the sprout working directory +(default: `/app`). + +You will need to mount your configuration and seed data: + +## Command Line + +This will run sprout with seed data in the default `/seed` path. + +```bash +docker run --rm -v $$(pwd)/config/sprout:/app/config -v $$(pwd)/seed:/seed graze/sprout [command] +``` + +## Docker Compose + +An example `docker-compose.yml` file. + +```yaml +version: '2' + +services: + sprout: + image: graze/sprout + depends_on: + - db + volumes: + - ./config/sprout:/app/config:cached + - ./seed:/seed:delegated + + db: + image: mysql:5 + environment: + MYSQL_USER: dev + MYSQL_PASSWORD: password + MYSQL_DATABASE: the_schema + MYSQL_ROOT_PASSWORD: rootpassword +``` + +With this file you can run sprout using the commands: + +```bash +docker-compose run --rm sprout [command] +``` + +## Versions + +Different versions can be used by defining tags on the image: + +```yaml +services: + sprout: + image: graze/sprout:0.1 +``` + +```bash +docker run --rm -v $(pwd)/config/sprout:/app/config -v $(pwd)/seed:/seed graze/sprout:0.1 [command] +``` diff --git a/docs/sprout.md b/docs/sprout.md deleted file mode 100644 index 2744533..0000000 --- a/docs/sprout.md +++ /dev/null @@ -1,7 +0,0 @@ -# Sprout - -Sprout is a tool to help you Populate your databases with seed data. - -It's core method is by using .sql files that can be committed and diffed by your code repository. - - diff --git a/dump-parser/process-mysqldump.c b/dump-parser/process-mysqldump.c new file mode 100644 index 0000000..adbd2f5 --- /dev/null +++ b/dump-parser/process-mysqldump.c @@ -0,0 +1,130 @@ +// taken from: https://gist.github.com/colinmollenhour/cf23b0f7e955267ed1107c9edb07f7c2 + +// gcc -O2 -Wall -pedantic process-mysqldump.c -o process-mysqldump +// Usage: cat dump.sql | process-mysqldump +// Or : process-mysqldump dump.sql + +#include +#include +#include +#include + +#define BUFFER 100000 + +bool is_escaped(char* string, int offset) { + if (offset == 0) { + return false; + } else if (string[offset - 1] == '\\') { + return !is_escaped(string, offset - 1); + } else { + return false; + } +} + +bool is_insert(char* string) { + char buffer[] = "INSERT INTO "; + + return strncmp(buffer, string, 12) == 0; +} + +int main(int argc, char *argv[]) +{ + FILE* file = argc > 1 ? fopen(argv[1], "rb") : stdin; + + char buffer[BUFFER]; + char* line; + int pos; + int parenthesis = 0; + bool quote = false; + bool escape = false; + bool check_prefix = true; + bool wasnt_insert = false; + + while (fgets(buffer, BUFFER, file) != NULL) { + line = buffer; + + // skip non-INSERT INTO statements + if (check_prefix && (wasnt_insert || ! is_insert(line))) { + check_prefix = line[strlen(line) - 1] == '\n'; + wasnt_insert = ! check_prefix; + fputs(line, stdout); + continue; + } + + check_prefix = line[strlen(line) - 1] == '\n'; + pos = 0; + + nullchar: + while (line[pos] != '\0') { + // if we are still in escape state, we need to check first char. + if (!escape) { + // find any character in ()' + pos = strcspn(line, "()'\\"); + } + + if (pos > 0) { + // print before match + printf("%.*s", pos, line); + } + + switch (line[pos]) { + case '(': + if (!quote) { + if (parenthesis == 0) { + puts(""); + } + parenthesis++; + } + if (escape) { + escape = false; + } + break; + + case ')': + if (!quote) { + if (parenthesis > 0) { + parenthesis--; + } else { + // whoops + puts(""); + fputs(line, stdout); + fputs("Found closing parenthesis without opening one.\n", stderr); + exit(1); + } + } + if (escape) { + escape = false; + } + break; + + case '\\': + escape = !escape; + break; + + case '\'': + if (escape) { + escape = false; + } else { + quote = !quote; + } + break; + + case '\0': + goto nullchar; + + default: + if (escape) { + escape = false; + } + break; + } + + // print char then skip it (to make sure we don’t double match) + putchar(line[pos]); + line = line + pos + 1; + pos = 0; + } + } + + return 0; +} diff --git a/example/Makefile b/example/Makefile index ef5edff..b9ae9bf 100644 --- a/example/Makefile +++ b/example/Makefile @@ -10,11 +10,11 @@ morphism: start-db seed: ## Seed the data for the database seed: start-db - docker-compose run --rm sprout seed -t the_schema -v + docker-compose run --rm sprout seed the_schema -v seed-group: ## Seed the data in a group for a database seed-group: start-db - docker-compose run --rm sprout seed -t --group=core -v + docker-compose run --rm sprout seed --group=core -v chop: ## Truncate the data in the database chop: start-db diff --git a/example/data/core/the_schema/country.sql b/example/data/core/the_schema/country.sql index 45e09f6..7fdd57a 100644 --- a/example/data/core/the_schema/country.sql +++ b/example/data/core/the_schema/country.sql @@ -1,4 +1,5 @@ -INSERT INTO `country` (`id`, `country_code`, `country_code_iso2`, `country_code_iso3`, `name`, `decimal_point`, `thousands_separator`, `added`, `updated`, `deleted`) VALUES +INSERT INTO `country` +(`id`, `country_code`, `country_code_iso2`, `country_code_iso3`, `name`, `decimal_point`, `thousands_separator`, `added`, `updated`, `deleted`) VALUES (1,'AF','AF','AFG','Afghanistan','.',',','2012-09-20 12:43:09','2015-10-07 11:05:58',NULL), (2,'AX','AX','ALA','Aland Islands','.',',','2012-09-20 12:43:09','2015-10-07 11:05:58',NULL), (3,'AL','AL','ALB','Albania','.',',','2012-09-20 12:43:09','2015-10-07 11:05:58',NULL), diff --git a/mkdocs.yml b/mkdocs.yml new file mode 100644 index 0000000..54e45b7 --- /dev/null +++ b/mkdocs.yml @@ -0,0 +1,55 @@ +site_name: Sprout +site_description: Sprout seeds your data +site_author: graze.com + +repo_url: https://github.com/graze/sprout +repo_name: 'GitHub' + +docs_dir: 'docs' + +theme: + name: 'material' + custom_dir: theme + favicon: assets/images/favicon.ico + language: en + include_sidebar: true + palette: + primary: 'cyan' + +copyright: "Copyright © 2018 Nature Delivered Ltd." + +nav: + - 'Getting Started': 'index.md' + - Setup: + - Configuration File: 'setup/configuration.md' + - Composer: 'setup/composer.md' + - Docker: 'setup/docker.md' + - Commands: + - Seed: 'commands/seed.md' + - Dump: 'commands/dump.md' + - Chop: 'commands/chop.md' + - Groups: 'groups.md' + - 'Schemas and Tables': 'schemas_tables.md' + +markdown_extensions: + - admonition: + - markdown.extensions.tables: + - pymdownx.magiclink: + - pymdownx.tilde: + - pymdownx.betterem: + - pymdownx.superfences: + - pymdownx.highlight: + css_class: codehilite + extend_pygments_lang: + - name: php + lang: php + options: + startinline: True + - pymdownx.inlinehilite: + - pymdownx.emoji: + - pymdownx.escapeall: + hardbreak: True + nbsp: True + - pymdownx.tasklist: + - toc: + permalink: true diff --git a/src/Command/SeedCommand.php b/src/Command/SeedCommand.php index b82b2bd..72e1250 100644 --- a/src/Command/SeedCommand.php +++ b/src/Command/SeedCommand.php @@ -31,7 +31,7 @@ class SeedCommand extends Command { const OPTION_CONFIG = 'config'; - const OPTION_CHOP = 'chop'; + const OPTION_NO_CHOP = 'no-chop'; const OPTION_GROUP = 'group'; const ARGUMENT_SCHEMA_TABLES = 'schemaTables'; @@ -49,10 +49,10 @@ protected function configure() ); $this->addOption( - static::OPTION_CHOP, - 't', + static::OPTION_NO_CHOP, + '', InputOption::VALUE_NONE, - 'chop (truncate) tables before seeding them' + 'do not chop (truncate) tables before seeding them' ); $this->addOption( @@ -82,7 +82,7 @@ protected function execute(InputInterface $input, OutputInterface $output) $config = (new Config())->parse($input->getOption('config')); $group = $input->getOption('group') ?: $config->get(Config::CONFIG_DEFAULT_GROUP); - if ($input->getOption(static::OPTION_CHOP)) { + if (!$input->getOption(static::OPTION_NO_CHOP)) { $chopCommand = new ChopCommand(); $exitCode = $chopCommand->run( new ArrayInput([ diff --git a/src/Config/Config.php b/src/Config/Config.php index ec00741..6f50c3a 100644 --- a/src/Config/Config.php +++ b/src/Config/Config.php @@ -89,11 +89,11 @@ public function parse(string $path): Config $config = $this->validator->validate($fileConfig); - // populates the schema / connection.dbname properties for each defined schema if not set + // populates the schema / connection.dbName properties for each defined schema if not set $schemas = $config['schemas']; foreach ($schemas as $schema => $value) { + // this will populate any missing values with their defaults $value = SchemaConfig::getValidator()->validate($value); - $value['connection'] = ConnectionConfig::getValidator()->validate($value['connection']); if (is_null($value['schema'])) { $config['schemas'][$schema]['schema'] = $schema; diff --git a/src/Dump/Mysql/MysqlTableDumper.php b/src/Dump/Mysql/MysqlTableDumper.php index 7999221..a4e9303 100644 --- a/src/Dump/Mysql/MysqlTableDumper.php +++ b/src/Dump/Mysql/MysqlTableDumper.php @@ -48,8 +48,8 @@ public function dump(string $schema, string $table, string $file) $process->setCommandLine( sprintf( 'mysqldump -h%1$s -u%2$s -p%3$s --compress --compact --no-create-info' . - ' --extended-insert --quick --complete-insert %4$s %5$s' . - '| sed \'s$VALUES ($VALUES\n($g\' | sed \'s$),($),\n($g\' > %6$s', + ' --extended-insert --hex-blob --quick --complete-insert %4$s %5$s ' . + '| process-mysqldump > %6$s', escapeshellarg($this->connection->getHost()), escapeshellarg($this->connection->getUser()), escapeshellarg($this->connection->getPassword()), diff --git a/src/Parser/SchemaParser.php b/src/Parser/SchemaParser.php index 725ef05..5d55189 100644 --- a/src/Parser/SchemaParser.php +++ b/src/Parser/SchemaParser.php @@ -44,7 +44,7 @@ function (Config\SchemaConfigInterface $schemaConfig) { foreach ($schemaTables as $schemaTable) { if (preg_match('/^([a-z_]+):(.+)$/i', $schemaTable, $matches)) { $schema = $matches[1]; - $tables = $matches[2] === '*' ? [] : explode(',', $matches[2]); + $tables = explode(',', $matches[2]); } else { $schema = $schemaTable; $tables = []; diff --git a/tests/unit/Dump/Mysql/MysqlTableDumperTest.php b/tests/unit/Dump/Mysql/MysqlTableDumperTest.php index 65c7e46..a16d517 100644 --- a/tests/unit/Dump/Mysql/MysqlTableDumperTest.php +++ b/tests/unit/Dump/Mysql/MysqlTableDumperTest.php @@ -21,8 +21,8 @@ public function testDump() $process->shouldReceive('setCommandLine') ->with( 'mysqldump -h\'some-host\' -u\'some-user\' -p\'some-pass\' --compress --compact --no-create-info' . - ' --extended-insert --quick --complete-insert \'some-schema\' \'some-table\'' . - '| sed \'s$VALUES ($VALUES\n($g\' | sed \'s$),($),\n($g\' > \'some-file\'' + ' --extended-insert --hex-blob --quick --complete-insert \'some-schema\' \'some-table\' ' . + '| process-mysqldump > \'some-file\'' ) ->once(); diff --git a/theme/assets/images/favicon.ico b/theme/assets/images/favicon.ico new file mode 100644 index 0000000..0a42ea7 Binary files /dev/null and b/theme/assets/images/favicon.ico differ