Skip to content
This repository has been archived by the owner on Feb 3, 2022. It is now read-only.

Commit

Permalink
#87: feat(plugin): add jobs plugin (alpha)
Browse files Browse the repository at this point in the history
#87: feat(plugin): add jobs plugin (alpha)
  • Loading branch information
rustatian committed Aug 12, 2021
2 parents 21a08ea + 87d501b commit f9a2106
Show file tree
Hide file tree
Showing 11 changed files with 317 additions and 28 deletions.
12 changes: 6 additions & 6 deletions .github/dependabot.yml
Original file line number Diff line number Diff line change
Expand Up @@ -10,24 +10,24 @@ updates:
schedule:
interval: daily
reviewers:
- "48d90782"
- "rustatian"
assignees:
- "48d90782"
- "rustatian"

- package-ecosystem: "github-actions"
directory: "/"
schedule:
interval: weekly
reviewers:
- "48d90782"
- "rustatian"
assignees:
- "48d90782"
- "rustatian"

- package-ecosystem: "docker"
directory: "/"
schedule:
interval: daily
reviewers:
- "48d90782"
- "rustatian"
assignees:
- "48d90782"
- "rustatian"
2 changes: 1 addition & 1 deletion .github/workflows/release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ jobs:
- name: Set up Go
uses: actions/setup-go@v2
with:
go-version: 1.16.4
go-version: 1.16.6

- name: Check out code
uses: actions/checkout@v2
Expand Down
8 changes: 4 additions & 4 deletions .github/workflows/tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ jobs:
run: npm install -g ajv-cli # Package page: <https://www.npmjs.com/package/ajv-cli>

- name: Run linter
run: ajv validate --all-errors --verbose -s ./schemas/config/2.0.schema.json -d ./.rr*.y*ml
run: ajv validate --all-errors --verbose -s ./schemas/config/2.0.schema.json -d ./.rr.yaml

golangci-lint:
name: Golang-CI (lint)
Expand All @@ -43,7 +43,7 @@ jobs:
- name: Run linter
uses: golangci/golangci-lint-action@v2 # Action page: <https://github.com/golangci/golangci-lint-action>
with:
version: v1.38 # without patch version
version: v1.41 # without patch version
only-new-issues: false # show only new issues if it's a pull request
args: --timeout=10m

Expand All @@ -54,7 +54,7 @@ jobs:
- name: Set up Go
uses: actions/setup-go@v2
with:
go-version: 1.16.4
go-version: 1.16.6

- name: Check out code
uses: actions/checkout@v2
Expand Down Expand Up @@ -92,7 +92,7 @@ jobs:
- name: Set up Go
uses: actions/setup-go@v2 # action page: <https://github.com/actions/setup-go>
with:
go-version: 1.16.4
go-version: 1.16.6

- name: Check out code
uses: actions/checkout@v2
Expand Down
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -25,3 +25,4 @@ dir1
coverage
/rr
rr.exe
.rr-sample-bench.yaml
22 changes: 22 additions & 0 deletions .rr-sample-bench.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
rpc:
listen: unix:///tmp/rr.sock

server:
command: "php ../roadrunner/tests/psr-worker-bench.php"
relay: "pipes"
relay_timeout: "20s"

http:
address: 127.0.0.1:15395
max_request_size: 1024
middleware: [ ]
trusted_subnets: [ "10.0.0.0/8", "127.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16", "::1/128", "fc00::/7", "fe80::/10" ]
pool:
num_workers: 10
max_jobs: 0
allocate_timeout: 60s
destroy_timeout: 60s

logs:
mode: production
level: error
204 changes: 203 additions & 1 deletion .rr.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ server:
# This option is required.
command: "php psr-worker.php"

# User name (not UID) for the worker processes. An empty value means to use the RR process user.
# Username (not UID) for the worker processes. An empty value means to use the RR process user.
#
# Default: ""
user: ""
Expand Down Expand Up @@ -717,6 +717,208 @@ reload:
# Default: []
patterns: [ ".php", ".go", ".md" ]

# AMQP jobs driver
#
# This option is required to use AMQP driver
amqp:
# AMQP Uri to connect to the rabbitmq server https://www.rabbitmq.com/uri-spec.html
#
# This option is required for the production. Default: amqp://guest:[email protected]:5672
addr: amqp://guest:[email protected]:5672/

# Beanstalk jobs driver
#
# This option is required to use Beanstalk driver
beanstalk:
# Beanstalk address
#
# This option is required for the production. Default: tcp://127.0.0.1:11300
addr: tcp://127.0.0.1:11300

# Beanstalk connect timeout.
#
# Default: 30s
timeout: 10s

# SQS jobs driver (https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html)
#
# This option is required to use SQS driver
sqs:
# AccessKey ID
#
# This option is required for the production. Default: empty
key: api-key

# Secret access key
#
# This option is required for the production. Default: empty
secret: api-secret

# AWS region
#
# This option is required for the production. Default: empty
region: us-west-1

# AWS session token
#
# This option is required for the production. Default: empty
session_token: test

# AWS SQS endpoint to connect
#
# This option is required for the production. Default: http://127.0.0.1:9324
endpoint: http://127.0.0.1:9324

jobs:
# Number of threads which will try to obtain the job from the priority queue
#
# Default: number of the logical CPU cores
num_pollers: 32

# Size of the internal priority queue
#
# Default: 1_000_000
pipeline_size: 100000

# worker pool configuration
pool:
num_workers: 10
max_jobs: 0
allocate_timeout: 60s
destroy_timeout: 60s

# List of broker pipelines associated with the drivers.
#
# This option is not required since you can declare pipelines in the runtime. Pipeline driver should exist.
pipelines:
# Pipeline name
#
# This option is required when defining pipelines via configuration.
test-local:
# Driver associated with the pipeline
#
# This option is required. Possible values: amqp, ephemeral, sqs, beanstalk
driver: ephemeral
# Pipeline priority
#
# If the job has priority set to 0, it will inherit the pipeline's priority. Default: 10.
priority: 10
# Number of job to prefetch from the driver.
#
# Default: 100_000.
prefetch: 10000

test-local-2:
# Driver name
#
# This option is required.
driver: amqp
# QoS - prefetch.
#
# Default: 10
prefetch: 10
# Queue name
#
# Default: default
queue: test-1-queue
# Pipeline jobs priority, 1 - highest
#
# Default: 10
priority: 1
# Exchange name
#
# Default: amqp.default
exchange: default
# Exchange type
#
# Default: direct.
exchange_type: direct
# Routing key for the queue
#
# Default: empty.
routing_key: test
# Declare a queue exclusive at the exchange
#
# Default: false
exclusive: false
# When multiple is true, this delivery and all prior unacknowledged deliveries
# on the same channel will be acknowledged. This is useful for batch processing
# of deliveries
#
# Default:false
multiple_ack: false
# When multiple is true, this delivery and all prior unacknowledged deliveries
# on the same channel will be acknowledged. This is useful for batch processing
# of deliveries
#
# Default: false
requeue_on_fail: false


test-local-3:
# Driver name
#
# This option is required.
driver: beanstalk
# Pipeline jobs priority, 1 - highest
#
# Default: 10
priority: 11
# Beanstalk internal tube priority
#
# Default: 1
tube_priority: 1
# Tube name
#
# Default: default
tube: default-1
# If no job is available before this timeout has passed, Reserve returns a ConnError recording ErrTimeout.
#
# Default: 5s
reserve_timeout: 10s

test-local-4:
# Driver name
#
# This option is required.
driver: sqs
# Number of jobs to prefetch from the SQS. mazon SQS never returns more messages than this value
# (however, fewer messages might be returned). Valid values: 1 to 10.
#
# Default: 10
prefetch: 10
# The duration (in seconds) that the received messages are hidden from subsequent
# retrieve requests after being retrieved by a ReceiveMessage request
#
# Default: 0
visibility_timeout: 0
# The duration (in seconds) for which the call waits for a message to arrive
# in the queue before returning. If a message is available, the call returns
# sooner than WaitTimeSeconds. If no messages are available and the wait time
# expires, the call returns successfully with an empty list of messages.
#
# Default: 0
wait_time_seconds: 0
# Queue name.
#
# Default: default
queue: default
# List of the AWS SQS attributes https://docs.aws.amazon.com/AWSSimpleQueueService/latest/APIReference/API_SetQueueAttributes.html.
attributes:
DelaySeconds: 0
MaximumMessageSize: 262144
MessageRetentionPeriod: 345600
ReceiveMessageWaitTimeSeconds: 0
VisibilityTimeout: 30
# Tags don't have any semantic meaning. Amazon SQS interprets tags as character
# strings.
tags:
test: "tag"


# list of pipelines to be consumed by the server, keep empty if you want to start consuming manually
consume: [ "test-local", "test-local-2", "test-local-3", "test-local-4" ]

## RoadRunner internal container configuration (docs: https://github.com/spiral/endure).
endure:
# How long to wait for stopping.
Expand Down
18 changes: 18 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,12 +1,30 @@
CHANGELOG
=========

v2.4.0 (-.08.2021)
-------------------

## 💔 Internal BC:

- 🔨 Pool, worker interfaces: payload now passed and returned by pointer.

## 👀 New:

- ✏️ Long awaited, reworked `Jobs` plugin with pluggable drivers. Now you can allocate/destroy pipelines in the runtime.
Drivers included in the initial release: `RabbitMQ (0-9-1)`, `SQS v2`, `beanstalk`, `ephemeral`
. [PR](https://github.com/spiral/roadrunner/pull/726)

## 🩹 Fixes:

- 🐛 Fix: add `debug` pool config key to the `.rr.yaml`
configuration [reference](https://github.com/spiral/roadrunner-binary/issues/79).
- 🐛 Fix: fixed bug with waiting goroutines on the internal worker's container channel.

## 📦 Packages:

- 📦 Update goridge to `v3.2.0`
- 📦 Update temporal to `v1.0.9`
- 📦 Update RR to `v2.4.0`

## 📈 Summary:

Expand Down
6 changes: 3 additions & 3 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -14,10 +14,10 @@ require (
// SPIRAL ------------
github.com/spiral/endure v1.0.2
github.com/spiral/errors v1.0.11
github.com/spiral/goridge/v3 v3.1.4
github.com/spiral/roadrunner/v2 v2.3.2
github.com/spiral/goridge/v3 v3.2.0
github.com/spiral/roadrunner/v2 v2.4.0-alpha.1
// ---------------------
github.com/stretchr/testify v1.7.0
github.com/temporalio/roadrunner-temporal v1.0.8
github.com/temporalio/roadrunner-temporal v1.0.9-alpha.1
github.com/vbauerster/mpb/v5 v5.4.0
)
Loading

0 comments on commit f9a2106

Please sign in to comment.