Skip to content

Commit ee929c1

Browse files
daniellienertdaniellienert
committed
FEATURE: Define patterns for index names and clone aliases
The index can now contain wildcards to define a pattern to specify which indices to copy. Aliases are now also cloned.
1 parent 7b12d34 commit ee929c1

File tree

8 files changed

+173
-51
lines changed

8 files changed

+173
-51
lines changed

‎Classes/Configuration/PresetConfiguration.php

+14
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,9 @@
1010

1111
use PunktDe\Elastic\Sync\Exception\ConfigurationException;
1212

13+
/*
14+
* Defines a clone preset including the remote elasticearch configuration
15+
*/
1316
class PresetConfiguration
1417
{
1518
/**
@@ -126,4 +129,15 @@ public function getPostCloneConfiguration(string $postCloneStep = ''): ?array
126129

127130
return $this->postCloneConfiguration[$postCloneStep] ?? [];
128131
}
132+
133+
/**
134+
* @return $this
135+
*/
136+
public function withTunneledConnection(): self
137+
{
138+
$new = clone $this;
139+
$new->elasticsearchPort = 9210;
140+
$new->elasticsearchHost = '127.0.0.1';
141+
return $new;
142+
}
129143
}

‎Classes/Exception/ElasticsearchException.php

+14
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
<?php
2+
declare(strict_types=1);
3+
4+
namespace PunktDe\Elastic\Sync\Exception;
5+
6+
/*
7+
* (c) 2020 punkt.de GmbH - Karlsruhe, Germany - http://punkt.de
8+
* All rights reserved.
9+
*/
10+
11+
class ElasticsearchException extends \Exception
12+
{
13+
14+
}

‎Classes/Service/ElasticsearchService.php

+38-1
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,6 @@
88
* All rights reserved.
99
*/
1010

11-
1211
use Neos\Flow\Annotations as Flow;
1312
use Neos\Flow\Http\Client\CurlEngine;
1413
use Neos\Flow\Http\Client\CurlEngineException;
@@ -17,7 +16,9 @@
1716
use Neos\Http\Factories\UriFactory;
1817
use Psr\Http\Message\ServerRequestFactoryInterface;
1918
use PunktDe\Elastic\Sync\Configuration\PresetConfiguration;
19+
use PunktDe\Elastic\Sync\Exception\ElasticsearchException;
2020
use PunktDe\Elastic\Sync\Exception\HttpException;
21+
use PunktDe\Elastic\Sync\Exception\SynchronizationException;
2122

2223
/**
2324
* @Flow\Scope("singleton")
@@ -59,6 +60,42 @@ public function deleteIndex(PresetConfiguration $configuration, string $indexNam
5960
return (int)$response->getStatusCode();
6061
}
6162

63+
/**
64+
* @param PresetConfiguration $configuration
65+
* @param string $indexName
66+
* @return array
67+
* @throws CurlEngineException
68+
* @throws Exception
69+
* @throws ElasticsearchException
70+
*/
71+
public function getIndices(PresetConfiguration $configuration, string $indexName): array
72+
{
73+
$uri = $this->getBaseUri($configuration)
74+
->withPath('/_cat/indices/' . $indexName)
75+
->withQuery('format=JSON');
76+
77+
$request = $this->serverRequestFactory->createServerRequest('GET', $uri);
78+
$response = (new CurlEngine())->sendRequest($request);
79+
80+
$result = json_decode($response->getBody()->getContents(), true);
81+
$this->checkForElasticsearchErrorObject($result);
82+
return $result;
83+
}
84+
85+
/**
86+
* @param array $result
87+
* @throws ElasticsearchException
88+
*/
89+
public function checkForElasticsearchErrorObject(array $result): void
90+
{
91+
if (isset($result['error'])) {
92+
$reason = current($result['error']['root_cause'])['reason'] ?? 'No Reason given';
93+
$status = $result['status'] ?? 'Unknown status';
94+
95+
throw new ElasticsearchException(sprintf('Elasticsearch query failed with status "%s": %s', $status, $reason), 1603000587);
96+
}
97+
}
98+
6299
/**
63100
* @param PresetConfiguration $configuration
64101
* @param string $aliasName

‎Classes/Synchronizer.php

+55-24
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
use PunktDe\Elastic\Sync\Exception\ConfigurationException;
1919
use PunktDe\Elastic\Sync\Exception\SynchronizationException;
2020
use PunktDe\Elastic\Sync\Service\ElasticsearchService;
21+
use PunktDe\Elastic\Sync\Service\ShellCommandService;
2122

2223
class Synchronizer
2324
{
@@ -50,6 +51,12 @@ class Synchronizer
5051
*/
5152
protected $elasticSeacrhService;
5253

54+
/**
55+
* @Flow\Inject
56+
* @var ShellCommandService
57+
*/
58+
protected $shellCommandService;
59+
5360
public function __construct()
5461
{
5562
$this->consoleOutput = new ConsoleOutput();
@@ -65,43 +72,48 @@ public function sync(string $presetName): void
6572
$remoteConfiguration = $this->configurationService->getRemoteConfiguration($presetName);
6673
$remoteInstanceConfiguration = $this->presetConfigurationFactory->getRemoteInstanceConfiguration($presetName);
6774

68-
$this->compileAndRunCloneScript($remoteConfiguration, $localConfiguration, $remoteInstanceConfiguration);
69-
$this->createAliases($localConfiguration);
75+
$sshTunnelPid = $this->shellCommandService->openSshTunnelToRemoteElasticsearchServer($remoteConfiguration, $remoteInstanceConfiguration);
76+
77+
try {
78+
$this->compileAndRunCloneScript($remoteConfiguration, $localConfiguration, $remoteInstanceConfiguration);
79+
$this->createAdditionalAliases($localConfiguration);
80+
} catch (\Exception $exception) {
81+
$this->consoleOutput->output('<error>%s</error>', [$exception->getMessage()]);
82+
} finally {
83+
$this->shellCommandService->closeSshTunnelToRemoteElasticsearchServer($sshTunnelPid);
84+
}
7085
}
7186

7287

7388
/**
7489
* @param PresetConfiguration $remoteConfiguration
7590
* @param PresetConfiguration $localConfiguration
7691
* @param RemoteInstanceConfiguration $remoteInstanceConfiguration
92+
* @throws SynchronizationException
93+
* @throws \Neos\Flow\Http\Client\CurlEngineException
94+
* @throws \Neos\Flow\Http\Exception
7795
*/
7896
private function compileAndRunCloneScript(PresetConfiguration $remoteConfiguration, PresetConfiguration $localConfiguration, RemoteInstanceConfiguration $remoteInstanceConfiguration): void
7997
{
98+
$tunneledRemoteConfiguration = $remoteConfiguration->withTunneledConnection();
8099

81-
$indexConfiguration = [];
100+
$indexConfigurations = [];
82101
foreach ($remoteConfiguration->getIndices() as $key => $index) {
83-
$indexConfiguration[$key] = [
84-
'remote' => $index,
85-
'local' => $localConfiguration->getIndices()[$key]
86-
];
102+
$indexConfigurations[$key] = $this->checkAndExpandRemoteIndices($tunneledRemoteConfiguration, $index['indexName']);
87103
}
88104

89-
try {
90-
$view = new StandaloneView();
91-
$view->setTemplatePathAndFilename('resource://PunktDe.Elastic.Sync/Private/Template/CopyElastic.sh.template');
92-
$view->assignMultiple([
93-
'localConfiguration' => $localConfiguration,
94-
'remoteConfiguration' => $remoteConfiguration,
95-
'remoteInstance' => $remoteInstanceConfiguration,
96-
'indices' => $indexConfiguration,
97-
'elasticDumpPath' => $this->elasticDumpPath,
98-
]);
99-
100-
$script = $view->render();
101-
passthru($script);
102-
} catch (\Neos\FluidAdaptor\Exception $exception) {
103-
$this->consoleOutput->output('<error>%s</error>', [$exception->getMessage()]);
104-
}
105+
$view = new StandaloneView();
106+
$view->setTemplatePathAndFilename('resource://PunktDe.Elastic.Sync/Private/Template/CopyElastic.sh.template');
107+
$view->assignMultiple([
108+
'localConfiguration' => $localConfiguration,
109+
'remoteConfiguration' => $remoteConfiguration,
110+
'remoteInstance' => $remoteInstanceConfiguration,
111+
'indexConfigurations' => $indexConfigurations,
112+
'elasticDumpPath' => $this->elasticDumpPath,
113+
]);
114+
115+
$script = $view->render();
116+
passthru($script);
105117
}
106118

107119
/**
@@ -110,7 +122,7 @@ private function compileAndRunCloneScript(PresetConfiguration $remoteConfigurati
110122
* @throws \Neos\Flow\Http\Client\CurlEngineException
111123
* @throws \Neos\Flow\Http\Exception
112124
*/
113-
private function createAliases(PresetConfiguration $localConfiguration): void
125+
private function createAdditionalAliases(PresetConfiguration $localConfiguration): void
114126
{
115127
$definedAliases = $localConfiguration->getPostCloneConfiguration('createAliases');
116128

@@ -125,4 +137,23 @@ private function createAliases(PresetConfiguration $localConfiguration): void
125137
$this->consoleOutput->outputLine('%s -> %s', [$alias, $index]);
126138
}
127139
}
140+
141+
/**
142+
* @param PresetConfiguration $remoteConfiguration
143+
* @param string $indexTarget
144+
* @return array
145+
* @throws SynchronizationException
146+
* @throws \Neos\Flow\Http\Client\CurlEngineException
147+
* @throws \Neos\Flow\Http\Exception
148+
*/
149+
private function checkAndExpandRemoteIndices(PresetConfiguration $remoteConfiguration, string $indexTarget): array
150+
{
151+
$indices = $this->elasticSeacrhService->getIndices($remoteConfiguration, $indexTarget);
152+
153+
if (empty($indices)) {
154+
throw new SynchronizationException(sprintf('No index was found with the pattern "%s" on the remote server.', $indices), 1602999099);
155+
}
156+
157+
return array_column($indices, 'index');
158+
}
128159
}

‎Configuration/Settings.yaml

+5-5
Original file line numberDiff line numberDiff line change
@@ -23,8 +23,8 @@ PunktDe:
2323
port: '%env:ELASTICSEARCH_PORT%'
2424
indices:
2525
contentRepository:
26-
indexName: 'neos'
27-
postClone:
28-
createAliases:
29-
# After cloning the remote index, create the following aliases
30-
neos-default: neos
26+
indexName: 'neos*'
27+
# postClone:
28+
# createAliases:
29+
# # After cloning the remote index, create the following aliases
30+
# neos-default: neos

‎Readme.md

+17-4
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ The package uses [elasticsearch-dump](https://github.com/taskrabbit/elasticsearc
99
How the package works:
1010

1111
1. It gathers the Elasticsearch sync configuration from the remote server
12-
2. Establishs a ssh tunnel to the remote server and syncs the index mapping and data through it
12+
2. Establishs a ssh tunnel to the remote server and syncs the index mapping, data and aliases through it
1313

1414
## Installation
1515

@@ -25,10 +25,23 @@ Install the required JavaScript library:
2525

2626
You can add several presets. A preset consists of three parts
2727

28-
* **remoteInstance** Configures how the remote server and the remote installation can be reached.
29-
* **elasticsearch** Describes how the Eleasticsearch server instance can be reached. For the remote instance, the config is fetched from there.
30-
* **indices** Several indices to be fetched can be defined
28+
**remoteInstance**
29+
30+
Configures how the remote server and the remote installation can be reached.
3131

32+
**elasticsearch**
33+
34+
Describes how the Elasticsearch server instance can be reached. For the remote instance, the config is fetched from there.
35+
36+
**indices**
37+
38+
Several indices to be fetched can be defined. The index name can contain '*' to define a group of indices:
39+
40+
Example for cloning the content repository indices for all content dimensions, including all aliases:
41+
42+
indices:
43+
contentRepository:
44+
indexName: 'neos*'
3245

3346
## Usage
3447

‎Resources/Private/Partials/OpenSshTunnel.include

+3
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
ssh {remoteInstance.sshOptions->f:format.raw()} -C -N -L 127.0.0.1:9210:{remoteConfiguration.elasticsearchHost}:{remoteConfiguration.elasticsearchPort} {remoteInstance.user}@{remoteInstance.host} &
2+
sshpid=$!
3+
trap "kill $sshpid" EXIT

‎Resources/Private/Template/CopyElastic.sh.template

+27-17
Original file line numberDiff line numberDiff line change
@@ -1,36 +1,46 @@
11
#!/usr/bin/env bash
22
export NODE_TLS_REJECT_UNAUTHORIZED=0
33

4-
ssh {remoteInstance.sshOptions->f:format.raw()} -C -N -L 127.0.0.1:9210:{remoteConfiguration.elasticsearchHost}:{remoteConfiguration.elasticsearchPort} {remoteInstance.user}@{remoteInstance.host} &
5-
sshpid=$!
6-
trap "kill $sshpid" EXIT
7-
8-
<f:for each="{indices}" as="index">
9-
echo "-> Copying Index Settings (Errors can be ignored)"
4+
<f:for each="{indexConfigurations}" as="indexConfiguration" key="configurationKey">
5+
echo ""
6+
echo -e "\e[1m# Working on index configuration {configurationKey}"
7+
<f:for each="{indexConfiguration}" as="index">
8+
echo ""
9+
echo -e "\e[1m## Cloning index {index}"
10+
echo -e "\e[0m-> Copying Index Settings (Errors can be ignored)"
1011
{elasticDumpPath} \
11-
--input=http://127.0.0.1:9210/{index.remote.indexName} \
12-
--output={localConfiguration.elasticsearchScheme}://{localConfiguration.elasticsearchHost}:{localConfiguration.elasticsearchPort}/{index.local.indexName} \
12+
--input=http://127.0.0.1:9210/{index} \
13+
--output={localConfiguration.elasticsearchScheme}://{localConfiguration.elasticsearchHost}:{localConfiguration.elasticsearchPort}/{index} \
1314
--type=settings
1415

1516
echo ""
16-
echo "-> Copying Index Template"
17+
echo -e "\e[4m-> Copying Index Template\e[0m"
1718
{elasticDumpPath} \
18-
--input=http://127.0.0.1:9210/{index.remote.indexName} \
19-
--output={localConfiguration.elasticsearchScheme}://{localConfiguration.elasticsearchHost}:{localConfiguration.elasticsearchPort}/{index.local.indexName} \
19+
--input=http://127.0.0.1:9210/{index} \
20+
--output={localConfiguration.elasticsearchScheme}://{localConfiguration.elasticsearchHost}:{localConfiguration.elasticsearchPort}/{index} \
2021
--type=template
2122

2223
echo ""
23-
echo "-> Copying Index Mapping"
24+
echo -e "\e[4m-> Copying Index Mapping\e[0m"
2425
{elasticDumpPath} \
25-
--input=http://127.0.0.1:9210/{index.remote.indexName} \
26-
--output={localConfiguration.elasticsearchScheme}://{localConfiguration.elasticsearchHost}:{localConfiguration.elasticsearchPort}/{index.local.indexName} \
26+
--input=http://127.0.0.1:9210/{index} \
27+
--output={localConfiguration.elasticsearchScheme}://{localConfiguration.elasticsearchHost}:{localConfiguration.elasticsearchPort}/{index} \
2728
--type=mapping
2829

2930
echo ""
30-
echo "-> Copying Index Data"
31+
echo -e "\e[4m-> Copying Index Data\e[0m"
3132
{elasticDumpPath} \
3233
--limit=5000 \
33-
--input=http://127.0.0.1:9210/{index.remote.indexName} \
34-
--output={localConfiguration.elasticsearchScheme}://{localConfiguration.elasticsearchHost}:{localConfiguration.elasticsearchPort}/{index.local.indexName} \
34+
--input=http://127.0.0.1:9210/{index} \
35+
--output={localConfiguration.elasticsearchScheme}://{localConfiguration.elasticsearchHost}:{localConfiguration.elasticsearchPort}/{index} \
3536
--type=data
37+
38+
echo ""
39+
echo -e "\e[4m-> Copying Aliases\e[0m"
40+
{elasticDumpPath} \
41+
--limit=5000 \
42+
--input=http://127.0.0.1:9210/{index} \
43+
--output={localConfiguration.elasticsearchScheme}://{localConfiguration.elasticsearchHost}:{localConfiguration.elasticsearchPort} \
44+
--type=alias
45+
</f:for>
3646
</f:for>

0 commit comments

Comments
 (0)