Skip to content

Commit 7752a23

Browse files
romanetarromanetar
committed
logs upload command
Signed-off-by: romanetar <[email protected]>
1 parent 652ead5 commit 7752a23

File tree

7 files changed

+143
-42
lines changed

7 files changed

+143
-42
lines changed

‎.env.example

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ APP_SCOPE_BASE_REALM=http://localhost
77
APP_OAUTH_2_0_CLIENT_ID=clientid
88
APP_OAUTH_2_0_CLIENT_SECRET=clientsecret
99
APP_OAUTH_2_0_AUTH_SERVER_BASE_URL=http://localhost
10+
APP_NODE_NBR=
1011

1112
DB_HOST=localhost
1213
DB_DATABASE=homestead
@@ -185,4 +186,13 @@ AWS_ENDPOINT=
185186
REGISTRATION_ORDER_PUBLIC_EDIT_TTL=10
186187
DEFAULT_PROFILE_IMAGE=
187188
SCHEDULE_USE_REALTIME_UPDATE=1
188-
SAMSUNG_REGISTRATION_API_ENDPOINT=
189+
SAMSUNG_REGISTRATION_API_ENDPOINT=
190+
191+
# Remote logs storage
192+
REMOTE_LOGS_STORAGE_NAME='logs_s3'
193+
194+
AWS_ACCESS_KEY_ID_LOGS=
195+
AWS_SECRET_ACCESS_KEY_LOGS=
196+
AWS_DEFAULT_REGION_LOGS=
197+
AWS_ENDPOINT_LOGS=
198+
AWS_BUCKET_LOGS=

‎app/Console/Commands/LogsUploader.php

Lines changed: 15 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -12,12 +12,8 @@
1212
* limitations under the License.
1313
**/
1414

15-
use Carbon\Carbon;
16-
use Carbon\CarbonPeriod;
17-
use Exception;
15+
use App\Services\FileSystem\ILogsUploadService;
1816
use Illuminate\Console\Command;
19-
use Illuminate\Support\Facades\Log;
20-
use Illuminate\Support\Facades\Storage;
2117

2218
/**
2319
* Class LogsUploader
@@ -47,50 +43,28 @@ final class LogsUploader extends Command
4743
*/
4844
protected $description = 'Upload local logs to external storage';
4945

50-
private function formatFileName(string $date_str) {
51-
return "laravel-{$date_str}.log";
46+
/**
47+
* @var ILogsUploadService
48+
*/
49+
protected $logs_upload_service;
50+
51+
/**
52+
* SummitEventSetAvgRateProcessor constructor.
53+
* @param ILogsUploadService $logs_upload_service
54+
*/
55+
public function __construct(ILogsUploadService $logs_upload_service)
56+
{
57+
parent::__construct();
58+
$this->logs_upload_service = $logs_upload_service;
5259
}
5360

5461
/**
5562
* Execute the console command.
5663
*
5764
* @return mixed
58-
* @throws \Illuminate\Contracts\Filesystem\FileNotFoundException
5965
*/
6066
public function handle()
6167
{
62-
try {
63-
$date_format = 'Y-m-d';
64-
$ui_file_name = 'uploads_info';
65-
$pending_uploads = [];
66-
$yesterday = Carbon::yesterday('UTC');
67-
68-
$logs_fs = Storage::disk('logs');
69-
if ($logs_fs->exists($ui_file_name)) {
70-
$uploads_info = explode(PHP_EOL, $logs_fs->get($ui_file_name));
71-
sort($uploads_info);
72-
$first_date_str = $uploads_info[0];
73-
$date_from = Carbon::createFromFormat($date_format, $first_date_str);
74-
//get upload gaps
75-
$period = CarbonPeriod::create($date_from, $yesterday);
76-
77-
foreach ($period as $date) {
78-
$date_str = $date->format($date_format);
79-
if (!in_array($date_str, $uploads_info)) {
80-
$pending_uploads[] = $this->formatFileName($date_str);
81-
}
82-
}
83-
} else {
84-
$pending_uploads[] = $this->formatFileName($yesterday->format($date_format));
85-
}
86-
87-
foreach ($pending_uploads as $pending_upload) {
88-
$logs_fs->append($ui_file_name, $pending_upload);
89-
}
90-
91-
//Storage::disk('logs_s3')->put($log_name, $content);
92-
} catch (Exception $ex) {
93-
Log::error($ex);
94-
}
68+
$this->logs_upload_service->startUpload();
9569
}
9670
}

‎app/Services/BaseServicesProvider.php

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,8 @@
2323
use App\Services\Apis\PasswordlessAPI;
2424
use App\Services\Apis\Samsung\ISamsungRegistrationAPI;
2525
use App\Services\Apis\Samsung\SamsungRegistrationAPI;
26+
use App\Services\FileSystem\ILogsUploadService;
27+
use App\Services\FileSystem\LogsUploadService;
2628
use App\Services\Model\FolderService;
2729
use App\Services\Model\IFolderService;
2830
use App\Services\utils\EmailExcerptService;
@@ -148,6 +150,11 @@ function(){
148150
);
149151
}
150152
);
153+
154+
App::singleton(
155+
ILogsUploadService::class,
156+
LogsUploadService::class
157+
);
151158
}
152159

153160
/**
@@ -172,6 +179,7 @@ public function provides()
172179
ILockManagerService::class,
173180
IPasswordlessAPI::class,
174181
ISamsungRegistrationAPI::class,
182+
ILogsUploadService::class,
175183
];
176184
}
177185
}

‎app/Services/FileSystem/ILogsUploadService.php

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
<?php namespace App\Services\FileSystem;
2+
/**
3+
* Copyright 2023 OpenStack Foundation
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
* http://www.apache.org/licenses/LICENSE-2.0
8+
* Unless required by applicable law or agreed to in writing, software
9+
* distributed under the License is distributed on an "AS IS" BASIS,
10+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11+
* See the License for the specific language governing permissions and
12+
* limitations under the License.
13+
**/
14+
15+
/**
16+
* Interface ILogsUploadService
17+
* @package App\Services\FileSystem
18+
*/
19+
interface ILogsUploadService
20+
{
21+
/**
22+
* @return void
23+
*/
24+
public function startUpload(): void;
25+
}

‎app/Services/FileSystem/LogsUploadService.php

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
<?php namespace App\Services\FileSystem;
2+
3+
/**
4+
* Copyright 2023 OpenStack Foundation
5+
* Licensed under the Apache License, Version 2.0 (the "License");
6+
* you may not use this file except in compliance with the License.
7+
* You may obtain a copy of the License at
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
* Unless required by applicable law or agreed to in writing, software
10+
* distributed under the License is distributed on an "AS IS" BASIS,
11+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
* See the License for the specific language governing permissions and
13+
* limitations under the License.
14+
**/
15+
use Carbon\Carbon;
16+
use Exception;
17+
use Illuminate\Support\Facades\Config;
18+
use Illuminate\Support\Facades\Log;
19+
use Illuminate\Support\Facades\Storage;
20+
/**
21+
* Class LogsUploadService
22+
* @package App\Services\Model
23+
*/
24+
final class LogsUploadService
25+
implements ILogsUploadService
26+
{
27+
/**
28+
* Returns the log file name based on the date parameter.
29+
*/
30+
private function formatFileName(Carbon $date): string
31+
{
32+
$date_format = 'Y-m-d';
33+
return "laravel-{$date->format($date_format)}.log";
34+
}
35+
36+
/**
37+
* Returns the remote log file path based on the local name and the node number.
38+
*/
39+
private function formatRemoteLogFilePath(string $local_name, string $node_number): string
40+
{
41+
return "{$node_number}/{$local_name}.gz";
42+
}
43+
44+
/**
45+
* @inheritdoc
46+
*/
47+
public function startUpload(): void
48+
{
49+
try {
50+
$node_number = Config::get("server.app_node_number");
51+
$remote_storage_name = Config::get("log.remote_storage_name");
52+
53+
$today_log_file_name = $this->formatFileName(Carbon::today('UTC'));
54+
55+
$logs_fs = Storage::disk('logs');
56+
$remote_logs_fs = Storage::disk($remote_storage_name);
57+
58+
$local_log_files = array_filter($logs_fs->allFiles(), function ($file_name) use ($today_log_file_name) {
59+
return $file_name != $today_log_file_name && str_ends_with($file_name, '.log');
60+
});
61+
sort($local_log_files);
62+
63+
foreach ($local_log_files as $local_log_file) {
64+
$content = $logs_fs->get($local_log_file);
65+
if (empty($content)) continue;
66+
67+
$remote_file_path = $this->formatRemoteLogFilePath($local_log_file, $node_number);
68+
if ($remote_logs_fs->exists($remote_file_path)) continue;
69+
70+
$deflateContext = deflate_init(ZLIB_ENCODING_GZIP);
71+
$compressed = deflate_add($deflateContext, $content, ZLIB_FINISH);
72+
73+
$remote_logs_fs->put($remote_file_path, $compressed);
74+
75+
$logs_fs->put("{$local_log_file}.gz", $compressed);
76+
$logs_fs->delete($local_log_file);
77+
}
78+
} catch (Exception $ex) {
79+
Log::error($ex);
80+
}
81+
}
82+
}

‎config/log.php

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,4 +11,5 @@
1111
'level' => env('LOG_LEVEL', 'error'),
1212
'email_level' => env('LOG_EMAIL_LEVEL', 'error'),
1313
'email_subject' => env('LOG_EMAIL_SUBJECT', ''),
14+
'remote_storage_name' => env('REMOTE_LOGS_STORAGE_NAME', 'logs_s3'),
1415
];

‎config/server.php

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,4 +26,5 @@
2626
'ss_encrypt_cypher' => env('SS_ENCRYPT_CYPHER', ''),
2727
'google_geocoding_api_key' => env('GOOGLE_GEO_CODING_API_KEY', ''),
2828
'samsung_registration_api_endpoint' => env('SAMSUNG_REGISTRATION_API_ENDPOINT', ''),
29+
'app_node_number' => env('APP_NODE_NBR', 1),
2930
);

0 commit comments

Comments
 (0)