Skip to content

Commit b2d72b9

Browse files
sgiehlsgiehl
committed
Handle invalidation of currently processed archives correctly
1 parent c893b54 commit b2d72b9

File tree

4 files changed

+141
-9
lines changed

4 files changed

+141
-9
lines changed

‎core/DataAccess/ArchiveTableDao.php

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -55,15 +55,15 @@ public function getArchiveTableAnalysis($tableDate)
5555
SUM(CASE WHEN name LIKE 'done%' THEN 1 ELSE 0 END) AS count_archives,
5656
SUM(CASE WHEN name LIKE 'done%' AND value = ? THEN 1 ELSE 0 END) AS count_invalidated_archives,
5757
SUM(CASE WHEN name LIKE 'done%' AND value = ? THEN 1 ELSE 0 END) AS count_temporary_archives,
58-
SUM(CASE WHEN name LIKE 'done%' AND value = ? THEN 1 ELSE 0 END) AS count_error_archives,
58+
SUM(CASE WHEN name LIKE 'done%' AND value IN (?, ?) THEN 1 ELSE 0 END) AS count_error_archives,
5959
SUM(CASE WHEN name LIKE 'done%' AND CHAR_LENGTH(name) > 32 THEN 1 ELSE 0 END) AS count_segment_archives,
6060
SUM(CASE WHEN name NOT LIKE 'done%' THEN 1 ELSE 0 END) AS count_numeric_rows,
6161
0 AS count_blob_rows
6262
FROM `$numericTable`
6363
GROUP BY idsite, date1, date2, period";
6464

6565
$rows = Db::fetchAll($sql, array(ArchiveWriter::DONE_INVALIDATED, ArchiveWriter::DONE_OK_TEMPORARY,
66-
ArchiveWriter::DONE_ERROR));
66+
ArchiveWriter::DONE_ERROR, ArchiveWriter::DONE_ERROR_INVALIDATED));
6767

6868
// index result
6969
$result = array();

‎core/DataAccess/ArchiveWriter.php

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,11 @@ class ArchiveWriter
6767
*/
6868
public const DONE_PARTIAL = 5;
6969

70+
/**
71+
* Flag indicates an archive that is currently been processed, but has already been invalidated again
72+
*/
73+
public const DONE_ERROR_INVALIDATED = 6;
74+
7075
protected $fields = ['idarchive',
7176
'idsite',
7277
'date1',
@@ -200,6 +205,13 @@ public function finalizeArchive()
200205
$doneValue = $this->parameters->isPartialArchive() ? self::DONE_PARTIAL : self::DONE_OK;
201206
$this->checkDoneValueIsOnlyPartialForPluginArchives($doneValue); // check and log
202207

208+
$currentStatus = $this->getModel()->getArchiveStatus($numericTable, $idArchive, $this->doneFlag);
209+
210+
// If the current archive was already invalidated during runtime, directly update status to invalidated instead of done
211+
if ($currentStatus == self::DONE_ERROR_INVALIDATED) {
212+
$doneValue = self::DONE_INVALIDATED;
213+
}
214+
203215
$this->getModel()->updateArchiveStatus($numericTable, $idArchive, $this->doneFlag, $doneValue);
204216

205217
if (

‎core/DataAccess/Model.php

Lines changed: 18 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -65,7 +65,7 @@ public function getInvalidatedArchiveIdsSafeToDelete($archiveTable, $setGroupCon
6565
GROUP_CONCAT(idarchive, '.', value ORDER BY ts_archived DESC) as archives
6666
FROM `$archiveTable`
6767
WHERE name LIKE 'done%'
68-
AND `value` NOT IN (" . ArchiveWriter::DONE_ERROR . ")
68+
AND `value` NOT IN (" . ArchiveWriter::DONE_ERROR . ", " . ArchiveWriter::DONE_ERROR_INVALIDATED . ")
6969
GROUP BY idsite, date1, date2, period, name HAVING count(*) > 1";
7070

7171
$archiveIds = array();
@@ -184,8 +184,15 @@ public function updateArchiveAsInvalidated(
184184
if (!empty($idArchives)) {
185185
$idArchives = array_map('intval', $idArchives);
186186

187+
// set status to DONE_INVALIDATED for finished archives
187188
$sql = "UPDATE `$archiveTable` SET `value` = " . ArchiveWriter::DONE_INVALIDATED . " WHERE idarchive IN ("
188-
. implode(',', $idArchives) . ") AND $nameCondition";
189+
. implode(',', $idArchives) . ") AND value != " . ArchiveWriter::DONE_ERROR . " AND $nameCondition";
190+
191+
Db::query($sql);
192+
193+
// set status to DONE_ERROR_INVALIDATED for currently processed archives
194+
$sql = "UPDATE `$archiveTable` SET `value` = " . ArchiveWriter::DONE_ERROR_INVALIDATED . " WHERE idarchive IN ("
195+
. implode(',', $idArchives) . ") AND value = " . ArchiveWriter::DONE_ERROR . " AND $nameCondition";
189196

190197
Db::query($sql);
191198
}
@@ -359,7 +366,7 @@ public function getTemporaryArchivesOlderThan($archiveTable, $purgeArchivesOlder
359366
WHERE name LIKE 'done%'
360367
AND (( value = " . ArchiveWriter::DONE_OK_TEMPORARY . "
361368
AND ts_archived < ?)
362-
OR value = " . ArchiveWriter::DONE_ERROR . ")";
369+
OR value IN (" . ArchiveWriter::DONE_ERROR . ", " . ArchiveWriter::DONE_ERROR_INVALIDATED . "))";
363370

364371
return Db::fetchAll($query, array($purgeArchivesOlderThan));
365372
}
@@ -545,6 +552,14 @@ public function updateArchiveStatus($numericTable, $archiveId, $doneFlag, $value
545552
);
546553
}
547554

555+
public function getArchiveStatus($numericTable, $archiveId, $doneFlag)
556+
{
557+
return Db::fetchOne(
558+
"SELECT value FROM $numericTable WHERE idarchive = ? AND `name` = ?",
559+
array($archiveId, $doneFlag)
560+
);
561+
}
562+
548563
public function insertRecord($tableName, $fields, $record, $name, $value)
549564
{
550565
// duplicate idarchives are Ignored, see https://github.com/piwik/piwik/issues/987

‎tests/PHPUnit/Integration/DataAccess/ArchiveInvalidatorTest.php

Lines changed: 109 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -193,6 +193,91 @@ public function testMarkArchivesAsInvalidatedDoesNotInvalidatePartialArchives()
193193
$actualInvalidations = $this->getInvalidatedArchiveTableEntries();
194194
$this->assertEquals($expectedInvalidations, $actualInvalidations);
195195
}
196+
public function testMarkArchivesAsInvalidatedDoesHandleInProgressArchivesCorrectly()
197+
{
198+
// Insert an archive/invalidation that is currently in progress
199+
$this->insertArchiveRow(1, '2020-03-03', 'day', $doneValue = ArchiveWriter::DONE_ERROR, '', false);
200+
$this->insertInvalidations([
201+
['name' => 'done', 'idsite' => 1, 'date1' => '2020-03-03', 'date2' => '2020-03-03', 'period' => 1, 'report' => null, 'ts_started' => Date::now()->getDatetime(), 'status' => 1]
202+
]);
203+
204+
/** @var ArchiveInvalidator $archiveInvalidator */
205+
$archiveInvalidator = self::$fixture->piwikEnvironment->getContainer()->get('Piwik\Archive\ArchiveInvalidator');
206+
207+
$archiveInvalidator->markArchivesAsInvalidated(
208+
[1],
209+
['2020-03-03'],
210+
'day',
211+
null,
212+
$cascadeDown = true,
213+
false
214+
);
215+
216+
$invalidatedArchives = $this->getAvailableArchives();
217+
$expectedArchives = [
218+
'2020_03' => [
219+
['idsite' => 1, 'date1' => '2020-03-03', 'date2' => '2020-03-03', 'period' => 1, 'name' => 'done', 'value' => ArchiveWriter::DONE_ERROR_INVALIDATED]
220+
],
221+
];
222+
223+
$this->assertEquals($expectedArchives, $invalidatedArchives);
224+
225+
$expectedInvalidations = [
226+
[
227+
'idarchive' => null,
228+
'idsite' => '1',
229+
'period' => '1',
230+
'name' => 'done',
231+
'date1' => '2020-03-03',
232+
'date2' => '2020-03-03',
233+
'report' => null,
234+
'status' => '1'
235+
],
236+
[
237+
'idarchive' => '1',
238+
'idsite' => '1',
239+
'period' => '1',
240+
'name' => 'done',
241+
'date1' => '2020-03-03',
242+
'date2' => '2020-03-03',
243+
'report' => null,
244+
'status' => '0'
245+
],
246+
[
247+
'idarchive' => null,
248+
'idsite' => '1',
249+
'period' => '2',
250+
'name' => 'done',
251+
'date1' => '2020-03-02',
252+
'date2' => '2020-03-08',
253+
'report' => null,
254+
'status' => '0'
255+
],
256+
[
257+
'idarchive' => null,
258+
'idsite' => '1',
259+
'period' => '3',
260+
'name' => 'done',
261+
'date1' => '2020-03-01',
262+
'date2' => '2020-03-31',
263+
'report' => null,
264+
'status' => '0'
265+
],
266+
[
267+
'idarchive' => null,
268+
'idsite' => '1',
269+
'period' => '4',
270+
'name' => 'done',
271+
'date1' => '2020-01-01',
272+
'date2' => '2020-12-31',
273+
'report' => null,
274+
'status' => '0'
275+
],
276+
];
277+
278+
$actualInvalidations = $this->getInvalidatedArchiveTableEntries(true);
279+
$this->assertEquals($expectedInvalidations, $actualInvalidations);
280+
}
196281

197282
public function testReArchiveReportDoesNothingIfIniSettingSetToZero()
198283
{
@@ -1203,7 +1288,6 @@ public function getTestDataForMarkArchivesAsInvalidated()
12031288
),
12041289
),
12051290
[
1206-
// TODO: super strange, there are two idarchive = 106 values here
12071291
['idarchive' => '106', 'idsite' => '1', 'date1' => '2015-01-01', 'date2' => '2015-01-31', 'period' => '3', 'name' => 'done3736b708e4d20cfc10610e816a1b2341', 'report' => null],
12081292
['idarchive' => '1', 'idsite' => '1', 'date1' => '2015-01-01', 'date2' => '2015-01-01', 'period' => '1', 'name' => 'done3736b708e4d20cfc10610e816a1b2341', 'report' => null],
12091293
['idarchive' => null, 'idsite' => '1', 'date1' => '2015-01-01', 'date2' => '2015-12-31', 'period' => '4', 'name' => 'done3736b708e4d20cfc10610e816a1b2341', 'report' => null],
@@ -1240,6 +1324,7 @@ public function getTestDataForMarkArchivesAsInvalidated()
12401324
['idarchive' => null, 'idsite' => '1', 'date1' => '2015-01-28', 'date2' => '2015-01-28', 'period' => '1', 'name' => 'done3736b708e4d20cfc10610e816a1b2341', 'report' => null],
12411325
['idarchive' => null, 'idsite' => '1', 'date1' => '2015-01-29', 'date2' => '2015-01-29', 'period' => '1', 'name' => 'done3736b708e4d20cfc10610e816a1b2341', 'report' => null],
12421326
['idarchive' => null, 'idsite' => '1', 'date1' => '2015-01-30', 'date2' => '2015-01-30', 'period' => '1', 'name' => 'done3736b708e4d20cfc10610e816a1b2341', 'report' => null],
1327+
// archive id 106 occurs a second time as it comes from a different archive table
12431328
['idarchive' => '106', 'idsite' => '1', 'date1' => '2014-12-29', 'date2' => '2015-01-04', 'period' => '2', 'name' => 'done3736b708e4d20cfc10610e816a1b2341', 'report' => null],
12441329
['idarchive' => '91', 'idsite' => '1', 'date1' => '2015-01-31', 'date2' => '2015-01-31', 'period' => '1', 'name' => 'done3736b708e4d20cfc10610e816a1b2341', 'report' => null],
12451330
],
@@ -2445,6 +2530,24 @@ private function getInvalidatedIdArchives()
24452530
return $result;
24462531
}
24472532

2533+
private function getAvailableArchives()
2534+
{
2535+
$result = array();
2536+
foreach (ArchiveTableCreator::getTablesArchivesInstalled(ArchiveTableCreator::NUMERIC_TABLE) as $table) {
2537+
$date = ArchiveTableCreator::getDateFromTableName($table);
2538+
2539+
$sql = "SELECT idsite, date1, date2, period, name, value FROM $table WHERE name LIKE 'done%'";
2540+
2541+
$archiveSpecs = Db::query($sql)->fetchAll();
2542+
if (empty($archiveSpecs)) {
2543+
continue;
2544+
}
2545+
2546+
$result[$date] = $archiveSpecs;
2547+
}
2548+
return $result;
2549+
}
2550+
24482551
private function getInvalidatedArchives($anyTsArchived = true)
24492552
{
24502553
$result = array();
@@ -2529,9 +2632,9 @@ private function insertArchiveRow($idSite, $date, $periodLabel, $doneValue = Arc
25292632
Db::query($sql);
25302633
}
25312634

2532-
private function getInvalidatedArchiveTableEntries()
2635+
private function getInvalidatedArchiveTableEntries($includeStatus = false)
25332636
{
2534-
return Db::fetchAll("SELECT idarchive, idsite, date1, date2, period, name, report FROM " . Common::prefixTable('archive_invalidations'));
2637+
return Db::fetchAll("SELECT idarchive, idsite, date1, date2, period, name, report" . ($includeStatus ? ', status' : '') . " FROM " . Common::prefixTable('archive_invalidations'));
25352638
}
25362639

25372640
private function assertEqualsSorted(array $expectedEntries, array $invalidatedArchiveTableEntries)
@@ -2579,14 +2682,16 @@ private function insertInvalidations(array $invalidations)
25792682
{
25802683
$table = Common::prefixTable('archive_invalidations');
25812684
foreach ($invalidations as $invalidation) {
2582-
$sql = "INSERT INTO $table (name, idsite, date1, date2, period, ts_invalidated, report) VALUES (?, ?, ?, ?, ?, NOW(), ?)";
2685+
$sql = "INSERT INTO $table (name, idsite, date1, date2, period, ts_invalidated, report, ts_started, status) VALUES (?, ?, ?, ?, ?, NOW(), ?, ?, ?)";
25832686
Db::query($sql, [
25842687
$invalidation['name'],
25852688
$invalidation['idsite'],
25862689
$invalidation['date1'],
25872690
$invalidation['date2'],
25882691
$invalidation['period'],
25892692
$invalidation['report'],
2693+
$invalidation['ts_started'] ?? null,
2694+
$invalidation['status'] ?? 0,
25902695
]);
25912696
}
25922697
}

0 commit comments

Comments
 (0)