diff --git a/lib/models/ObjectMD.ts b/lib/models/ObjectMD.ts index c19cefa23..45f72cc02 100644 --- a/lib/models/ObjectMD.ts +++ b/lib/models/ObjectMD.ts @@ -1,4 +1,3 @@ -import * as crypto from 'crypto'; import * as constants from '../constants'; import * as VersionIDUtils from '../versioning/VersionID'; import { VersioningConstants } from '../versioning/constants'; @@ -49,6 +48,7 @@ export type ReplicationInfo = { /** @deprecated in favor of per-backend dataStoreVersionId for multi-destination. */ dataStoreVersionId?: string; isNFS?: boolean; + isReplica?: boolean; }; export type ObjectMDTag = { @@ -260,6 +260,7 @@ export default class ObjectMD { storageType: undefined, dataStoreVersionId: undefined, isNFS: undefined, + isReplica: undefined, }, dataStoreName: '', originOp: '', @@ -1168,6 +1169,26 @@ export default class ObjectMD { return this; } + /** + * Mark this object as the result of a replication write (replica), + * as opposed to a write originating from a user request. + * + * @param isReplica - true if this object was written by replication + * @return itself + */ + setReplicationIsReplica(isReplica: boolean) { + this._data.replicationInfo.isReplica = isReplica; + return this; + } + + /** + * Get whether this object was written by replication (replica). + * @return true if this object is a replica + */ + getReplicationIsReplica(): boolean { + return this._data.replicationInfo.isReplica === true || this._data.replicationInfo.status === 'REPLICA'; + } + getReplicationSiteStatus(key: BackendKey): string | undefined { return this._findBackend(key)?.status; } @@ -1348,35 +1369,28 @@ export default class ObjectMD { } /** - * Create or update the microVersionId field - * - * This field can be used to force an update in MongoDB. This can - * be needed in the following cases: + * Update the microVersionId * - * - in case no other metadata field changes - * - * - to detect a change when fields change but object version does - * not change e.g. when ingesting a putObjectTagging coming from - * S3C to Zenko - * - * - to manage conflicts during concurrent updates, using - * conditions on the microVersionId field. - * - * It's a field of 16 hexadecimal characters randomly generated + * This field can be used to force an update in MongoDB when no other + * metadata field changes, to detect a change for CRR, + * and to manage conflicts during concurrent updates using conditions on this field. * + * @param instanceId - instance identifier (e.g. config.instanceId) + * @param replicationGroupId - replication group ID (e.g. config.replicationGroupId) * @return itself */ - updateMicroVersionId() { - this._data.microVersionId = crypto.randomBytes(8).toString('hex'); + updateMicroVersionId(instanceId: string, replicationGroupId: string) { + this._data.microVersionId = VersionIDUtils.generateVersionId(instanceId, replicationGroupId); + return this; } /** - * Get the microVersionId field, or null if not set + * Get the microVersionId field, or undefined if not set * - * @return the microVersionId field if exists, or {null} if it does not exist + * @return the microVersionId field if set, or undefined if not */ getMicroVersionId() { - return this._data.microVersionId || null; + return this._data.microVersionId || undefined; } /** diff --git a/lib/versioning/VersionID.ts b/lib/versioning/VersionID.ts index f18e3159c..438d58c9a 100644 --- a/lib/versioning/VersionID.ts +++ b/lib/versioning/VersionID.ts @@ -53,8 +53,7 @@ export const S3_VERSION_ID_ENCODING_TYPE = process.env.S3_VERSION_ID_ENCODING_TY // - Uses old format: timestamp + sequential_position + rep_group_id (legacy 27-char format) // Falls back to hex encoding if S3_VERSION_ID_ENCODING_TYPE is 'hex' or unset export const ENABLE_FORMATTED_VERSION_ID = - process.env.ENABLE_FORMATTED_VERSION_ID === 'true' || - process.env.ENABLE_FORMATTED_VERSION_ID === '1'; + process.env.ENABLE_FORMATTED_VERSION_ID === 'true' || process.env.ENABLE_FORMATTED_VERSION_ID === '1'; // version ID format added to the end of the version ID const VERSION_ID_FORMAT_VERSION = '1'; @@ -101,11 +100,7 @@ const MAX_SEQ = Math.pow(10, LENGTH_SEQ) - 1; // good for 1 billion ops */ export function getInfVid(replicationGroupId: string) { const repGroupId = padRight(replicationGroupId, TEMPLATE_RG); - return ( - padLeft(MAX_TS, TEMPLATE_TS) + - padLeft(MAX_SEQ, TEMPLATE_SEQ) + - repGroupId - ); + return padLeft(MAX_TS, TEMPLATE_TS) + padLeft(MAX_SEQ, TEMPLATE_SEQ) + repGroupId; } // internal state of the module @@ -232,9 +227,7 @@ export function hexDecode(str: string): string | Error { */ const B62V_TOTAL = LENGTH_TS + LENGTH_SEQ; const B62V_HALF = B62V_TOTAL / 2; -const B62V_EPAD = '0'.repeat( - Math.ceil(B62V_HALF * (Math.log(10) / Math.log(62))) -); +const B62V_EPAD = '0'.repeat(Math.ceil(B62V_HALF * (Math.log(10) / Math.log(62)))); const B62V_DPAD = '0'.repeat(B62V_HALF); const B62V_STRING_EPAD = '0'.repeat(32 - 2 * B62V_EPAD.length); @@ -355,11 +348,14 @@ export function decode(str: string): string | Error { const decoded: string | Error = base62Decode(str); // Legacy base62 version IDs (without 'info' field) are always 27 characters long. // The new base62 format is always 35 characters long. - if (typeof decoded === 'string' && + if ( + typeof decoded === 'string' && decoded.length !== LEGACY_BASE62_DECODED_LENGTH && - decoded.length !== BASE62_DECODED_LENGTH) { - return new Error(`decoded ${str} is not length ` + - `${LEGACY_BASE62_DECODED_LENGTH} or ${BASE62_DECODED_LENGTH}`); + decoded.length !== BASE62_DECODED_LENGTH + ) { + return new Error( + `decoded ${str} is not length ` + `${LEGACY_BASE62_DECODED_LENGTH} or ${BASE62_DECODED_LENGTH}`, + ); } return decoded; } @@ -369,3 +365,33 @@ export function decode(str: string): string | Error { } return new Error(`cannot decode str ${str.length}`); } + +export enum Ordering { + OLDER = 'older', + YOUNGER = 'younger', + EQUAL = 'equal', + NOT_COMPARABLE = 'notComparable', +} + +/** + * Compare two raw (decoded) version IDs. + * + * Returns NOT_COMPARABLE when either value is absent or shorter than + * LEGACY_BASE62_DECODED_LENGTH (rejects the old 16-char random-hex format). + * Callers are responsible for deciding how to handle NOT_COMPARABLE. + */ +export function compare(v1: string | null | undefined, v2: string | null | undefined): Ordering { + if ( + typeof v1 !== 'string' || + v1.length < LEGACY_BASE62_DECODED_LENGTH || + typeof v2 !== 'string' || + v2.length < LEGACY_BASE62_DECODED_LENGTH + ) { + return Ordering.NOT_COMPARABLE; + } + if (v1 === v2) { + return Ordering.EQUAL; + } + + return v1 > v2 ? Ordering.OLDER : Ordering.YOUNGER; +} diff --git a/package.json b/package.json index b88ddc47b..235ce0be4 100644 --- a/package.json +++ b/package.json @@ -3,7 +3,7 @@ "engines": { "node": ">=20" }, - "version": "8.4.8", + "version": "8.4.9", "config": { "mongodbMemoryServer": { "version": "8.0.23" diff --git a/tests/unit/models/ObjectMD.spec.js b/tests/unit/models/ObjectMD.spec.js index 8e89186da..4d1420c10 100644 --- a/tests/unit/models/ObjectMD.spec.js +++ b/tests/unit/models/ObjectMD.spec.js @@ -112,6 +112,7 @@ describe('ObjectMD class setters/getters', () => { storageType: undefined, dataStoreVersionId: undefined, isNFS: undefined, + isReplica: undefined, }, ], [ @@ -132,6 +133,7 @@ describe('ObjectMD class setters/getters', () => { storageType: 'aws_s3', dataStoreVersionId: '', isNFS: undefined, + isReplica: undefined, }, ], ['DataStoreName', null, ''], @@ -183,6 +185,28 @@ describe('ObjectMD class setters/getters', () => { }); }); + it('getReplicationIsReplica: returns true when status is REPLICA', () => { + md.setReplicationStatus('REPLICA'); + assert.strictEqual( + md.getReplicationIsReplica(), + true, + 'status REPLICA alone can be used to determine isReplica', + ); + }); + + it('getReplicationIsReplica: survives status transition to PENDING (cascade)', () => { + md.setReplicationInfo({ + status: 'REPLICA', + isReplica: true, + backends: [], + content: [], + destination: '', + role: '', + }); + md.setReplicationStatus('PENDING'); + assert.strictEqual(md.getReplicationIsReplica(), true, 'isReplica must survive status changes'); + }); + it('ObjectMD::setReplicationSiteStatus', () => { md.setReplicationInfo({ backends: [ @@ -386,22 +410,22 @@ describe('ObjectMD class setters/getters', () => { }); it('ObjectMD::microVersionId unset', () => { - assert.strictEqual(md.getMicroVersionId(), null); + assert.strictEqual(md.getMicroVersionId(), undefined); }); it('ObjectMD::microVersionId set', () => { - const generatedIds = new Set(); + const generatedIds = []; for (let i = 0; i < 100; ++i) { - md.updateMicroVersionId(); - generatedIds.add(md.getMicroVersionId()); + md.updateMicroVersionId('instance', 'RG001'); + generatedIds.push(md.getMicroVersionId()); } // all generated IDs should be different - assert.strictEqual(generatedIds.size, 100); - generatedIds.forEach(key => { - // length is always 16 in hex because leading 0s are - // also encoded in the 8-byte random buffer. - assert.strictEqual(key.length, 16); - }); + assert.strictEqual(new Set(generatedIds).size, 100); + // microVersionIds use the versionId format (reversed time ordered): + // newer values sort before older ones lexicographically. + for (let i = 1; i < generatedIds.length; ++i) { + assert(generatedIds[i] < generatedIds[i - 1]); + } }); it('ObjectMD::set/getRetentionMode', () => { diff --git a/tests/unit/versioning/VersionID.spec.js b/tests/unit/versioning/VersionID.spec.js index e41b3b9e3..8472c9611 100644 --- a/tests/unit/versioning/VersionID.spec.js +++ b/tests/unit/versioning/VersionID.spec.js @@ -1,3 +1,4 @@ +const crypto = require('crypto'); const VID = require('../../../lib/versioning/VersionID'); const VersioningConstants = require('../../../lib/versioning/constants').VersioningConstants; const assert = require('assert'); @@ -8,7 +9,8 @@ function randkey(length) { // Generate ASCII characters from 32-125, excluding '?' (63) // as '?' is reserved for the version ID format marker let charCode = Math.floor(Math.random() * 94 + 32); - if (charCode === 63) { // Skip '?' character + if (charCode === 63) { + // Skip '?' character charCode = 126; // Use '~' instead } key += String.fromCharCode(charCode); @@ -46,8 +48,7 @@ describe('test generating versionIds', () => { it('sorted in reversed chronological and alphabetical order', () => { for (let i = 1; i < count; i++) { - assert(vids[i - 1] > vids[i], - 'previous VersionID is higher than its next'); + assert(vids[i - 1] > vids[i], 'previous VersionID is higher than its next'); } }); @@ -67,8 +68,14 @@ describe('test generating versionIds', () => { it('should encode and decode correctly with legacy format', () => { const encoded = vids.map(VID.encode); const decoded = encoded.map(VID.decode); - assert.strictEqual(vids.every(x => x.length > 27), true); - assert.strictEqual(encoded.every(x => x.length > 32), true); + assert.strictEqual( + vids.every(x => x.length > 27), + true, + ); + assert.strictEqual( + encoded.every(x => x.length > 32), + true, + ); assert.deepStrictEqual(vids, decoded); }); @@ -115,8 +122,7 @@ describe('test generating versionIds', () => { it('sorted in reversed chronological and alphabetical order', () => { for (let i = 1; i < count; i++) { - assert(vids[i - 1] > vids[i], - 'previous VersionID is higher than its next'); + assert(vids[i - 1] > vids[i], 'previous VersionID is higher than its next'); } }); @@ -188,3 +194,49 @@ describe('test generating versionIds', () => { }); }); }); + +describe('compare', () => { + const { compare, Ordering, generateVersionId } = VID; + + it('should return NOT_COMPARABLE when existing is null', () => { + assert.strictEqual(compare(generateVersionId('test', 'RG001'), null), Ordering.NOT_COMPARABLE); + }); + + it('should return NOT_COMPARABLE when existing is undefined', () => { + assert.strictEqual(compare(generateVersionId('test', 'RG001'), undefined), Ordering.NOT_COMPARABLE); + }); + + it('should return NOT_COMPARABLE when existing is a legacy 16-char hex microVersionId', () => { + const legacyHex = crypto.randomBytes(8).toString('hex'); + assert.strictEqual(legacyHex.length, 16); + assert.strictEqual(compare(generateVersionId('test', 'RG001'), legacyHex), Ordering.NOT_COMPARABLE); + }); + + it('should return NOT_COMPARABLE when incoming is a legacy 16-char hex microVersionId', () => { + const legacyHex = crypto.randomBytes(8).toString('hex'); + assert.strictEqual(compare(legacyHex, generateVersionId('test', 'RG001')), Ordering.NOT_COMPARABLE); + }); + + it('should return NOT_COMPARABLE for a string shorter than 27 chars', () => { + assert.strictEqual(compare('tooshort', generateVersionId('test', 'RG001')), Ordering.NOT_COMPARABLE); + }); + + it('should return EQUAL when incoming equals existing', () => { + const raw = generateVersionId('test', 'RG001'); + assert.strictEqual(compare(raw, raw), Ordering.EQUAL); + }); + + it('should return OLDER when incoming is older than existing', () => { + const older = generateVersionId('test', 'RG001'); + const newer = generateVersionId('test', 'RG001'); + assert.ok(older > newer, 'test requires two distinct versionIds in order'); + assert.strictEqual(compare(older, newer), Ordering.OLDER); + }); + + it('should return YOUNGER when incoming is newer than existing', () => { + const older = generateVersionId('test', 'RG001'); + const newer = generateVersionId('test', 'RG001'); + assert.ok(older > newer, 'test requires two distinct versionIds in order'); + assert.strictEqual(compare(newer, older), Ordering.YOUNGER); + }); +});