Skip to content

v8.1.0 Update #86

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 8 commits into
base: master
Choose a base branch
from
Open
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 14 additions & 5 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
@@ -3,15 +3,24 @@ on:
- push
- pull_request
jobs:
build:
name: Build with tsc
runs-on: ubuntu-latest
continue-on-error: true
steps:
- uses: actions/checkout@v4
- uses: actions/setup-node@v4
with:
node-version: lts/*
- run: npm install
- run: npx tsc
test:
name: Node.js ${{ matrix.node-version }} on ${{ matrix.os }}
name: Test ${{ matrix.node-version }} on ${{ matrix.os }}
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
matrix:
node-version:
- 18
- 20
node: [ 18, 20 ]
os:
- ubuntu-latest
- windows-latest
@@ -21,4 +30,4 @@ jobs:
with:
node-version: ${{ matrix.node-version }}
- run: npm install
- run: npm test
- run: npm run test:coverage
2 changes: 2 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
/node_modules
/lib
/@types
npm-debug.log
.DS_Store
1 change: 1 addition & 0 deletions .npmignore
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
/test
/src
.eslintrc.js
.gitignore
17 changes: 0 additions & 17 deletions index.js

This file was deleted.

289 changes: 0 additions & 289 deletions lib/pocompiler.js

This file was deleted.

19 changes: 13 additions & 6 deletions package.json
Original file line number Diff line number Diff line change
@@ -18,28 +18,35 @@
"node": ">=18"
},
"scripts": {
"lint": "eslint lib/*.js test/*.js index.js",
"lint": "eslint src/*.js test/*.js",
"test-generate-mo": "msgfmt test/fixtures/latin13.po -o test/fixtures/latin13.mo & msgfmt test/fixtures/utf8.po -o test/fixtures/utf8.mo & msgfmt test/fixtures/obsolete.po -o test/fixtures/obsolete.mo",
"test": "mocha",
"test:coverage": "npx c8 --check-coverage npm run test",
"preversion": "npm run lint && npm test",
"postversion": "git push && git push --tags"
"postversion": "git push && git push --tags",
"prepublishOnly": "npm i && tsc && npm run lint && npm run test"
},
"main": "./index.js",
"main": "./lib/index.js",
"types": "./lib/index.d.ts",
"license": "MIT",
"dependencies": {
"content-type": "^1.0.5",
"encoding": "^0.1.13",
"readable-stream": "^4.5.2",
"safe-buffer": "^5.2.1"
"readable-stream": "^4.5.2"
},
"devDependencies": {
"@types/chai": "latest",
"@types/content-type": "^1.1.8",
"@types/mocha": "latest",
"@types/readable-stream": "^4.0.11",
"chai": "^5.0.3",
"eslint": "^8.56.0",
"eslint-config-standard": "^17.1.0",
"eslint-plugin-import": "^2.29.1",
"eslint-plugin-n": "^16.6.2",
"eslint-plugin-promise": "^6.1.1",
"mocha": "^10.3.0"
"mocha": "^10.4.0",
"typescript": "^5.4.5"
},
"keywords": [
"i18n",
25 changes: 25 additions & 0 deletions src/index.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
import { poParse, poStream } from './poparser.js';
import poCompiler from './pocompiler.js';
import moParser from './moparser.js';
import moCompiler from './mocompiler.js';

/**
* Translation parser and compiler for PO files
* @see https://www.gnu.org/software/gettext/manual/html_node/PO.html
*/
export const po = {
parse: poParse,
createParseStream: poStream,
compile: poCompiler
};

/**
* Translation parser and compiler for MO files
* @see https://www.gnu.org/software/gettext/manual/html_node/MO.html
*/
export const mo = {
parse: moParser,
compile: moCompiler
};

export default { mo, po };
144 changes: 93 additions & 51 deletions lib/mocompiler.js → src/mocompiler.js
Original file line number Diff line number Diff line change
@@ -1,81 +1,119 @@
import { Buffer } from 'safe-buffer';
import encoding from 'encoding';
import { HEADERS, formatCharset, generateHeader, compareMsgid } from './shared.js';
import contentType from 'content-type';

/**
* @typedef {import('node:stream').Transform} Transform
* @typedef {import('./types.js').GetTextTranslation} GetTextTranslation
* @typedef {import('./types.js').GetTextTranslations} GetTextTranslations
* @typedef {import('./types.js').Translations} Translations
* @typedef {import('./types.js').WriteFunc} WriteFunc
*/

/**
* @typedef {Object} Size Data about the size of the compiled MO object.
* @property {number} msgid The size of the msgid section.
* @property {number} msgstr The size of the msgstr section.
* @property {number} total The total size of the compiled MO object.
*/

/**
* @typedef {{ msgid: Buffer, msgstr: Buffer }} TranslationBuffers A translation object partially parsed.
*/

/**
* Exposes general compiler function. Takes a translation
* object as a parameter and returns binary MO object
*
* @param {Object} table Translation object
* @param {GetTextTranslations} table Translation object
* @return {Buffer} Compiled binary MO object
*/
export default function (table) {
const compiler = new Compiler(table);

return compiler.compile();
};
}

/**
* Creates a MO compiler object.
*
* @constructor
* @param {Object} table Translation table as defined in the README
* Prepare the header object to be compatible with MO compiler
* @param {Record<string, string>} headers the headers
* @return {Record<string, string>} The prepared header
*/
function Compiler (table = {}) {
this._table = table;

let { headers = {}, translations = {} } = this._table;

headers = Object.keys(headers).reduce((result, key) => {
function prepareMoHeaders (headers) {
return Object.keys(headers).reduce((result, key) => {
const lowerKey = key.toLowerCase();

if (HEADERS.has(lowerKey)) {
// POT-Creation-Date is removed in MO (see https://savannah.gnu.org/bugs/?49654)
if (lowerKey !== 'pot-creation-date') {
result[HEADERS.get(lowerKey)] = headers[key];
const value = HEADERS.get(lowerKey);
if (value) {
result[value] = headers[key];
}
}
} else {
result[key] = headers[key];
}

return result;
}, {});
}, /** @type {Record<string, string>} */ ({}));
}

// filter out empty translations
translations = Object.keys(translations).reduce((result, msgctxt) => {
/**
* Prepare the translation object to be compatible with MO compiler
* @param {Translations} translations
* @return {Translations}
*/
function prepareTranslations (translations) {
return Object.keys(translations).reduce((result, msgctxt) => {
const context = translations[msgctxt];
const msgs = Object.keys(context).reduce((result, msgid) => {
const hasTranslation = context[msgid].msgstr.some(item => !!item.length);
const TranslationMsgstr = context[msgid].msgstr;
const hasTranslation = TranslationMsgstr.some(item => !!item.length);

if (hasTranslation) {
result[msgid] = context[msgid];
}

return result;
}, {});
}, /** @type {Record<string, GetTextTranslation>} */({}));

if (Object.keys(msgs).length) {
result[msgctxt] = msgs;
}

return result;
}, {});
}, /** @type {Translations} */({}));
}

this._table.translations = translations;
this._table.headers = headers;
/**
* Creates a MO compiler object.
* @this {Compiler & Transform}
*
* @param {GetTextTranslations} [table] Translation table as defined in the README
*/
function Compiler (table) {
/** @type {GetTextTranslations} _table The translation table */
this._table = {
charset: undefined,
translations: prepareTranslations(table?.translations ?? {}),
headers: prepareMoHeaders(table?.headers ?? {})
};

this._translations = [];

/**
* @type {WriteFunc}
*/
this._writeFunc = 'writeUInt32LE';

this._handleCharset();
}

/**
* Magic bytes for the generated binary data
*/
Compiler.prototype.MAGIC = 0x950412de;
/**
* Magic bytes for the generated binary data
* @type {number} MAGIC file header magic value of mo file
*/
this.MAGIC = 0x950412de;
}

/**
* Handles header values, replaces or adds (if needed) a charset property
@@ -96,17 +134,19 @@ Compiler.prototype._handleCharset = function () {

/**
* Generates an array of translation strings
* in the form of [{msgid:... , msgstr:...}]
* in the form of [{msgid:..., msgstr: ...}]
*
* @return {Array} Translation strings array
*/
Compiler.prototype._generateList = function () {
/** @type {TranslationBuffers[]} */
const list = [];

list.push({
msgid: Buffer.alloc(0),
msgstr: encoding.convert(generateHeader(this._table.headers), this._table.charset)
});
if ('headers' in this._table) {
list.push({
msgid: Buffer.alloc(0),
msgstr: encoding.convert(generateHeader(this._table.headers), this._table.charset)
});
}

Object.keys(this._table.translations).forEach(msgctxt => {
if (typeof this._table.translations[msgctxt] !== 'object') {
@@ -133,7 +173,7 @@ Compiler.prototype._generateList = function () {
key += '\u0000' + msgidPlural;
}

const value = [].concat(this._table.translations[msgctxt][msgid].msgstr || []).join('\u0000');
const value = /** @type {string[]} */([]).concat(this._table.translations[msgctxt][msgid].msgstr ?? []).join('\u0000');

list.push({
msgid: encoding.convert(key, this._table.charset),
@@ -148,20 +188,19 @@ Compiler.prototype._generateList = function () {
/**
* Calculate buffer size for the final binary object
*
* @param {Array} list An array of translation strings from _generateList
* @return {Object} Size data of {msgid, msgstr, total}
* @param {TranslationBuffers[]} list An array of translation strings from _generateList
* @return {Size} Size data of {msgid, msgstr, total}
*/
Compiler.prototype._calculateSize = function (list) {
let msgidLength = 0;
let msgstrLength = 0;
let totalLength = 0;

list.forEach(translation => {
msgidLength += translation.msgid.length + 1; // + extra 0x00
msgstrLength += translation.msgstr.length + 1; // + extra 0x00
});

totalLength = 4 + // magic number
const totalLength = 4 + // magic number
4 + // revision
4 + // string count
4 + // original string table offset
@@ -183,9 +222,9 @@ Compiler.prototype._calculateSize = function (list) {
/**
* Generates the binary MO object from the translation list
*
* @param {Array} list translation list
* @param {Object} size Byte size information
* @return {Buffer} Compiled MO object
* @param {TranslationBuffers[]} list translation list
* @param {Size} size Byte size information
* @return {Buffer} Compiled MO object
Comment on lines +226 to +227

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We should remove these spaces... also would be nice to add a JSDoc eslint rules.

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

yep you are right, it seems that jsdocs aren't formatted at all.

What do you suggest to fix that issue?

Copy link

@johnhooks johnhooks May 13, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@erikyo I have a branch that is almost done, in which I manually format them. I looked into using eslint-plugin-jsdoc though I was having some trouble because eslint-config-standard hasn't be updated (by it's maintainers) and I would need to figure out what version of eslint-plugin-jsdoc should be added as a dev dependency. If there is support for adding the JSDoc plugin, I can figure it out but didn't want to commit much time to it, if there wasn't.

*/
Compiler.prototype._build = function (list, size) {
const returnBuffer = Buffer.alloc(size.total);
@@ -214,21 +253,23 @@ Compiler.prototype._build = function (list, size) {
// hash table offset
returnBuffer[this._writeFunc](28 + (4 + 4) * list.length * 2, 24);

// build originals table
// Build original table
curPosition = 28 + 2 * (4 + 4) * list.length;
for (i = 0, len = list.length; i < len; i++) {
list[i].msgid.copy(returnBuffer, curPosition);
returnBuffer[this._writeFunc](list[i].msgid.length, 28 + i * 8);
returnBuffer[this._writeFunc](curPosition, 28 + i * 8 + 4);
const msgidLength = /** @type {Buffer} */(/** @type {unknown} */(list[i].msgid));
msgidLength.copy(returnBuffer, curPosition);
returnBuffer.writeUInt32LE(list[i].msgid.length, 28 + i * 8);
returnBuffer.writeUInt32LE(curPosition, 28 + i * 8 + 4);
returnBuffer[curPosition + list[i].msgid.length] = 0x00;
curPosition += list[i].msgid.length + 1;
}

// build translations table
// build translation table
for (i = 0, len = list.length; i < len; i++) {
list[i].msgstr.copy(returnBuffer, curPosition);
returnBuffer[this._writeFunc](list[i].msgstr.length, 28 + (4 + 4) * list.length + i * 8);
returnBuffer[this._writeFunc](curPosition, 28 + (4 + 4) * list.length + i * 8 + 4);
const msgstrLength = /** @type {Buffer} */(/** @type {unknown} */(list[i].msgstr));
msgstrLength.copy(returnBuffer, curPosition);
returnBuffer.writeUInt32LE(list[i].msgstr.length, 28 + (4 + 4) * list.length + i * 8);
returnBuffer.writeUInt32LE(curPosition, 28 + (4 + 4) * list.length + i * 8 + 4);
returnBuffer[curPosition + list[i].msgstr.length] = 0x00;
curPosition += list[i].msgstr.length + 1;
}
@@ -237,8 +278,9 @@ Compiler.prototype._build = function (list, size) {
};

/**
* Compiles translation object into a binary MO object
* Compiles a translation object into a binary MO object
*
* @interface
* @return {Buffer} Compiled MO object
*/
Compiler.prototype.compile = function () {
130 changes: 75 additions & 55 deletions lib/moparser.js → src/moparser.js
Original file line number Diff line number Diff line change
@@ -1,65 +1,77 @@
import encoding from 'encoding';
import { formatCharset, parseHeader } from './shared.js';

/**
* @typedef {import('./types.js').GetTextTranslations} GetTextTranslations
* @typedef {import('./types.js').GetTextTranslation} GetTextTranslation
* @typedef {import('./types.js').Translations} Translations
* @typedef {import('./types.js').WriteFunc} WriteFunc
* @typedef {import('./types.js').ReadFunc} ReadFunc
*/

/**
* Parses a binary MO object into translation table
*
* @param {Buffer} buffer Binary MO object
* @param {String} [defaultCharset] Default charset to use
* @return {Object} Translation object
* @param {string} [defaultCharset] Default charset to use
*/
export default function (buffer, defaultCharset) {
const parser = new Parser(buffer, defaultCharset);

return parser.parse();
};
}

/**
* Creates a MO parser object.
*
* @constructor
* @param {Buffer} fileContents Binary MO object
* @param {String} [defaultCharset] Default charset to use
* @param {Buffer|null} fileContents Binary MO object
* @param {string} [defaultCharset] Default charset to use
*/
function Parser (fileContents, defaultCharset = 'iso-8859-1') {
this._fileContents = fileContents;

this._charset = defaultCharset;

/**
* Method name for writing int32 values, default littleendian
*/
* @type {WriteFunc}
*/
this._writeFunc = 'writeUInt32LE';

/**
* Method name for reading int32 values, default littleendian
*/
* @type {ReadFunc}
*/
this._readFunc = 'readUInt32LE';

this._charset = defaultCharset;

/**
* Translation table
*
* @type {GetTextTranslations} table Translation object
*/
this._table = {
charset: this._charset,
headers: undefined,
headers: {},
translations: {}
};
}

/**
* Magic constant to check the endianness of the input file
*/
Parser.prototype.MAGIC = 0x950412de;
/**
* Magic constant to check the endianness of the input file
*/
this.MAGIC = 0x950412de;
}

/**
* Checks if number values in the input file are in big- or littleendian format.
* Checks if number values in the input file are in big- or little endian format.
*
* @return {Boolean} Return true if magic was detected
* @return {boolean} Return true if magic was detected
*/
Parser.prototype._checkMagick = function () {
if (this._fileContents.readUInt32LE(0) === this.MAGIC) {
if (this._fileContents?.readUInt32LE(0) === this.MAGIC) {
this._readFunc = 'readUInt32LE';
this._writeFunc = 'writeUInt32LE';

return true;
} else if (this._fileContents.readUInt32BE(0) === this.MAGIC) {
} else if (this._fileContents?.readUInt32BE(0) === this.MAGIC) {
this._readFunc = 'readUInt32BE';
this._writeFunc = 'writeUInt32BE';

@@ -70,31 +82,42 @@ Parser.prototype._checkMagick = function () {
};

/**
* Read the original strings and translations from the input MO file. Use the
* first translation string in the file as the header.
* Read the original strings and translations from the input MO file.
* Use the first translation string in the file as the header.
*/
Parser.prototype._loadTranslationTable = function () {
let offsetOriginals = this._offsetOriginals;
let offsetTranslations = this._offsetTranslations;
let offsetOriginals = this._offsetOriginals || 0;
let offsetTranslations = this._offsetTranslations || 0;
let position;
let length;
let msgid;
let msgstr;

// Return if there are no translations
if (!this._total) { this._fileContents = null; return; }

// Loop through all strings in the MO file
for (let i = 0; i < this._total; i++) {
if (this._fileContents === null) continue;
// msgid string
length = this._fileContents[this._readFunc](offsetOriginals);
offsetOriginals += 4;
position = this._fileContents[this._readFunc](offsetOriginals);
offsetOriginals += 4;
msgid = this._fileContents.slice(position, position + length);
msgid = this._fileContents.subarray(
position,
position + length
);

// matching msgstr
length = this._fileContents[this._readFunc](offsetTranslations);
offsetTranslations += 4;
position = this._fileContents[this._readFunc](offsetTranslations);
offsetTranslations += 4;
msgstr = this._fileContents.slice(position, position + length);
msgstr = this._fileContents.subarray(
position,
position + length
);

if (!i && !msgid.toString()) {
this._handleCharset(msgstr);
@@ -125,43 +148,40 @@ Parser.prototype._handleCharset = function (headers) {
this._charset = this._table.charset = formatCharset(match[1], this._charset);
}

headers = encoding.convert(headers, 'utf-8', this._charset)
.toString('utf8');
headers = encoding.convert(headers, 'utf-8', this._charset);

this._table.headers = parseHeader(headers);
this._table.headers = parseHeader(headers.toString('utf8'));
};

/**
* Adds a translation to the translation object
*
* @param {String} msgid Original string
* @params {String} msgstr Translation for the original string
* @param {string} msgidRaw Original string
* @param {string} msgstrRaw Translation for the original string
*/
Parser.prototype._addString = function (msgid, msgstr) {
Parser.prototype._addString = function (msgidRaw, msgstrRaw) {
const translation = {};
let msgctxt;
let msgctxt = '';
let msgidPlural;

msgid = msgid.split('\u0004');
if (msgid.length > 1) {
msgctxt = msgid.shift();
const msgidArray = msgidRaw.split('\u0004');
if (msgidArray.length > 1) {
msgctxt = msgidArray.shift() || '';
translation.msgctxt = msgctxt;
} else {
msgctxt = '';
}
msgid = msgid.join('\u0004');
msgidRaw = msgidArray.join('\u0004');

const parts = msgid.split('\u0000');
msgid = parts.shift();
const parts = msgidRaw.split('\u0000');
const msgid = parts.shift() || '';

translation.msgid = msgid;

if ((msgidPlural = parts.join('\u0000'))) {
translation.msgid_plural = msgidPlural;
}

msgstr = msgstr.split('\u0000');
translation.msgstr = [].concat(msgstr || []);
const msgstr = msgstrRaw.split('\u0000');
translation.msgstr = [...msgstr];

if (!this._table.translations[msgctxt]) {
this._table.translations[msgctxt] = {};
@@ -173,31 +193,31 @@ Parser.prototype._addString = function (msgid, msgstr) {
/**
* Parses the MO object and returns translation table
*
* @return {Object} Translation table
* @return {GetTextTranslations | false} Translation table
*/
Parser.prototype.parse = function () {
if (!this._checkMagick()) {
if (!this._checkMagick() || this._fileContents === null) {
return false;
}

/**
* GetText revision nr, usually 0
*/
* GetText revision nr, usually 0
*/
this._revision = this._fileContents[this._readFunc](4);

/**
* Total count of translated strings
*/
this._total = this._fileContents[this._readFunc](8);
* @type {number} Total count of translated strings
*/
this._total = this._fileContents[this._readFunc](8) ?? 0;

/**
* Offset position for original strings table
*/
* @type {number} Offset position for original strings table
*/
this._offsetOriginals = this._fileContents[this._readFunc](12);

/**
* Offset position for translation strings table
*/
* @type {number} Offset position for translation strings table
*/
this._offsetTranslations = this._fileContents[this._readFunc](16);

// Load translations into this._translationTable
324 changes: 324 additions & 0 deletions src/pocompiler.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,324 @@
import { HEADERS, foldLine, compareMsgid, formatCharset, generateHeader } from './shared.js';
import contentType from 'content-type';

import encoding from 'encoding';

/**
* @typedef {import('./types.js').GetTextTranslations} GetTextTranslations
* @typedef {import('./types.js').GetTextTranslation} GetTextTranslation
* @typedef {import('./types.js').GetTextComment} GetTextComment
* @typedef {import('./types.js').Translations} Translations
* @typedef {import('./types.js').ParserOptions} ParserOptions
*/

/**
* @typedef {Partial<Omit<GetTextTranslation, 'msgstr'>> & { msgstr?: string | string[] }} PreOutputTranslation
*/

/**
* Exposes general compiler function. Takes a translation
* object as a parameter and returns PO object
*
* @param {GetTextTranslations} table Translation object
* @param {ParserOptions} [options] Options
* @return {Buffer} The compiled PO object
*/
export default function (table, options) {
const compiler = new Compiler(table, options);

return compiler.compile();
}

/**
* Takes the header object and converts all headers into the lowercase format
*
* @param {Record<string, string>} headersRaw the headers to prepare
* @returns {Record<string, string>} the headers in the lowercase format
*/
export function preparePoHeaders (headersRaw) {
return Object.keys(headersRaw).reduce((result, key) => {
const lowerKey = key.toLowerCase();
const value = HEADERS.get(lowerKey);

if (typeof value === 'string') {
result[value] = headersRaw[key];
} else {
result[key] = headersRaw[key];
}

return result;
}, /** @type {Record<string, string>} */ ({}));
}

/**
* Creates a PO compiler object.
*
* @constructor
* @param {GetTextTranslations} [table] Translation table to be compiled
* @param {ParserOptions} [options] Options
*/
function Compiler (table, options) {
this._table = table ?? {
headers: {},
charset: undefined,
translations: {}
};
this._table.translations = { ...this._table.translations };

/** @type {ParserOptions} _options The Options object */
this._options = {
foldLength: 76,
escapeCharacters: true,
sort: false,
eol: '\n',
...options
};

/** @type {Record<string, string>}} the translation table */
this._table.headers = preparePoHeaders(this._table.headers ?? {});

this._translations = [];

this._handleCharset();
}

/**
* Converts a comment object to a comment string. The comment object is
* in the form of {translator: '', reference: '', extracted: '', flag: '', previous: ''}
*
* @param {Record<string, string>} comments A comments object
* @return {string} A comment string for the PO file
*/
Compiler.prototype._drawComments = function (comments) {
/** @var {Record<string, string[]>[]} lines The comment lines to be returned */
const lines = [];
/** @var {{key: GetTextComment, prefix: string}} type The comment type */
const types = [{
key: 'translator',
prefix: '# '
}, {
key: 'reference',
prefix: '#: '
}, {
key: 'extracted',
prefix: '#. '
}, {
key: 'flag',
prefix: '#, '
}, {
key: 'previous',
prefix: '#| '
}];

for (const type of types) {
/** @var {string} value The comment type */
const value = type.key;

// ignore empty comments
if (!(value in comments)) { continue; }

const commentLines = comments[value].split(/\r?\n|\r/);

// add comment lines to comments Array
for (const line of commentLines) {
lines.push(`${type.prefix}${line}`);
}
}

return lines.length ? lines.join(this._options.eol) : '';
};

/**
* Builds a PO string for a single translation object
*
* @param {PreOutputTranslation} block Translation object
* @param {Partial<PreOutputTranslation>} [override] Properties of this object will override `block` properties
* @param {boolean} [obsolete] Block is obsolete and must be commented out
* @return {string} Translation string for a single object
*/
Compiler.prototype._drawBlock = function (block, override = {}, obsolete = false) {
const response = [];
const msgctxt = override.msgctxt || block.msgctxt;
const msgid = override.msgid || block.msgid;
const msgidPlural = override.msgid_plural || block.msgid_plural;
const msgstrData = override.msgstr || block.msgstr;
const msgstr = Array.isArray(msgstrData) ? [...msgstrData] : [msgstrData];

/** @type {GetTextComment|undefined} */
const comments = override.comments || block.comments;
if (comments) {
const drawnComments = this._drawComments(comments);
if (drawnComments) {
response.push(drawnComments);
}
}

if (msgctxt) {
response.push(this._addPOString('msgctxt', msgctxt, obsolete));
}

response.push(this._addPOString('msgid', msgid || '', obsolete));

if (msgidPlural) {
response.push(this._addPOString('msgid_plural', msgidPlural, obsolete));

msgstr.forEach((msgstr, i) => {
response.push(this._addPOString(`msgstr[${i}]`, msgstr || '', obsolete));
});
} else {
response.push(this._addPOString('msgstr', msgstr[0] || '', obsolete));
}

return response.join(this._options.eol);
};

/**
* Escapes and joins a key and a value for the PO string
*
* @param {string} key Key name
* @param {string} value Key value
* @param {boolean} [obsolete] PO string is obsolete and must be commented out
* @return {string} Joined and escaped key-value pair
*/
Compiler.prototype._addPOString = function (key = '', value = '', obsolete = false) {
key = key.toString();
if (obsolete) {
key = '#~ ' + key;
}

let { foldLength, eol, escapeCharacters } = this._options;

// escape newlines and quotes
if (escapeCharacters) {
value = value.toString()
.replace(/\\/g, '\\\\')
.replace(/"/g, '\\"')
.replace(/\t/g, '\\t')
.replace(/\r/g, '\\r');
}

value = value.replace(/\n/g, '\\n'); // need to escape new line characters regardless

let lines = [value];

if (obsolete) {
eol = eol + '#~ ';
}

if (foldLength && foldLength > 0) {
lines = foldLine(value, foldLength);
} else {
// split only on new lines
if (escapeCharacters) {
lines = value.split(/\\n/g);
for (let i = 0; i < lines.length - 1; i++) {
lines[i] = `${lines[i]}\\n`;
}
if (lines.length && lines[lines.length - 1] === '') {
lines.splice(-1, 1);
}
}
}

if (lines.length < 2) {
return `${key} "${lines.shift() || ''}"`;
}

return `${key} ""${eol}"${lines.join(`"${eol}"`)}"`;
};

/**
* Handles header values, replaces or adds (if needed) a charset property
*/
Compiler.prototype._handleCharset = function () {
if (this._table.headers) {
const ct = contentType.parse(this._table.headers['Content-Type'] || 'text/plain');

const charset = formatCharset(this._table.charset || ct.parameters.charset || 'utf-8');

// clean up content-type charset independently using fallback if missing
if (ct.parameters.charset) {
ct.parameters.charset = formatCharset(ct.parameters.charset);
}

this._table.charset = charset;
this._table.headers['Content-Type'] = contentType.format(ct);
}
};

/**
* Flatten and sort translations object
*
* @param {Translations} section Object to be prepared (translations or obsolete)
* @returns {PreOutputTranslation[]|undefined} Prepared array
*/
Compiler.prototype._prepareSection = function (section) {
/** @type {GetTextTranslation[]} response Prepared array */
let response = [];

for (const msgctxt in section) {
if (typeof section[msgctxt] !== 'object') {
return;
}

for (const msgid of Object.keys(section[msgctxt])) {
if (typeof section[msgctxt][msgid] !== 'object') {
continue;
}

if (msgctxt === '' && msgid === '') {
continue;
}

response.push(section[msgctxt][msgid]);
}
}

const { sort } = this._options;

if (sort) {
if (typeof sort === 'function') {
response = response.sort(sort);
} else {
response = response.sort(compareMsgid);
}
}

return response;
};

/**
* Compiles a translation object into a PO object
*
* @interface
* @return {Buffer} Compiled a PO object
*/
Compiler.prototype.compile = function () {
if (!this._table.translations) {
throw new Error('No translations found');
}
/** @type {PreOutputTranslation} headerBlock */
const headerBlock = (this._table.translations[''] && this._table.translations['']['']) || {};

const translations = this._prepareSection(this._table.translations);
let response = /** @type {(PreOutputTranslation|string)[]} */ (/** @type {unknown[]} */ (translations?.map(t => this._drawBlock(t))));

if (typeof this._table.obsolete === 'object') {
const obsolete = this._prepareSection(this._table.obsolete);
if (obsolete && obsolete.length) {
response = response?.concat(obsolete.map(r => this._drawBlock(r, {}, true)));
}
}

const eol = this._options.eol ?? '\n';

response?.unshift(this._drawBlock(headerBlock, {
msgstr: generateHeader(this._table.headers)
}));

if (this._table.charset === 'utf-8' || this._table.charset === 'ascii') {
return Buffer.from(response?.join(eol + eol) + eol, 'utf-8');
}

return encoding.convert(response?.join(eol + eol) + eol, this._table.charset);
};
261 changes: 160 additions & 101 deletions lib/poparser.js → src/poparser.js

Large diffs are not rendered by default.

59 changes: 40 additions & 19 deletions lib/shared.js → src/shared.js
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
// see https://www.gnu.org/software/gettext/manual/html_node/Header-Entry.html
/** @type {string} Header name for "Plural-Forms" */
const PLURAL_FORMS = 'Plural-Forms';
/** @typedef {Map<string, string>} Headers Map of header keys to header names */
export const HEADERS = new Map([
['project-id-version', 'Project-Id-Version'],
['report-msgid-bugs-to', 'Report-Msgid-Bugs-To'],
@@ -18,12 +20,14 @@ const PLURAL_FORM_HEADER_NPLURALS_REGEX = /nplurals\s*=\s*(?<nplurals>\d+)/;
/**
* Parses a header string into an object of key-value pairs
*
* @param {String} str Header string
* @return {Object} An object of key-value pairs
* @param {string} str Header string
* @return {Record<string, string>} An object of key-value pairs
*/
export function parseHeader (str = '') {
return str.split('\n')
.reduce((headers, line) => {
/** @type {string} Header string */
return str
.split('\n')
.reduce((/** @type {Record<string, string>} */ headers, line) => {
const parts = line.split(':');
let key = (parts.shift() || '').trim();

@@ -42,11 +46,12 @@ export function parseHeader (str = '') {
/**
* Attempts to safely parse 'nplurals" value from "Plural-Forms" header
*
* @param {Object} [headers = {}] An object with parsed headers
* @param {Record<string, string>} [headers] An object with parsed headers
* @param {number} fallback Fallback value if "Plural-Forms" header is absent
* @returns {number} Parsed result
*/
export function parseNPluralFromHeadersSafely (headers = {}, fallback = 1) {
const pluralForms = headers[PLURAL_FORMS];
export function parseNPluralFromHeadersSafely (headers, fallback = 1) {
const pluralForms = headers ? headers[PLURAL_FORMS] : false;

if (!pluralForms) {
return fallback;
@@ -62,8 +67,8 @@ export function parseNPluralFromHeadersSafely (headers = {}, fallback = 1) {
/**
* Joins a header object of key value pairs into a header string
*
* @param {Object} header Object of key value pairs
* @return {String} Header string
* @param {Record<string, string>} header Object of key value pairs
* @return {string} An object of key-value pairs
*/
export function generateHeader (header = {}) {
const keys = Object.keys(header)
@@ -82,8 +87,9 @@ export function generateHeader (header = {}) {
/**
* Normalizes charset name. Converts utf8 to utf-8, WIN1257 to windows-1257 etc.
*
* @param {String} charset Charset name
* @return {String} Normalized charset name
* @param {string} charset Charset name
* @param {string} defaultCharset Default charset name, defaults to 'iso-8859-1'
* @return {string} Normalized charset name
*/
export function formatCharset (charset = 'iso-8859-1', defaultCharset = 'iso-8859-1') {
return charset.toString()
@@ -99,9 +105,9 @@ export function formatCharset (charset = 'iso-8859-1', defaultCharset = 'iso-885
/**
* Folds long lines according to PO format
*
* @param {String} str PO formatted string to be folded
* @param {Number} [maxLen=76] Maximum allowed length for folded lines
* @return {Array} An array of lines
* @param {string} str PO formatted string to be folded
* @param {number} [maxLen=76] Maximum allowed length for folded lines
* @return {string[]} An array of lines
*/
export function foldLine (str, maxLen = 76) {
const lines = [];
@@ -111,11 +117,11 @@ export function foldLine (str, maxLen = 76) {
let match;

while (pos < len) {
curLine = str.substr(pos, maxLen);
curLine = str.substring(pos, pos + maxLen);

// ensure that the line never ends with a partial escaping
// make longer lines if needed
while (curLine.substr(-1) === '\\' && pos + curLine.length < len) {
while (curLine.substring(-1) === '\\' && pos + curLine.length < len) {
curLine += str.charAt(pos + curLine.length);
}

@@ -125,7 +131,7 @@ export function foldLine (str, maxLen = 76) {
curLine = match[0];
} else if (pos + curLine.length < len) {
// if we're not at the end
if ((match = /.*\s+/.exec(curLine)) && /[^\s]/.test(match[0])) {
if ((match = /.*\s+/.exec(curLine)) && /\S/.test(match[0])) {
// use everything before and including the last white space character (if anything)
curLine = match[0];
} else if ((match = /.*[\x21-\x2f0-9\x5b-\x60\x7b-\x7e]+/.exec(curLine)) && /[^\x21-\x2f0-9\x5b-\x60\x7b-\x7e]/.test(match[0])) {
@@ -144,8 +150,9 @@ export function foldLine (str, maxLen = 76) {
/**
* Comparator function for comparing msgid
*
* @param {Object} object with msgid prev
* @param {Object} object with msgid next
* @template {Buffer|string} T
* @param {{msgid: T}} left with msgid prev
* @param {{msgid: T}} right with msgid next
* @returns {number} comparator index
*/
export function compareMsgid ({ msgid: left }, { msgid: right }) {
@@ -159,3 +166,17 @@ export function compareMsgid ({ msgid: left }, { msgid: right }) {

return 0;
}

/**
* Custom SyntaxError subclass that includes the lineNumber property.
*/
export class ParserError extends SyntaxError {
/**
* @param {string} message - Error message.
* @param {number} lineNumber - Line number where the error occurred.
*/
constructor (message, lineNumber) {
super(message);
this.lineNumber = lineNumber;
}
}
52 changes: 52 additions & 0 deletions src/types.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
/**
* Represents a GetText comment.
* @typedef {Object} GetTextComment
* @property {string} [translator] Translator information.
* @property {string} [reference] Reference information.
* @property {string} [extracted] Extracted comments.
* @property {string} [flag] Flags.
* @property {string} [previous] Previous string.
*/

/**
* Represents a GetText translation.
* @typedef {Object} GetTextTranslation
* @property {string} [msgctxt] Context of the message.
* @property {string} msgid The singular message ID.
* @property {string} [msgid_plural] The plural message ID.
* @property {string[]} msgstr Array of translated strings.
* @property {GetTextComment} [comments] Comments associated with the translation.
* @property {boolean} [obsolete] Whether the translation is obsolete.
*/

/**
* @typedef {Record<string, Record<string, GetTextTranslation>>} Translations The translations index.
*/

/**
* Represents GetText translations.
* @typedef {Object} GetTextTranslations
* @property {string|undefined} charset Character set.
* @property {Record<string, string>} headers Headers.
* @property {Translations} [obsolete] Obsolete messages.
* @property {Translations} translations Translations.
*/

/**
* Options for the parser.
* @typedef {Object} ParserOptions
* @property {string} [defaultCharset] Default character set.
* @property {boolean} [validation] Whether to perform validation.
* @property {number} [foldLength] the fold length.
* @property {boolean} [escapeCharacters] Whether to escape characters.
* @property {boolean} [sort] Whether to sort messages.
* @property {string} [eol] End of line character.
*/

/**
* @typedef {('writeUInt32LE'|'writeUInt32BE')} WriteFunc Type definition for write functions.
*/

/**
* @typedef {('readUInt32LE'|'readUInt32BE')} ReadFunc Type definition for read functions.
*/
10 changes: 5 additions & 5 deletions test/mo-compiler-test.js
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
import { promisify } from 'node:util';
import path from 'node:path';
import { mo } from '../src/index.js';
import { readFile as fsReadFile } from 'node:fs';
import { fileURLToPath } from 'node:url';
import * as chai from 'chai';
import { promisify } from 'util';
import path from 'path';
import { mo } from '../index.js';
import { readFile as fsReadFile } from 'fs';
import { fileURLToPath } from 'url';

const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);
10 changes: 5 additions & 5 deletions test/mo-parser-test.js
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
import { promisify } from 'node:util';
import path from 'node:path';
import { readFile as fsReadFile } from 'node:fs';
import { fileURLToPath } from 'node:url';
import * as chai from 'chai';
import { promisify } from 'util';
import path from 'path';
import { mo } from '../index.js';
import { readFile as fsReadFile } from 'fs';
import { fileURLToPath } from 'url';
import { mo } from '../src/index.js';

const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);
2 changes: 1 addition & 1 deletion test/module.mjs
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import { expect } from 'chai';
import { po, mo } from '../index.js';
import { po, mo } from '../src/index.js';

describe('esm module', () => {
it('should allow named imports', () => {
12 changes: 6 additions & 6 deletions test/po-compiler-test.js
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
import { EOL } from 'os';
import { promisify } from 'util';
import path from 'path';
import { readFile as fsReadFile } from 'fs';
import { readFile as fsReadFile } from 'node:fs';
import { promisify } from 'node:util';
import path from 'node:path';
import { EOL } from 'node:os';
import { fileURLToPath } from 'node:url';
import { po } from '../src/index.js';
import * as chai from 'chai';
import { po } from '../index.js';
import { fileURLToPath } from 'url';

const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);
12 changes: 6 additions & 6 deletions test/po-obsolete-test.js
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
import { EOL } from 'os';
import { EOL } from 'node:os';
import path from 'node:path';
import fs from 'node:fs';
import { promisify } from 'node:util';
import * as chai from 'chai';
import { promisify } from 'util';
import path from 'path';
import fs from 'fs';
import * as gettextParser from '../index.js';
import { fileURLToPath } from 'url';
import * as gettextParser from '../src/index.js';
import { fileURLToPath } from 'node:url';

const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);
10 changes: 5 additions & 5 deletions test/po-parser-test.js
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
import * as chai from 'chai';
import { promisify } from 'util';
import path from 'path';
import fs from 'fs';
import * as gettextParser from '../index.js';
import { fileURLToPath } from 'url';
import { promisify } from 'node:util';
import path from 'node:path';
import fs from 'node:fs';
import * as gettextParser from '../src/index.js';
import { fileURLToPath } from 'node:url';

const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);
12 changes: 5 additions & 7 deletions test/shared.js
Original file line number Diff line number Diff line change
@@ -1,11 +1,9 @@
'use strict';

import { promisify } from 'node:util';
import path from 'node:path';
import { readFile as fsReadFile } from 'node:fs';
import { fileURLToPath } from 'node:url';
import * as chai from 'chai';
import { promisify } from 'util';
import path from 'path';
import { formatCharset, parseHeader, generateHeader, foldLine, parseNPluralFromHeadersSafely } from '../lib/shared.js';
import { readFile as fsReadFile } from 'fs';
import { fileURLToPath } from 'url';
import { formatCharset, parseHeader, generateHeader, foldLine, parseNPluralFromHeadersSafely } from '../src/shared.js';

const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);
37 changes: 37 additions & 0 deletions tsconfig.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
{
"compilerOptions": {
"noImplicitAny": true,
"removeComments": false,
"module": "Node16",
"moduleResolution": "Node16",
"target": "ES2018",
"lib": [
"ES2018"
],
// Strict mode
"strict": true,
// Allow javascript files
"allowJs": true,
// Check js files for errors
"checkJs": true,
// the directory sources are in
"rootDir": "src",
// Output d.ts files to @types
"outDir": "lib",
// Generate d.ts files
"declaration": true,
// Minify
"pretty": false,
// Skip lib check when compiling
"skipLibCheck": true,
// For providing missing package types
"typeRoots": [
"./types",
"./node_modules/@types"
],
},
"include": [
"src/**/*",
"types/**/*"
]
}
3 changes: 3 additions & 0 deletions types/encoding/index.d.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
declare module 'encoding' {
function convert(buffer: Buffer | string, charset?: string, fromCharset?: string): Buffer;
}