From a8fe8fffcaba29ca1766e3d4f6d9638669aab65a Mon Sep 17 00:00:00 2001 From: Dong Nguyen Date: Fri, 26 Apr 2024 13:59:57 +0700 Subject: [PATCH 1/2] v7.1.1 - Decode content using detected charset - Update dependencies - Update eslint config - Add node 22 to CI test Related issue: #123 --- .eslintignore | 2 - .eslintrc.json | 121 ------------------------------- .github/workflows/ci-test.yml | 2 +- eslint.config.js | 129 ++++++++++++++++++++++++++++++++++ package.json | 5 +- src/utils/linker.js | 6 +- src/utils/normalizer.js | 2 +- src/utils/retrieve.js | 11 ++- 8 files changed, 145 insertions(+), 133 deletions(-) delete mode 100644 .eslintignore delete mode 100644 .eslintrc.json create mode 100644 eslint.config.js diff --git a/.eslintignore b/.eslintignore deleted file mode 100644 index f06235c..0000000 --- a/.eslintignore +++ /dev/null @@ -1,2 +0,0 @@ -node_modules -dist diff --git a/.eslintrc.json b/.eslintrc.json deleted file mode 100644 index 42c1225..0000000 --- a/.eslintrc.json +++ /dev/null @@ -1,121 +0,0 @@ -{ - "parserOptions": { - "ecmaVersion": "latest", - "sourceType": "module" - }, - "env": { - "es6": true, - "node": true, - "browser": true, - "jest": true - }, - "globals": { - "globalThis": true - }, - "plugins": [], - "overrides": [], - "extends": ["eslint:recommended"], - "rules": { - "arrow-spacing": ["error", { "before": true, "after": true }], - "block-spacing": ["error", "always"], - "brace-style": ["error", "1tbs", { "allowSingleLine": true }], - "camelcase": ["error", { - "allow": ["^UNSAFE_"], - "properties": "never", - "ignoreGlobals": true - }], - "comma-dangle": ["error", { - "arrays": "always-multiline", - "objects": "always-multiline", - "imports": "never", - "exports": "never", - "functions": "never" - }], - "comma-spacing": ["error", { "before": false, "after": true }], - "eol-last": "error", - "eqeqeq": ["error", "always", { "null": "ignore" }], - "func-call-spacing": ["error", "never"], - "indent": [ - "error", - 2, - { - "MemberExpression": 1, - "FunctionDeclaration": { - "body": 1, - "parameters": 2 - }, - "SwitchCase": 1 - } - ], - "key-spacing": ["error", { "beforeColon": false, "afterColon": true }], - "keyword-spacing": ["error", { "before": true, "after": true }], - "lines-between-class-members": ["error", "always", { "exceptAfterSingleLine": true }], - "max-len": [ - "error", - { - "code": 120, - "ignoreTrailingComments": true, - "ignoreComments": true, - "ignoreUrls": true - } - ], - "max-lines": [ - "error", - { - "max": 520, - "skipBlankLines": true, - "skipComments": false - } - ], - "max-lines-per-function": [ - "error", - { - "max": 240, - "skipBlankLines": true - } - ], - "max-params": ["error", 3], - "no-array-constructor": "error", - "no-mixed-spaces-and-tabs": "error", - "no-multi-spaces": "error", - "no-multi-str": "error", - "no-multiple-empty-lines": [ - "error", - { - "max": 1, - "maxEOF": 0 - } - ], - "no-restricted-syntax": [ - "error", - "WithStatement", - "BinaryExpression[operator='in']" - ], - "no-trailing-spaces": "error", - "no-use-before-define": [ - "error", - { - "functions": true, - "classes": true, - "variables": false - } - ], - "no-var": "warn", - "object-curly-spacing": ["error", "always"], - "padded-blocks": [ - "error", - { - "blocks": "never", - "switches": "never", - "classes": "never" - } - ], - "quotes": ["error", "single"], - "space-before-blocks": ["error", "always"], - "space-before-function-paren": ["error", "always"], - "space-infix-ops": "error", - "space-unary-ops": ["error", { "words": true, "nonwords": false }], - "space-in-parens": ["error", "never"], - "semi": ["error", "never"] - } -} diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml index c2c2785..dcc0c84 100755 --- a/.github/workflows/ci-test.yml +++ b/.github/workflows/ci-test.yml @@ -12,7 +12,7 @@ jobs: strategy: matrix: - node_version: [18.x, 20.x, 21.x] + node_version: [18.x, 20.x, 22.x] steps: - uses: actions/checkout@v4 diff --git a/eslint.config.js b/eslint.config.js new file mode 100644 index 0000000..1d16599 --- /dev/null +++ b/eslint.config.js @@ -0,0 +1,129 @@ +// eslint.config.js + +import eslintjs from '@eslint/js' +import globals from 'globals' + +export default [ + eslintjs.configs.recommended, + { + languageOptions: { + ecmaVersion: 'latest', + sourceType: 'module', + globals: { + ...globals.node, + ...globals.browser, + ...globals.jest, + Intl: 'readonly', + }, + }, + ignores: [ + 'node_modules', + 'storage', + '*.cjs', + ], + rules: { + 'arrow-spacing': ['error', { 'before': true, 'after': true }], + 'block-spacing': ['error', 'always'], + 'brace-style': ['error', '1tbs', { 'allowSingleLine': true }], + 'camelcase': ['error', { + 'allow': ['^UNSAFE_'], + 'properties': 'never', + 'ignoreGlobals': true, + }], + 'comma-dangle': ['error', { + 'arrays': 'always-multiline', + 'objects': 'always-multiline', + 'imports': 'never', + 'exports': 'never', + 'functions': 'never', + }], + 'comma-spacing': ['error', { 'before': false, 'after': true }], + 'eol-last': 'error', + 'eqeqeq': ['error', 'always', { 'null': 'ignore' }], + 'func-call-spacing': ['error', 'never'], + 'indent': [ + 'error', + 2, + { + 'MemberExpression': 1, + 'FunctionDeclaration': { + 'body': 1, + 'parameters': 2, + }, + 'SwitchCase': 1, + 'ignoredNodes': ['TemplateLiteral > *'], + }, + ], + 'key-spacing': ['error', { 'beforeColon': false, 'afterColon': true }], + 'keyword-spacing': ['error', { 'before': true, 'after': true }], + 'lines-between-class-members': ['error', 'always', { 'exceptAfterSingleLine': true }], + 'max-len': [ + 'error', + { + 'code': 120, + 'ignoreTrailingComments': true, + 'ignoreComments': true, + 'ignoreUrls': true, + }, + ], + 'max-lines': [ + 'error', + { + 'max': 500, + 'skipBlankLines': true, + 'skipComments': false, + }, + ], + 'max-lines-per-function': [ + 'error', + { + 'max': 200, + 'skipBlankLines': true, + }, + ], + 'max-params': ['error', 3], + 'no-array-constructor': 'error', + 'no-mixed-spaces-and-tabs': 'error', + 'no-multi-spaces': 'error', + 'no-multi-str': 'error', + 'no-multiple-empty-lines': [ + 'error', + { + 'max': 1, + 'maxEOF': 0, + }, + ], + 'no-restricted-syntax': [ + 'error', + 'WithStatement', + 'BinaryExpression[operator=\'in\']', + ], + 'no-trailing-spaces': 'error', + 'no-use-before-define': [ + 'error', + { + 'functions': true, + 'classes': true, + 'variables': false, + }, + ], + 'no-var': 'warn', + 'object-curly-spacing': ['error', 'always'], + 'padded-blocks': [ + 'error', + { + 'blocks': 'never', + 'switches': 'never', + 'classes': 'never', + }, + ], + 'quotes': ['error', 'single'], + 'space-before-blocks': ['error', 'always'], + 'space-before-function-paren': ['error', 'always'], + 'space-infix-ops': 'error', + 'space-unary-ops': ['error', { 'words': true, 'nonwords': false }], + 'space-in-parens': ['error', 'never'], + 'semi': ['error', 'never'], + }, + }, +] diff --git a/package.json b/package.json index 5d5168b..b053f7e 100755 --- a/package.json +++ b/package.json @@ -1,5 +1,5 @@ { - "version": "7.1.0", + "version": "7.1.1", "name": "@extractus/feed-extractor", "description": "To read and normalize RSS/ATOM/JSON feed data", "homepage": "https://extractor-demos.pages.dev", @@ -46,7 +46,8 @@ }, "devDependencies": { "esbuild": "^0.20.2", - "eslint": "^8.57.0", + "eslint": "^9.1.1", + "globals": "^15.0.0", "https-proxy-agent": "^7.0.4", "jest": "^29.7.0", "nock": "^13.5.4" diff --git a/src/utils/linker.js b/src/utils/linker.js index 78b42e1..d86e47a 100755 --- a/src/utils/linker.js +++ b/src/utils/linker.js @@ -4,7 +4,7 @@ export const isValid = (url = '') => { try { const ourl = new URL(url) return ourl !== null && ourl.protocol.startsWith('http') - } catch (err) { + } catch { return false } } @@ -13,7 +13,7 @@ export const absolutify = (fullUrl = '', relativeUrl = '') => { try { const result = new URL(relativeUrl, fullUrl) return result.toString() - } catch (err) { + } catch { return '' } } @@ -87,7 +87,7 @@ export const purify = (url) => { }) return pureUrl.toString().replace(pureUrl.hash, '') - } catch (err) { + } catch { return null } } diff --git a/src/utils/normalizer.js b/src/utils/normalizer.js index 5a50b15..5b882bb 100644 --- a/src/utils/normalizer.js +++ b/src/utils/normalizer.js @@ -16,7 +16,7 @@ import { absolutify, isValid as isValidUrl, purify as purifyUrl } from './linker export const toISODateString = (dstr) => { try { return dstr ? (new Date(dstr)).toISOString() : '' - } catch (err) { + } catch { return '' } } diff --git a/src/utils/retrieve.js b/src/utils/retrieve.js index c4db1b3..c17127d 100755 --- a/src/utils/retrieve.js +++ b/src/utils/retrieve.js @@ -32,17 +32,22 @@ export default async (url, options = {}) => { throw new Error(`Request failed with error code ${status}`) } const contentType = res.headers.get('content-type') - const text = await res.text() + const buffer = await res.arrayBuffer() + const text = buffer ? Buffer.from(buffer).toString().trim() : '' if (/(\+|\/)(xml|html)/.test(contentType)) { - return { type: 'xml', text: text.trim(), status, contentType } + const arr = contentType.split('charset=') + const charset = arr.length === 2 ? arr[1].trim() : 'utf8' + const decoder = new TextDecoder(charset) + const xml = decoder.decode(buffer) + return { type: 'xml', text: xml.trim(), status, contentType } } if (/(\+|\/)json/.test(contentType)) { try { const data = JSON.parse(text) return { type: 'json', json: data, status, contentType } - } catch (err) { + } catch { throw new Error('Failed to convert data to JSON object') } } From 0bd1f4b5d166632d8ba9a671cf1cee97e7e008c6 Mon Sep 17 00:00:00 2001 From: Dong Nguyen Date: Fri, 26 Apr 2024 14:06:31 +0700 Subject: [PATCH 2/2] Remove build from CI test --- .github/workflows/ci-test.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml index dcc0c84..b541040 100755 --- a/.github/workflows/ci-test.yml +++ b/.github/workflows/ci-test.yml @@ -27,8 +27,6 @@ jobs: PROXY_SERVER: ${{ secrets.PROXY_SERVER }} run: | npm install - npm run lint - npm run build --if-present npm run test - name: Coveralls Parallel