Merge pull request #70 from extractus/6.1.7

v6.1.7

README.md

@@ -231,7 +231,7 @@ git clone https://github.com/extractus/feed-extractor.git
 cd feed-extractor
 npm install
 
-node eval.js --url=https://news.google.com/rss --normalization=y --useISODateFormat=y --includeEntryContent=n --includeOptionalElements=n
+npm run eval https://news.google.com/rss
 ```
 
 ## License

dist/cjs/package.json

@@ -1,5 +1,5 @@
 {
   "name": "@extractus/feed-extractor",
-  "version": "6.1.6",
+  "version": "6.1.7",
   "main": "./feed-extractor.js"
 }

dist/feed-extractor.esm.js

@@ -1,4 +1,4 @@
-// @extractus/[email protected].6, by @extractus - built with esbuild at 2022-12-05T01:51:24.880Z - published under MIT license
+// @extractus/[email protected].7, by @extractus - built with esbuild at 2022-12-06T05:40:03.694Z - published under MIT license
 var __create = Object.create;
 var __defProp = Object.defineProperty;
 var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
@@ -1896,7 +1896,7 @@ var retrieve_default = async (url, options = {}) => {
   }
   const contentType = res.headers.get("content-type");
   const text = await res.text();
-  if (/(\+|\/)xml/.test(contentType)) {
+  if (/(\+|\/)(xml|html)/.test(contentType)) {
     return { type: "xml", text: text.trim(), status, contentType };
   }
   if (/(\+|\/)json/.test(contentType)) {

eval.js

@@ -1,37 +1,23 @@
 // eval.js
-
-// import { writeFileSync } from 'fs'
-
-import parseArgs from 'args-parser'
+// to quickly test with a single url or file
 
 import { read } from './src/main.js'
 
-const extractFromUrl = async (url, options) => {
+const run = async (url) => {
   try {
-    const feed = await read(url, options)
-    console.log(feed)
-    // writeFileSync('output.json', JSON.stringify(feed, undefined, 2), 'utf8')
+    const art = await read(url)
+    console.log(art)
   } catch (err) {
-    console.log(err)
+    console.trace(err)
   }
 }
 
 const init = (argv) => {
-  const {
-    url,
-    normalization = 'y',
-    includeEntryContent = 'n',
-    includeOptionalElements = 'n',
-    useISODateFormat = 'y'
-  } = parseArgs(argv)
-
-  const options = {
-    includeEntryContent: includeEntryContent === 'y',
-    includeOptionalElements: includeOptionalElements === 'y',
-    useISODateFormat: useISODateFormat !== 'n',
-    normalization: normalization !== 'n'
+  if (argv.length === 3) {
+    const url = argv[2]
+    return run(url)
   }
-  return url ? extractFromUrl(url, options) : false
+  return 'Nothing to do!'
 }
 
 init(process.argv)

package.json

@@ -1,5 +1,5 @@
 {
-  "version": "6.1.6",
+  "version": "6.1.7",
   "name": "@extractus/feed-extractor",
   "description": "To read and normalize RSS/ATOM/JSON feed data",
   "homepage": "https://github.com/extractus/feed-extractor",
@@ -29,6 +29,7 @@
     "lint:fix": "standard --fix",
     "pretest": "npm run lint",
     "test": "NODE_ENV=test NODE_OPTIONS=--experimental-vm-modules jest --verbose --coverage=true",
+    "eval": "node eval",
     "build": "node build",
     "reset": "node reset"
   },
@@ -39,8 +40,7 @@
     "html-entities": "^2.3.3"
   },
   "devDependencies": {
-    "args-parser": "^1.3.0",
-    "esbuild": "^0.15.17",
+    "esbuild": "^0.15.18",
     "jest": "^29.3.1",
     "nock": "^13.2.9",
     "standard": "^17.0.0"

src/utils/retrieve.js

@@ -30,7 +30,7 @@ export default async (url, options = {}) => {
   const contentType = res.headers.get('content-type')
   const text = await res.text()
 
-  if (/(\+|\/)xml/.test(contentType)) {
+  if (/(\+|\/)(xml|html)/.test(contentType)) {
     return { type: 'xml', text: text.trim(), status, contentType }
   }