Merge pull request #79 from extractus/6.2.0

v6.2.0

dist/cjs/package.json

@@ -1,5 +1,5 @@
 {
   "name": "@extractus/feed-extractor",
-  "version": "6.1.9",
+  "version": "6.2.0",
   "main": "./feed-extractor.js"
 }

dist/feed-extractor.esm.js

@@ -1,4 +1,4 @@
-// @extractus/feed-extractor@6.1.9, by @extractus - built with esbuild at 2023-01-04T04:09:49.802Z - published under MIT license
+// @extractus/feed-extractor@6.2.0, by @extractus - built with esbuild at 2023-01-04T04:59:05.957Z - published under MIT license
 var __create = Object.create;
 var __defProp = Object.defineProperty;
 var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
@@ -2003,7 +2003,7 @@ var getLink = (val = [], id = "") => {
   if (id && isValid(id)) {
     return id;
   }
-  if (isObject(id) && hasProperty(id, "@_isPermaLink") && Boolean(id["@_isPermaLink"]) === true) {
+  if (isObject(id) && hasProperty(id, "@_isPermaLink") && id["@_isPermaLink"] === "true") {
     return getText(id);
   }
   const getEntryLink = (links) => {
@@ -2233,13 +2233,15 @@ var transform3 = (item, options) => {
   const {
     id = "",
     title = "",
+    issued = "",
+    modified = "",
     updated = "",
     published = "",
     link = "",
     summary = "",
     content = ""
   } = item;
-  const pubDate = updated || published;
+  const pubDate = updated || modified || published || issued;
   const htmlContent = getText(content || summary);
   const entry = {
     id: getEntryId(id, link, pubDate),

package.json

@@ -1,5 +1,5 @@
 {
-  "version": "6.1.9",
+  "version": "6.2.0",
   "name": "@extractus/feed-extractor",
   "description": "To read and normalize RSS/ATOM/JSON feed data",
   "homepage": "https://github.com/extractus/feed-extractor",

src/main.test.js

@@ -5,9 +5,10 @@ import { readFileSync } from 'fs'
 
 import nock from 'nock'
 
-import { hasProperty } from 'bellajs'
+import { hasProperty, isString } from 'bellajs'
 
 import { read } from './main.js'
+import { isValid as isValidUrl } from './utils/linker.js'
 
 const feedAttrs = 'title link description generator language published entries'.split(' ')
 const entryAttrs = 'title link description published id'.split(' ')
@@ -20,6 +21,19 @@ const parseUrl = (url) => {
   }
 }
 
+const isValidDate = (d) => {
+  return (new Date(d)).toString() !== 'Invalid Date'
+}
+
+const validateProps = (entry) => {
+  const { id, link, title, published, description } = entry
+  return isString(description) &&
+    isString(id) && id !== '' &&
+    isString(title) && title !== '' &&
+    isString(link) && isValidUrl(link) &&
+    isString(published) && isValidDate(published)
+}
+
 describe('test read() function with common issues', () => {
   test('read feed from a non-string link', () => {
     expect(read([])).rejects.toThrow(new Error('Input param must be a valid URL'))
@@ -76,6 +90,7 @@ describe('test read() standard feed', (done) => {
     entryAttrs.forEach((k) => {
       expect(hasProperty(result.entries[0], k)).toBe(true)
     })
+    expect(validateProps(result.entries[0])).toBe(true)
   })
 
   test('read atom feed from Google', async () => {
@@ -92,6 +107,7 @@ describe('test read() standard feed', (done) => {
     entryAttrs.forEach((k) => {
       expect(hasProperty(result.entries[0], k)).toBe(true)
     })
+    expect(validateProps(result.entries[0])).toBe(true)
   })
 
   test('read atom feed from Google with extraFields', async () => {
@@ -115,6 +131,7 @@ describe('test read() standard feed', (done) => {
     })
     expect(hasProperty(result, 'author')).toBe(true)
     expect(hasProperty(result.entries[0], 'id')).toBe(true)
+    expect(validateProps(result.entries[0])).toBe(true)
   })
 
   test('read atom feed which contains multi links', async () => {
@@ -131,6 +148,7 @@ describe('test read() standard feed', (done) => {
     entryAttrs.forEach((k) => {
       expect(hasProperty(result.entries[0], k)).toBe(true)
     })
+    expect(validateProps(result.entries[0])).toBe(true)
   })
 
   test('read json feed from Micro.blog', async () => {
@@ -147,6 +165,7 @@ describe('test read() standard feed', (done) => {
     entryAttrs.forEach((k) => {
       expect(hasProperty(result.entries[0], k)).toBe(true)
     })
+    expect(validateProps(result.entries[0])).toBe(true)
   })
 
   test('read json feed from Micro.blog with extra fields', async () => {
@@ -170,6 +189,24 @@ describe('test read() standard feed', (done) => {
     })
     expect(hasProperty(result, 'icon')).toBe(true)
     expect(hasProperty(result.entries[0], 'id')).toBe(true)
+    expect(validateProps(result.entries[0])).toBe(true)
+  })
+
+  test('read rss feed from huggingface.co (no link)', async () => {
+    const url = 'https://huggingface.co/no-link/rss'
+    const xml = readFileSync('test-data/rss-feed-miss-link.xml', 'utf8')
+    const { baseUrl, path } = parseUrl(url)
+    nock(baseUrl).get(path).reply(200, xml, {
+      'Content-Type': 'application/xml'
+    })
+    const result = await read(url)
+    feedAttrs.forEach((k) => {
+      expect(hasProperty(result, k)).toBe(true)
+    })
+    entryAttrs.forEach((k) => {
+      expect(hasProperty(result.entries[0], k)).toBe(true)
+    })
+    expect(validateProps(result.entries[0])).toBe(true)
   })
 })
 

src/utils/normalizer.js

@@ -35,7 +35,7 @@ export const getLink = (val = [], id = '') => {
   if (id && isValidUrl(id)) {
     return id
   }
-  if (isObject(id) && hasProperty(id, '@_isPermaLink') && Boolean(id['@_isPermaLink']) === true) {
+  if (isObject(id) && hasProperty(id, '@_isPermaLink') && id['@_isPermaLink'] === 'true') {
     return getText(id)
   }
   const getEntryLink = (links) => {

src/utils/parseAtomFeed.js

@@ -23,14 +23,16 @@ const transform = (item, options) => {
   const {
     id = '',
     title = '',
+    issued = '',
+    modified = '',
     updated = '',
     published = '',
     link = '',
     summary = '',
     content = ''
   } = item
 
-  const pubDate = updated || published
+  const pubDate = updated || modified || published || issued
   const htmlContent = getText(content || summary)
   const entry = {
     id: getEntryId(id, link, pubDate),