From 76868398d2448dc7db3f1b2621dfe389cb6a1331 Mon Sep 17 00:00:00 2001 From: Henry Sachs Date: Fri, 18 Sep 2020 14:23:35 +0000 Subject: [PATCH 01/12] feat: fallback to info when version fails when the version endpoint fails use the single version from the info endpoint. This occurs for example when using artifactory. --- lib/datasource/rubygems/get.ts | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/lib/datasource/rubygems/get.ts b/lib/datasource/rubygems/get.ts index 2aabd2877a16b8..a638412be39ee1 100644 --- a/lib/datasource/rubygems/get.ts +++ b/lib/datasource/rubygems/get.ts @@ -52,7 +52,7 @@ export async function getDependency( const versions = (await fetch(dependency, registry, VERSIONS_PATH)) || []; - const releases = versions.map( + let releases = versions.map( ({ number: version, platform: rubyPlatform, @@ -68,6 +68,19 @@ export async function getDependency( }) ); + if (versions.length === 0 && info.version) { + logger.warn('falling back to using the info version'); + releases = [ + { + version: info.version, + rubyPlatform: info.platform, + releaseTimestamp: null, + rubyVersion: null, + rubygemsVersion: '\u003e= 0', + }, + ]; + } + return { releases, homepage: info.homepage_uri, From cca7aa967dfc7d03e59bff40e1c53b0cc45c3c52 Mon Sep 17 00:00:00 2001 From: Henry Sachs Date: Fri, 18 Sep 2020 14:45:55 +0000 Subject: [PATCH 02/12] fix: catch error when request errors --- lib/datasource/rubygems/get.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/datasource/rubygems/get.ts b/lib/datasource/rubygems/get.ts index a638412be39ee1..e0ad88026c4475 100644 --- a/lib/datasource/rubygems/get.ts +++ b/lib/datasource/rubygems/get.ts @@ -50,7 +50,12 @@ export async function getDependency( return null; } - const versions = (await fetch(dependency, registry, VERSIONS_PATH)) || []; + let versions = []; + try { + versions = (await fetch(dependency, registry, VERSIONS_PATH)) || []; + } catch { + logger.warn('falling back to using the version from the info endpoint'); + } let releases = versions.map( ({ @@ -69,7 +74,6 @@ export async function getDependency( ); if (versions.length === 0 && info.version) { - logger.warn('falling back to using the info version'); releases = [ { version: info.version, From ab58b0c11965678d707e7895a8eeacc4fd6a9cbc Mon Sep 17 00:00:00 2001 From: Henry Sachs Date: Fri, 18 Sep 2020 14:49:04 +0000 Subject: [PATCH 03/12] fix: revert try catch --- lib/datasource/rubygems/get.ts | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/lib/datasource/rubygems/get.ts b/lib/datasource/rubygems/get.ts index e0ad88026c4475..29ecb80f02b373 100644 --- a/lib/datasource/rubygems/get.ts +++ b/lib/datasource/rubygems/get.ts @@ -50,12 +50,7 @@ export async function getDependency( return null; } - let versions = []; - try { - versions = (await fetch(dependency, registry, VERSIONS_PATH)) || []; - } catch { - logger.warn('falling back to using the version from the info endpoint'); - } + const versions = (await fetch(dependency, registry, VERSIONS_PATH)) || []; let releases = versions.map( ({ @@ -74,6 +69,7 @@ export async function getDependency( ); if (versions.length === 0 && info.version) { + logger.warn('falling back to the version from the info endpoint'); releases = [ { version: info.version, From 450eda1d547e7bd76d25159941740c1b86179cc5 Mon Sep 17 00:00:00 2001 From: Henry Sachs Date: Fri, 18 Sep 2020 14:50:02 +0000 Subject: [PATCH 04/12] build: add yarn lock --- yarn.lock | 36 ++---------------------------------- 1 file changed, 2 insertions(+), 34 deletions(-) diff --git a/yarn.lock b/yarn.lock index 30b580964701bd..7e033ea9d0ee3f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3543,7 +3543,7 @@ debug@^3.1.0: dependencies: ms "^2.1.1" -debuglog@*, debuglog@^1.0.1: +debuglog@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/debuglog/-/debuglog-1.0.1.tgz#aa24ffb9ac3df9a2351837cfb2d279360cd78492" integrity sha1-qiT/uaw9+aI1GDfPstJ5NgzXhJI= @@ -5179,7 +5179,7 @@ import-local@^3.0.2: pkg-dir "^4.2.0" resolve-cwd "^3.0.0" -imurmurhash@*, imurmurhash@^0.1.4: +imurmurhash@^0.1.4: version "0.1.4" resolved "https://registry.yarnpkg.com/imurmurhash/-/imurmurhash-0.1.4.tgz#9218b9b2b928a238b13dc4fb6b6d576f231453ea" integrity sha1-khi5srkoojixPcT7a21XbyMUU+o= @@ -6651,11 +6651,6 @@ lockfile@^1.0.4: dependencies: signal-exit "^3.0.2" -lodash._baseindexof@*: - version "3.1.0" - resolved "https://registry.yarnpkg.com/lodash._baseindexof/-/lodash._baseindexof-3.1.0.tgz#fe52b53a1c6761e42618d654e4a25789ed61822c" - integrity sha1-/lK1OhxnYeQmGNZU5KJXie1hgiw= - lodash._baseuniq@~4.6.0: version "4.6.0" resolved "https://registry.yarnpkg.com/lodash._baseuniq/-/lodash._baseuniq-4.6.0.tgz#0ebb44e456814af7905c6212fa2c9b2d51b841e8" @@ -6664,33 +6659,11 @@ lodash._baseuniq@~4.6.0: lodash._createset "~4.0.0" lodash._root "~3.0.0" -lodash._bindcallback@*: - version "3.0.1" - resolved "https://registry.yarnpkg.com/lodash._bindcallback/-/lodash._bindcallback-3.0.1.tgz#e531c27644cf8b57a99e17ed95b35c748789392e" - integrity sha1-5THCdkTPi1epnhftlbNcdIeJOS4= - -lodash._cacheindexof@*: - version "3.0.2" - resolved "https://registry.yarnpkg.com/lodash._cacheindexof/-/lodash._cacheindexof-3.0.2.tgz#3dc69ac82498d2ee5e3ce56091bafd2adc7bde92" - integrity sha1-PcaayCSY0u5ePOVgkbr9Ktx73pI= - -lodash._createcache@*: - version "3.1.2" - resolved "https://registry.yarnpkg.com/lodash._createcache/-/lodash._createcache-3.1.2.tgz#56d6a064017625e79ebca6b8018e17440bdcf093" - integrity sha1-VtagZAF2JeeevKa4AY4XRAvc8JM= - dependencies: - lodash._getnative "^3.0.0" - lodash._createset@~4.0.0: version "4.0.3" resolved "https://registry.yarnpkg.com/lodash._createset/-/lodash._createset-4.0.3.tgz#0f4659fbb09d75194fa9e2b88a6644d363c9fe26" integrity sha1-D0ZZ+7CddRlPqeK4imZE02PJ/iY= -lodash._getnative@*, lodash._getnative@^3.0.0: - version "3.9.1" - resolved "https://registry.yarnpkg.com/lodash._getnative/-/lodash._getnative-3.9.1.tgz#570bc7dede46d61cdcde687d65d3eecbaa3aaff5" - integrity sha1-VwvH3t5G1hzc3mh9ZdPuy6o6r/U= - lodash._reinterpolate@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/lodash._reinterpolate/-/lodash._reinterpolate-3.0.0.tgz#0ccf2d89166af03b3663c796538b75ac6e114d9d" @@ -6736,11 +6709,6 @@ lodash.isstring@^4.0.1: resolved "https://registry.yarnpkg.com/lodash.isstring/-/lodash.isstring-4.0.1.tgz#d527dfb5456eca7cc9bb95d5daeaf88ba54a5451" integrity sha1-1SfftUVuynzJu5XV2ur4i6VKVFE= -lodash.restparam@*: - version "3.6.1" - resolved "https://registry.yarnpkg.com/lodash.restparam/-/lodash.restparam-3.6.1.tgz#936a4e309ef330a7645ed4145986c85ae5b20805" - integrity sha1-k2pOMJ7zMKdkXtQUWYbIWuWyCAU= - lodash.set@^4.3.2: version "4.3.2" resolved "https://registry.yarnpkg.com/lodash.set/-/lodash.set-4.3.2.tgz#d8757b1da807dde24816b0d6a84bea1a76230b23" From 52f6ef22c8b77c7f703d63e66e183d6d3e42b998 Mon Sep 17 00:00:00 2001 From: Henry Sachs Date: Mon, 21 Sep 2020 12:08:23 +0000 Subject: [PATCH 05/12] fix: catch the error from the failed request --- lib/datasource/rubygems/get.ts | 40 +++++++++++++++++++--------------- 1 file changed, 23 insertions(+), 17 deletions(-) diff --git a/lib/datasource/rubygems/get.ts b/lib/datasource/rubygems/get.ts index 29ecb80f02b373..d472c174d3b289 100644 --- a/lib/datasource/rubygems/get.ts +++ b/lib/datasource/rubygems/get.ts @@ -50,23 +50,13 @@ export async function getDependency( return null; } - const versions = (await fetch(dependency, registry, VERSIONS_PATH)) || []; - - let releases = versions.map( - ({ - number: version, - platform: rubyPlatform, - created_at: releaseTimestamp, - rubygems_version: rubygemsVersion, - ruby_version: rubyVersion, - }) => ({ - version, - rubyPlatform, - releaseTimestamp, - rubygemsVersion, - rubyVersion, - }) - ); + let versions = []; + let releases = []; + try { + versions = await fetch(dependency, registry, VERSIONS_PATH); + } catch (err) { + logger.debug('version endpoint not errors or is not available'); + } if (versions.length === 0 && info.version) { logger.warn('falling back to the version from the info endpoint'); @@ -79,6 +69,22 @@ export async function getDependency( rubygemsVersion: '\u003e= 0', }, ]; + } else { + releases = versions.map( + ({ + number: version, + platform: rubyPlatform, + created_at: releaseTimestamp, + rubygems_version: rubygemsVersion, + ruby_version: rubyVersion, + }) => ({ + version, + rubyPlatform, + releaseTimestamp, + rubygemsVersion, + rubyVersion, + }) + ); } return { From cc1f436611e98a5d5f5ba04fe0a60ffa336af0a3 Mon Sep 17 00:00:00 2001 From: Henry Sachs Date: Mon, 21 Sep 2020 14:02:32 +0000 Subject: [PATCH 06/12] fix: add errorhandling --- lib/datasource/rubygems/get.ts | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/lib/datasource/rubygems/get.ts b/lib/datasource/rubygems/get.ts index d472c174d3b289..481dff964e5416 100644 --- a/lib/datasource/rubygems/get.ts +++ b/lib/datasource/rubygems/get.ts @@ -8,7 +8,7 @@ import { id } from './common'; const http = new Http(id); const INFO_PATH = '/api/v1/gems'; -const VERSIONS_PATH = '/api/v1/versions'; +const VERSIONS_PATH = '/api/v1/versionsblabla'; const getHeaders = (): OutgoingHttpHeaders => { return { hostType: id }; @@ -55,7 +55,17 @@ export async function getDependency( try { versions = await fetch(dependency, registry, VERSIONS_PATH); } catch (err) { - logger.debug('version endpoint not errors or is not available'); + if (err.statusCode === 400) { + logger.debug({ registry }, 'version endpoint errors or is not available'); + logger.debug({ err }); + return null; + } + if (err.statusCode === 404) { + logger.debug({ registry }, 'version endpoint errors or is not available'); + logger.debug({ err }); + return null; + } + throw err; } if (versions.length === 0 && info.version) { From a67a5450ca5fbd25f03d04caadbacaa3e75952e0 Mon Sep 17 00:00:00 2001 From: Henry Sachs Date: Mon, 21 Sep 2020 14:04:21 +0000 Subject: [PATCH 07/12] chore: add henry sachs to contributors --- package.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/package.json b/package.json index a6ef851f78a36e..bd57ccb7ffeeab 100644 --- a/package.json +++ b/package.json @@ -96,7 +96,8 @@ "Viral Ruparel ", "Vladimir Starkov ", "Mikhail Yakushin ", - "Sebastian Poxhofer " + "Sebastian Poxhofer ", + "Henry Sachs " ], "license": "AGPL-3.0", "bugs": { From 6e0506d71fc1338dc95cb6884bcd7d44a519d850 Mon Sep 17 00:00:00 2001 From: Henry Sachs Date: Mon, 21 Sep 2020 14:21:22 +0000 Subject: [PATCH 08/12] fix: only throw when its something else than 400 or 404 --- lib/datasource/rubygems/get.ts | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/lib/datasource/rubygems/get.ts b/lib/datasource/rubygems/get.ts index 481dff964e5416..36fa50d8be39fb 100644 --- a/lib/datasource/rubygems/get.ts +++ b/lib/datasource/rubygems/get.ts @@ -8,7 +8,7 @@ import { id } from './common'; const http = new Http(id); const INFO_PATH = '/api/v1/gems'; -const VERSIONS_PATH = '/api/v1/versionsblabla'; +const VERSIONS_PATH = '/api/v1/versions'; const getHeaders = (): OutgoingHttpHeaders => { return { hostType: id }; @@ -55,17 +55,12 @@ export async function getDependency( try { versions = await fetch(dependency, registry, VERSIONS_PATH); } catch (err) { - if (err.statusCode === 400) { + if (err.statusCode === 400 || err.statusCode === 404) { logger.debug({ registry }, 'version endpoint errors or is not available'); logger.debug({ err }); - return null; + } else { + throw err; } - if (err.statusCode === 404) { - logger.debug({ registry }, 'version endpoint errors or is not available'); - logger.debug({ err }); - return null; - } - throw err; } if (versions.length === 0 && info.version) { From 66def2862c00483fb9a2d54cc7cd507e64e9b4e9 Mon Sep 17 00:00:00 2001 From: henrysachs Date: Mon, 21 Sep 2020 16:43:33 +0200 Subject: [PATCH 09/12] Update lib/datasource/rubygems/get.ts Co-authored-by: Rhys Arkins --- lib/datasource/rubygems/get.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/datasource/rubygems/get.ts b/lib/datasource/rubygems/get.ts index 36fa50d8be39fb..c903e653d26a53 100644 --- a/lib/datasource/rubygems/get.ts +++ b/lib/datasource/rubygems/get.ts @@ -56,8 +56,7 @@ export async function getDependency( versions = await fetch(dependency, registry, VERSIONS_PATH); } catch (err) { if (err.statusCode === 400 || err.statusCode === 404) { - logger.debug({ registry }, 'version endpoint errors or is not available'); - logger.debug({ err }); + logger.debug({ registry }, 'versions endpoint returns error - falling back to info endpoint'); } else { throw err; } From 95cf3ea4c310d91015934886a1bb137caa39dff5 Mon Sep 17 00:00:00 2001 From: Henry Sachs Date: Tue, 22 Sep 2020 11:58:15 +0200 Subject: [PATCH 10/12] test: add test for fallback feature --- lib/datasource/rubygems/index.spec.ts | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/lib/datasource/rubygems/index.spec.ts b/lib/datasource/rubygems/index.spec.ts index b5f631b2c6071c..0c26f3ebcaa6e1 100644 --- a/lib/datasource/rubygems/index.spec.ts +++ b/lib/datasource/rubygems/index.spec.ts @@ -1,10 +1,10 @@ +import * as rubygems from '.'; import { getPkgReleases } from '..'; import * as httpMock from '../../../test/httpMock'; import * as rubyVersioning from '../../versioning/ruby'; +import { resetCache } from './get-rubygems-org'; import railsInfo from './__fixtures__/rails/info.json'; import railsVersions from './__fixtures__/rails/versions.json'; -import { resetCache } from './get-rubygems-org'; -import * as rubygems from '.'; const rubygemsOrgVersions = `created_at: 2017-03-27T04:38:13+00:00 --- @@ -167,5 +167,16 @@ describe('datasource/rubygems', () => { expect(await getPkgReleases(params)).toBeNull(); expect(httpMock.getTrace()).toMatchSnapshot(); }); + it('falls back to info when version request fails', async () => { + httpMock + .scope('https://thirdparty.com/') + .get('/api/v1/gems/rails.json') + .reply(200, railsInfo) + .get('/api/v1/versions/rails.json') + .reply(400, {}); + const res = await getPkgReleases(params); + expect(res.releases).toHaveLength(1); + expect(res.releases[0].version).toBe(railsInfo.version); + }); }); }); From 650105ddc60ca7106e65a5d7ab6220dacab7ed05 Mon Sep 17 00:00:00 2001 From: Henry Sachs Date: Tue, 22 Sep 2020 10:45:47 +0000 Subject: [PATCH 11/12] test: add test for unknow error case --- lib/datasource/rubygems/index.spec.ts | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/lib/datasource/rubygems/index.spec.ts b/lib/datasource/rubygems/index.spec.ts index 0c26f3ebcaa6e1..9f20e8d9358924 100644 --- a/lib/datasource/rubygems/index.spec.ts +++ b/lib/datasource/rubygems/index.spec.ts @@ -178,5 +178,14 @@ describe('datasource/rubygems', () => { expect(res.releases).toHaveLength(1); expect(res.releases[0].version).toBe(railsInfo.version); }); + it('errors when version request fails with anything other than 400 or 404', async () => { + httpMock + .scope('https://thirdparty.com/') + .get('/api/v1/gems/rails.json') + .reply(200, railsInfo) + .get('/api/v1/versions/rails.json') + .reply(500, {}); + expect(await getPkgReleases(params)).toBeNull(); + }); }); }); From 73074015ae2914937c7b803fb0d060de1ed00201 Mon Sep 17 00:00:00 2001 From: Henry Sachs Date: Tue, 22 Sep 2020 10:52:21 +0000 Subject: [PATCH 12/12] fix: eslint fixes and removed unused parameters --- lib/datasource/rubygems/get.ts | 8 ++++---- lib/datasource/rubygems/index.spec.ts | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/datasource/rubygems/get.ts b/lib/datasource/rubygems/get.ts index c903e653d26a53..ae8f002aecce3f 100644 --- a/lib/datasource/rubygems/get.ts +++ b/lib/datasource/rubygems/get.ts @@ -56,7 +56,10 @@ export async function getDependency( versions = await fetch(dependency, registry, VERSIONS_PATH); } catch (err) { if (err.statusCode === 400 || err.statusCode === 404) { - logger.debug({ registry }, 'versions endpoint returns error - falling back to info endpoint'); + logger.debug( + { registry }, + 'versions endpoint returns error - falling back to info endpoint' + ); } else { throw err; } @@ -68,9 +71,6 @@ export async function getDependency( { version: info.version, rubyPlatform: info.platform, - releaseTimestamp: null, - rubyVersion: null, - rubygemsVersion: '\u003e= 0', }, ]; } else { diff --git a/lib/datasource/rubygems/index.spec.ts b/lib/datasource/rubygems/index.spec.ts index 9f20e8d9358924..58321658af6871 100644 --- a/lib/datasource/rubygems/index.spec.ts +++ b/lib/datasource/rubygems/index.spec.ts @@ -1,10 +1,10 @@ -import * as rubygems from '.'; import { getPkgReleases } from '..'; import * as httpMock from '../../../test/httpMock'; import * as rubyVersioning from '../../versioning/ruby'; -import { resetCache } from './get-rubygems-org'; import railsInfo from './__fixtures__/rails/info.json'; import railsVersions from './__fixtures__/rails/versions.json'; +import { resetCache } from './get-rubygems-org'; +import * as rubygems from '.'; const rubygemsOrgVersions = `created_at: 2017-03-27T04:38:13+00:00 ---