From 0244f4e751d0779a24d8c534477fa610c2dbbcdc Mon Sep 17 00:00:00 2001 From: fangpengfei <1225751694@qq.com> Date: Wed, 19 Oct 2022 02:45:21 +0800 Subject: [PATCH] =?UTF-8?q?fix(update):=20=E4=BF=AE=E5=A4=8D=E8=85=BE?= =?UTF-8?q?=E8=AE=AF=E4=BA=91=E5=9B=BE=E5=BA=8A/Github=E5=9B=BE=E5=BA=8A?= =?UTF-8?q?=E4=B8=8A=E4=BC=A0=E9=97=AE=E9=A2=98=20=E5=9B=BE=E7=89=87?= =?UTF-8?q?=E4=B8=8A=E4=BC=A0=E5=A4=B1=E8=B4=A5=E6=97=B6=EF=BC=8C=E5=8F=96?= =?UTF-8?q?=E6=B6=88=E5=81=9C=E6=AD=A2=E8=BF=9B=E7=A8=8B=20=E6=96=B0?= =?UTF-8?q?=E5=A2=9E=E5=9B=BE=E7=89=87=E4=B8=8A=E4=BC=A0=E6=97=B6=E7=9A=84?= =?UTF-8?q?=E5=B9=B6=E5=8F=91=E6=95=B0concurrency=EF=BC=8C=E4=BD=BF?= =?UTF-8?q?=E7=94=A8Github=E5=9B=BE=E5=BA=8A=E6=97=B6=EF=BC=8C=E5=B9=B6?= =?UTF-8?q?=E5=8F=91=E9=97=AE=E9=A2=98=E4=B8=A5=E9=87=8D=EF=BC=8C=E5=BB=BA?= =?UTF-8?q?=E8=AE=AE=E8=AE=BE=E7=BD=AE=E4=B8=BA1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 7 ++++ config.js | 1 + lib/Downloader.js | 17 ++++++-- package.json | 2 +- util/imageBeds/cos.js | 6 +-- util/imageBeds/github.js | 9 ++-- util/imageBeds/oss.js | 4 +- util/imageBeds/qiniu.js | 8 ++-- util/imageBeds/upyun.js | 11 +++-- util/img2cdn.js | 88 ++++++++++++++++++++++++---------------- 10 files changed, 95 insertions(+), 58 deletions(-) diff --git a/README.md b/README.md index 2bbc231..c06653b 100644 --- a/README.md +++ b/README.md @@ -80,6 +80,7 @@ A downloader for articles from yuque（语雀知识库同步工具） "lastGeneratePath": "lastGeneratePath.log", "imgCdn": { "enabled": false, + "concurrency": 0, "imageBed": "qiniu", "host": "", "bucket": "", @@ -113,6 +114,7 @@ imgCdn 语雀图片转图床配置说明 | 参数名 | 含义 | 默认值 | |-----------|-----------------------------------------------------------------------------------------|---------| | enabled | 是否开启 | false | +| concurrency | 上传图片并发数, 0代表无限制，使用github图床时，并发问题严重，建议设置为1 | 0 | | imageBed | 选择将图片上传的图床
目前支持腾讯云(cos)、阿里云(oss)和七牛云(qiniu)，又拍云(upyun)，Github图床(github)
默认使用七牛云 | 'qiniu' | | host | 使用七牛云/又拍云图床时，需要指定CDN域名前缀 | | | bucket | 图床的bucket名称 | - | @@ -233,6 +235,11 @@ DEBUG=yuque-hexo.* yuque-hexo sync # Changelog +### v1.9.5 +- 修复腾讯云图床/Github图床上传问题 +- 图片上传失败时，取消停止进程 +- 新增图片上传时的并发数concurrency，使用Github图床时，并发问题严重，建议设置为1 + ### v1.9.4 - 🔥 新增GitHub图床和又拍云图床 diff --git a/config.js b/config.js index 71a1d09..b72ce1e 100644 --- a/config.js +++ b/config.js @@ -20,6 +20,7 @@ const defaultConfig = { onlyPublished: false, onlyPublic: false, imgCdn: { + concurrency: 0, enabled: false, imageBed: 'qiniu', host: '', diff --git a/lib/Downloader.js b/lib/Downloader.js index bf242bb..2ca45da 100644 --- a/lib/Downloader.js +++ b/lib/Downloader.js @@ -210,10 +210,21 @@ class Downloader { const { _cachedArticles, postBasicPath } = this; mkdirp.sync(postBasicPath); out.info(`create posts directory (if it not exists): ${postBasicPath}`); - const promiseList = _cachedArticles.map(async item => { - return await this.generatePost(item); + const promiseList = _cachedArticles.map(item => { + return async () => { + return await this.generatePost(item); + }; + }); + // 并发数 + const concurrency = this.config.imgCdn.concurrency || promiseList.length; + const queue = new Queue({ concurrency }); + queue.push(...promiseList); + await new Promise((resolve, reject) => { + queue.start(function(err) { + if (err) return reject(err); + resolve(); + }); }); - await Promise.all(promiseList); } // 文章下载 => 增量更新文章到缓存 json 文件 => 全量生成 markdown 文章 diff --git a/package.json b/package.json index 9526806..86b9f17 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "yuque-hexo", - "version": "1.9.4", + "version": "1.9.5", "description": "A downloader for articles from yuque", "main": "index.js", "scripts": { diff --git a/util/imageBeds/cos.js b/util/imageBeds/cos.js index 7f65eb6..0c2d424 100644 --- a/util/imageBeds/cos.js +++ b/util/imageBeds/cos.js @@ -3,7 +3,6 @@ // 腾讯云图床 const COS = require('cos-nodejs-sdk-v5'); const out = require('../../lib/out'); -const { transformRes } = require('../index'); const secretId = process.env.SECRET_ID; const secretKey = process.env.SECRET_KEY; @@ -40,7 +39,6 @@ class CosClient { }); return `https://${this.config.bucket}.cos.${this.config.region}.myqcloud.com/${this.config.prefixKey}/${fileName}`; } catch (e) { - out.warn(`检查图片信息时出错: ${transformRes(e)}`); return ''; } } @@ -63,8 +61,8 @@ class CosClient { }); return `https://${res.Location}`; } catch (e) { - out.error(`上传图片失败，请检查: ${transformRes(e)}`); - process.exit(-1); + out.warn(`上传图片失败，请检查: ${e}`); + return ''; } } } diff --git a/util/imageBeds/github.js b/util/imageBeds/github.js index 78b1d9c..9926d5f 100644 --- a/util/imageBeds/github.js +++ b/util/imageBeds/github.js @@ -51,6 +51,10 @@ class GithubClient { Authorization: `token ${secretKey}`, }, }); + if (result.status === 409) { + out.warn('由于github并发问题，图片上传失败'); + return ''; + } if (result.status === 200 || result.status === 201) { if (this.config.host) { return `${this.config.host}/gh/${secretId}/${this.config.bucket}/${this.config.prefixKey}/${fileName}`; @@ -96,11 +100,8 @@ class GithubClient { const base64File = imgBuffer.toString('base64'); const imgUrl = await this._fetch('PUT', fileName, base64File); if (imgUrl) return imgUrl; - out.error('上传图片失败，请检查'); - process.exit(-1); } catch (e) { - out.error(`上传图片失败，请检查: ${transformRes(e)}`); - process.exit(-1); + out.warn(`上传图片失败，请检查: ${e}`); } } } diff --git a/util/imageBeds/oss.js b/util/imageBeds/oss.js index cf4c2fb..aeed5ea 100644 --- a/util/imageBeds/oss.js +++ b/util/imageBeds/oss.js @@ -57,8 +57,8 @@ class OssClient { const res = await this.imageBedInstance.put(`${this.config.prefixKey}/${fileName}`, imgBuffer); return res.url; } catch (e) { - out.error(`上传图片失败，请检查: ${transformRes(e)}`); - process.exit(-1); + out.warn(`上传图片失败，请检查: ${e}`); + return ''; } } } diff --git a/util/imageBeds/qiniu.js b/util/imageBeds/qiniu.js index d3d7df7..199149c 100644 --- a/util/imageBeds/qiniu.js +++ b/util/imageBeds/qiniu.js @@ -72,14 +72,14 @@ class QiniuClient { this.formUploader.put(this.uploadToken, `${this.config.prefixKey}/${fileName}`, imgBuffer, this.putExtra, (respErr, respBody, respInfo) => { if (respErr) { - out.error(`上传图片失败，请检查: ${transformRes(respErr)}`); - process.exit(-1); + out.warn(`上传图片失败，请检查: ${transformRes(respErr)}`); + resolve(''); } if (respInfo.statusCode === 200) { resolve(`${this.config.host}/${this.config.prefixKey}/${fileName}`); } else { - out.error(`上传图片失败，请检查: ${transformRes(respInfo)}`); - process.exit(-1); + out.warn(`上传图片失败，请检查: ${transformRes(respInfo)}`); + resolve(''); } }); }); diff --git a/util/imageBeds/upyun.js b/util/imageBeds/upyun.js index a6779ee..57b7b85 100644 --- a/util/imageBeds/upyun.js +++ b/util/imageBeds/upyun.js @@ -3,7 +3,6 @@ // 又拍云图床 const upyun = require('upyun'); const out = require('../../lib/out'); -const { transformRes } = require('../index'); const secretId = process.env.SECRET_ID; const secretKey = process.env.SECRET_KEY; @@ -47,7 +46,7 @@ class UPClient { } return ''; } catch (e) { - out.error(`上传图片失败，请检查: ${transformRes(e)}`); + out.warn(`上传图片失败，请检查: ${e}`); return ''; } } @@ -65,11 +64,11 @@ class UPClient { if (res) { return `${this.config.host}/${this.config.prefixKey}/${fileName}`; } - out.error('上传图片失败，请检查又拍云配置'); - process.exit(-1); + out.warn('上传图片失败，请检查又拍云配置'); + return ''; } catch (e) { - out.error(`上传图片失败，请检查: ${transformRes(e)}`); - process.exit(-1); + out.warn(`上传图片失败，请检查: ${e}`); + return ''; } } } diff --git a/util/img2cdn.js b/util/img2cdn.js index b929b5a..5fdf01c 100644 --- a/util/img2cdn.js +++ b/util/img2cdn.js @@ -5,6 +5,8 @@ const getEtag = require('../lib/qetag'); const config = require('../config'); const out = require('../lib/out'); const ImageBed = require('./imageBeds'); +const Queue = require('queue'); +const lodash = require('lodash'); const imageBed = config.imgCdn.enabled ? ImageBed.getInstance(config.imgCdn) : null; @@ -83,49 +85,67 @@ async function img2Cdn(article) { // 1。从文章中获取语雀的图片URL列表 const matchYuqueImgUrlList = article.body.match(imageUrlRegExp); if (!matchYuqueImgUrlList) return article; - const promiseList = matchYuqueImgUrlList.map(async matchYuqueImgUrl => { - // 获取真正的图片url - const yuqueImgUrl = getImgUrl(matchYuqueImgUrl); - // 2。将图片转成buffer - const imgBuffer = await img2Buffer(yuqueImgUrl); - if (!imgBuffer) { - return { - originalUrl: matchYuqueImgUrl, - yuqueRealImgUrl: yuqueImgUrl, - url: yuqueImgUrl, - }; - } - // 3。根据buffer文件生成唯一的hash文件名 - const fileName = await getFileName(imgBuffer, yuqueImgUrl); - try { - // 4。检查图床是否存在该文件 - let url = await imageBed.hasImage(fileName); - let exists = true; - // 5。如果图床已经存在，直接替换；如果图床不存在，则先上传到图床，再将原本的语雀url进行替换 - if (!url) { - url = await imageBed.uploadImg(imgBuffer, fileName); - exists = false; + const promiseList = matchYuqueImgUrlList.map(matchYuqueImgUrl => { + return async () => { + // 获取真正的图片url + const yuqueImgUrl = getImgUrl(matchYuqueImgUrl); + // 2。将图片转成buffer + const imgBuffer = await img2Buffer(yuqueImgUrl); + if (!imgBuffer) { + return { + originalUrl: matchYuqueImgUrl, + yuqueRealImgUrl: yuqueImgUrl, + url: yuqueImgUrl, + }; } - return { - originalUrl: matchYuqueImgUrl, - yuqueRealImgUrl: yuqueImgUrl, - url, - exists, - }; - } catch (e) { - out.error(`访问图床出错，请检查配置: ${e}`); - process.exit(-1); - } + // 3。根据buffer文件生成唯一的hash文件名 + const fileName = await getFileName(imgBuffer, yuqueImgUrl); + try { + // 4。检查图床是否存在该文件 + let url = await imageBed.hasImage(fileName); + let exists = true; + // 5。如果图床已经存在，直接替换；如果图床不存在，则先上传到图床，再将原本的语雀url进行替换 + if (!url) { + url = await imageBed.uploadImg(imgBuffer, fileName); + exists = false; + } + return { + originalUrl: matchYuqueImgUrl, + yuqueRealImgUrl: yuqueImgUrl, + url, + exists, + }; + } catch (e) { + out.error(`访问图床出错，请检查配置: ${e}`); + return { + yuqueRealImgUrl: yuqueImgUrl, + url: '', + }; + } + }; + }); + // 并发数 + const concurrency = config.imgCdn.concurrency || promiseList.length; + const queue = new Queue({ concurrency, results: [] }); + queue.push(...promiseList); + await new Promise(resolve => { + queue.start(() => { + resolve(); + }); }); - const urlList = await Promise.all(promiseList); + const _urlList = queue.results; + const urlList = lodash.flatten(_urlList); + urlList.forEach(function(url) { - if (url) { + if (url.url) { article.body = article.body.replace(url.originalUrl, `![](${url.url})`); if (url.exists) { out.info(`图片已存在 skip: ${url.url}`); } else { out.info(`replace ${url.yuqueRealImgUrl} to ${url.url}`); } + } else { + out.warn(`图片替换失败，将使用原url： ${url.yuqueRealImgUrl}`); } }); return article;