From 025d43b75202496e600a2eb35bc48ad85bcfc723 Mon Sep 17 00:00:00 2001 From: Glax Date: Thu, 12 Dec 2024 22:18:06 +0100 Subject: [PATCH 01/34] Fix duplicate job check. We were still adding duplicate jobs if not *every* field in the Manga matched. We now only compare publicationId. --- Tranga/Jobs/DownloadNewChapters.cs | 2 +- Tranga/Jobs/UpdateMetadata.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Tranga/Jobs/DownloadNewChapters.cs b/Tranga/Jobs/DownloadNewChapters.cs index 29c3e602..b07d46b1 100644 --- a/Tranga/Jobs/DownloadNewChapters.cs +++ b/Tranga/Jobs/DownloadNewChapters.cs @@ -54,6 +54,6 @@ public override bool Equals(object? obj) if (obj is not DownloadNewChapters otherJob) return false; return otherJob.mangaConnector == this.mangaConnector && - otherJob.manga.Equals(this.manga); + otherJob.manga.publicationId == this.manga.publicationId; } } \ No newline at end of file diff --git a/Tranga/Jobs/UpdateMetadata.cs b/Tranga/Jobs/UpdateMetadata.cs index 72b5a602..ac8bdd64 100644 --- a/Tranga/Jobs/UpdateMetadata.cs +++ b/Tranga/Jobs/UpdateMetadata.cs @@ -71,6 +71,6 @@ public override bool Equals(object? obj) if (obj is not UpdateMetadata otherJob) return false; return otherJob.mangaConnector == this.mangaConnector && - otherJob.manga.Equals(this.manga); + otherJob.manga.publicationId == this.manga.publicationId; } } \ No newline at end of file From 1e6a65c0fdafb836282cccbbd1a398e2d9dd9f5f Mon Sep 17 00:00:00 2001 From: Glax Date: Thu, 12 Dec 2024 22:33:13 +0100 Subject: [PATCH 02/34] Chapter volume and chapternumber as float instead of string. Possible fix #293 --- Tranga/Chapter.cs | 45 ++++++-------- Tranga/Jobs/JobBoss.cs | 4 +- Tranga/MangaConnectors/AsuraToon.cs | 9 ++- Tranga/MangaConnectors/Bato.cs | 9 ++- Tranga/MangaConnectors/MangaConnector.cs | 76 +----------------------- Tranga/MangaConnectors/MangaDex.cs | 11 +++- Tranga/MangaConnectors/MangaHere.cs | 10 +++- Tranga/MangaConnectors/MangaKatana.cs | 9 ++- Tranga/MangaConnectors/MangaLife.cs | 9 ++- Tranga/MangaConnectors/Manganato.cs | 9 ++- Tranga/MangaConnectors/Mangasee.cs | 9 ++- Tranga/MangaConnectors/Mangaworld.cs | 18 +++++- Tranga/MangaConnectors/ManhuaPlus.cs | 9 ++- 13 files changed, 113 insertions(+), 114 deletions(-) diff --git a/Tranga/Chapter.cs b/Tranga/Chapter.cs index 54090964..7622bd78 100644 --- a/Tranga/Chapter.cs +++ b/Tranga/Chapter.cs @@ -14,8 +14,8 @@ namespace Tranga; // ReSharper disable once MemberCanBePrivate.Global public Manga parentManga { get; } public string? name { get; } - public string volumeNumber { get; } - public string chapterNumber { get; } + public float volumeNumber { get; } + public float chapterNumber { get; } public string url { get; } // ReSharper disable once MemberCanBePrivate.Global public string fileName { get; } @@ -23,13 +23,19 @@ namespace Tranga; private static readonly Regex LegalCharacters = new (@"([A-z]*[0-9]* *\.*-*,*\]*\[*'*\'*\)*\(*~*!*)*"); private static readonly Regex IllegalStrings = new(@"(Vol(ume)?|Ch(apter)?)\.?", RegexOptions.IgnoreCase); - private static readonly Regex Digits = new(@"[0-9\.]*"); + public Chapter(Manga parentManga, string? name, string? volumeNumber, string chapterNumber, string url, string? id = null) + : this(parentManga, name, float.Parse(volumeNumber??"0", GlobalBase.numberFormatDecimalPoint), + float.Parse(chapterNumber, GlobalBase.numberFormatDecimalPoint), url, id) + { + } + + public Chapter(Manga parentManga, string? name, float? volumeNumber, float chapterNumber, string url, string? id = null) { this.parentManga = parentManga; this.name = name; - this.volumeNumber = volumeNumber is not null ? string.Concat(Digits.Matches(volumeNumber).Select(x => x.Value)) : "0"; - this.chapterNumber = string.Concat(Digits.Matches(chapterNumber).Select(x => x.Value)); + this.volumeNumber = volumeNumber??0; + this.chapterNumber = chapterNumber; this.url = url; this.id = id; @@ -60,26 +66,12 @@ public int CompareTo(object? obj) { if(obj is not Chapter otherChapter) throw new ArgumentException($"{obj} can not be compared to {this}"); - - if (float.TryParse(volumeNumber, GlobalBase.numberFormatDecimalPoint, out float volumeNumberFloat) && - float.TryParse(chapterNumber, GlobalBase.numberFormatDecimalPoint, out float chapterNumberFloat) && - float.TryParse(otherChapter.volumeNumber, GlobalBase.numberFormatDecimalPoint, - out float otherVolumeNumberFloat) && - float.TryParse(otherChapter.chapterNumber, GlobalBase.numberFormatDecimalPoint, - out float otherChapterNumberFloat)) + return volumeNumber.CompareTo(otherChapter.volumeNumber) switch { - return volumeNumberFloat.CompareTo(otherVolumeNumberFloat) switch - { - <0 => -1, - >0 => 1, - _ => chapterNumberFloat.CompareTo(otherChapterNumberFloat) - }; - } - else throw new FormatException($"Value could not be parsed.\n" + - $"\tVolumeNumber: '{volumeNumber}' ChapterNumber: '{chapterNumber}'\n" + - $"\tOther-VolumeNumber: '{otherChapter.volumeNumber}' Other-ChapterNumber: '{otherChapter.chapterNumber}'\n" + - $"\t{this}\n" + - $"\t{otherChapter}"); + <0 => -1, + >0 => 1, + _ => chapterNumber.CompareTo(otherChapter.chapterNumber) + }; } /// @@ -111,9 +103,10 @@ internal bool CheckChapterIsDownloaded() { Match m = volChRex.Match(archive.Name); if (m.Groups[1].Success) - return m.Groups[1].Value == t.volumeNumber && m.Groups[2].Value == t.chapterNumber; + return m.Groups[1].Value == t.volumeNumber.ToString(GlobalBase.numberFormatDecimalPoint) && + m.Groups[2].Value == t.chapterNumber.ToString(GlobalBase.numberFormatDecimalPoint); else - return m.Groups[2].Value == t.chapterNumber; + return m.Groups[2].Value == t.chapterNumber.ToString(GlobalBase.numberFormatDecimalPoint); }); } diff --git a/Tranga/Jobs/JobBoss.cs b/Tranga/Jobs/JobBoss.cs index 9a7e7c87..781b978c 100644 --- a/Tranga/Jobs/JobBoss.cs +++ b/Tranga/Jobs/JobBoss.cs @@ -70,7 +70,7 @@ public void RemoveJobs(IEnumerable jobsToRemove) RemoveJob(job); } - public IEnumerable GetJobsLike(string? connectorName = null, string? internalId = null, string? chapterNumber = null) + public IEnumerable GetJobsLike(string? connectorName = null, string? internalId = null, float? chapterNumber = null) { IEnumerable ret = this.jobs; if (connectorName is not null) @@ -82,7 +82,7 @@ public IEnumerable GetJobsLike(string? connectorName = null, string? intern if (jjob is not DownloadChapter job) return false; return job.chapter.parentManga.internalId == internalId && - job.chapter.chapterNumber == chapterNumber; + job.chapter.chapterNumber.Equals(chapterNumber); }); else if (internalId is not null) ret = ret.Where(jjob => diff --git a/Tranga/MangaConnectors/AsuraToon.cs b/Tranga/MangaConnectors/AsuraToon.cs index bf3bdaa1..a954a95d 100644 --- a/Tranga/MangaConnectors/AsuraToon.cs +++ b/Tranga/MangaConnectors/AsuraToon.cs @@ -155,7 +155,14 @@ private List ParseChaptersFromHtml(Manga manga, string mangaUrl) string chapterNumber = match.Groups[1].Value; string? chapterName = match.Groups[2].Success && match.Groups[2].Length > 1 ? match.Groups[2].Value : null; string url = $"https://asuracomic.net/series/{chapterUrl}"; - ret.Add(new Chapter(manga, chapterName, null, chapterNumber, url)); + try + { + ret.Add(new Chapter(manga, chapterName, null, chapterNumber, url)); + } + catch (Exception e) + { + Log($"Failed to load chapter {chapterNumber}: {e.Message}"); + } } return ret; diff --git a/Tranga/MangaConnectors/Bato.cs b/Tranga/MangaConnectors/Bato.cs index c666a36e..1e00ca7b 100644 --- a/Tranga/MangaConnectors/Bato.cs +++ b/Tranga/MangaConnectors/Bato.cs @@ -163,7 +163,14 @@ private List ParseChaptersFromHtml(Manga manga, string mangaUrl) string chapterNumber = match.Groups[3].Value; string chapterName = chapterNumber; string url = $"https://bato.to{chapterUrl}?load=2"; - ret.Add(new Chapter(manga, chapterName, volumeNumber, chapterNumber, url)); + try + { + ret.Add(new Chapter(manga, chapterName, volumeNumber, chapterNumber, url)); + } + catch (Exception e) + { + Log($"Failed to load chapter {chapterNumber}: {e.Message}"); + } } return ret; diff --git a/Tranga/MangaConnectors/MangaConnector.cs b/Tranga/MangaConnectors/MangaConnector.cs index f9fbf69b..c98bfad2 100644 --- a/Tranga/MangaConnectors/MangaConnector.cs +++ b/Tranga/MangaConnectors/MangaConnector.cs @@ -60,8 +60,7 @@ public Chapter[] GetNewChapters(Manga manga, string language = "en") return Array.Empty(); Log($"Checking for duplicates {manga}"); - List newChaptersList = allChapters.Where(nChapter => float.TryParse(nChapter.chapterNumber, numberFormatDecimalPoint, out float chapterNumber) - && chapterNumber > manga.ignoreChaptersBelow + List newChaptersList = allChapters.Where(nChapter => nChapter.chapterNumber > manga.ignoreChaptersBelow && !nChapter.CheckChapterIsDownloaded()).ToList(); Log($"{newChaptersList.Count} new chapters. {manga}"); try @@ -79,79 +78,6 @@ public Chapter[] GetNewChapters(Manga manga, string language = "en") return newChaptersList.ToArray(); } - - public Chapter[] SelectChapters(Manga manga, string searchTerm, string? language = null) - { - Chapter[] availableChapters = this.GetChapters(manga, language??"en"); - Regex volumeRegex = new ("((v(ol)*(olume)*){1} *([0-9]+(-[0-9]+)?){1})", RegexOptions.IgnoreCase); - Regex chapterRegex = new ("((c(h)*(hapter)*){1} *([0-9]+(-[0-9]+)?){1})", RegexOptions.IgnoreCase); - Regex singleResultRegex = new("([0-9]+)", RegexOptions.IgnoreCase); - Regex rangeResultRegex = new("([0-9]+(-[0-9]+))", RegexOptions.IgnoreCase); - Regex allRegex = new("a(ll)?", RegexOptions.IgnoreCase); - if (volumeRegex.IsMatch(searchTerm) && chapterRegex.IsMatch(searchTerm)) - { - string volume = singleResultRegex.Match(volumeRegex.Match(searchTerm).Value).Value; - string chapter = singleResultRegex.Match(chapterRegex.Match(searchTerm).Value).Value; - return availableChapters.Where(aCh => aCh.volumeNumber is not null && - aCh.volumeNumber.Equals(volume, StringComparison.InvariantCultureIgnoreCase) && - aCh.chapterNumber.Equals(chapter, StringComparison.InvariantCultureIgnoreCase)) - .ToArray(); - } - else if (volumeRegex.IsMatch(searchTerm)) - { - string volume = volumeRegex.Match(searchTerm).Value; - if (rangeResultRegex.IsMatch(volume)) - { - string range = rangeResultRegex.Match(volume).Value; - int start = Convert.ToInt32(range.Split('-')[0]); - int end = Convert.ToInt32(range.Split('-')[1]); - return availableChapters.Where(aCh => aCh.volumeNumber is not null && - Convert.ToInt32(aCh.volumeNumber) >= start && - Convert.ToInt32(aCh.volumeNumber) <= end).ToArray(); - } - else if (singleResultRegex.IsMatch(volume)) - { - string volumeNumber = singleResultRegex.Match(volume).Value; - return availableChapters.Where(aCh => - aCh.volumeNumber is not null && - aCh.volumeNumber.Equals(volumeNumber, StringComparison.InvariantCultureIgnoreCase)).ToArray(); - } - - } - else if (chapterRegex.IsMatch(searchTerm)) - { - string chapter = chapterRegex.Match(searchTerm).Value; - if (rangeResultRegex.IsMatch(chapter)) - { - string range = rangeResultRegex.Match(chapter).Value; - int start = Convert.ToInt32(range.Split('-')[0]); - int end = Convert.ToInt32(range.Split('-')[1]); - return availableChapters.Where(aCh => Convert.ToInt32(aCh.chapterNumber) >= start && - Convert.ToInt32(aCh.chapterNumber) <= end).ToArray(); - } - else if (singleResultRegex.IsMatch(chapter)) - { - string chapterNumber = singleResultRegex.Match(chapter).Value; - return availableChapters.Where(aCh => - aCh.chapterNumber.Equals(chapterNumber, StringComparison.InvariantCultureIgnoreCase)).ToArray(); - } - } - else - { - if (rangeResultRegex.IsMatch(searchTerm)) - { - int start = Convert.ToInt32(searchTerm.Split('-')[0]); - int end = Convert.ToInt32(searchTerm.Split('-')[1]); - return availableChapters[start..(end + 1)]; - } - else if(singleResultRegex.IsMatch(searchTerm)) - return new [] { availableChapters[Convert.ToInt32(searchTerm)] }; - else if (allRegex.IsMatch(searchTerm)) - return availableChapters; - } - - return Array.Empty(); - } public abstract HttpStatusCode DownloadChapter(Chapter chapter, ProgressToken? progressToken = null); diff --git a/Tranga/MangaConnectors/MangaDex.cs b/Tranga/MangaConnectors/MangaDex.cs index 1dbb1d06..8b23c2d1 100644 --- a/Tranga/MangaConnectors/MangaDex.cs +++ b/Tranga/MangaConnectors/MangaDex.cs @@ -246,8 +246,17 @@ public override Chapter[] GetChapters(Manga manga, string language="en") continue; } - if(chapterNum is not "null" && !chapters.Any(chp => chp.volumeNumber.Equals(volume) && chp.chapterNumber.Equals(chapterNum))) + try + { + if(!chapters.Any(chp => + chp.volumeNumber.Equals(float.Parse(volume??"0", numberFormatDecimalPoint)) && + chp.chapterNumber.Equals(float.Parse(chapterNum, numberFormatDecimalPoint)))) chapters.Add(new Chapter(manga, title, volume, chapterNum, chapterId, chapterId)); + } + catch (Exception e) + { + Log($"Failed to load chapter {chapterNum}: {e.Message}"); + } } } diff --git a/Tranga/MangaConnectors/MangaHere.cs b/Tranga/MangaConnectors/MangaHere.cs index c14b6975..18c04d60 100644 --- a/Tranga/MangaConnectors/MangaHere.cs +++ b/Tranga/MangaConnectors/MangaHere.cs @@ -129,7 +129,15 @@ public override Chapter[] GetChapters(Manga manga, string language="en") string volumeNumber = rexMatch.Groups[1].Value == "TBD" ? "0" : rexMatch.Groups[1].Value; string chapterNumber = rexMatch.Groups[2].Value; string fullUrl = $"https://www.mangahere.cc{url}"; - chapters.Add(new Chapter(manga, "", volumeNumber, chapterNumber, fullUrl)); + + try + { + chapters.Add(new Chapter(manga, "", volumeNumber, chapterNumber, fullUrl)); + } + catch (Exception e) + { + Log($"Failed to load chapter {chapterNumber}: {e.Message}"); + } } //Return Chapters ordered by Chapter-Number Log($"Got {chapters.Count} chapters. {manga}"); diff --git a/Tranga/MangaConnectors/MangaKatana.cs b/Tranga/MangaConnectors/MangaKatana.cs index 5142c3b1..8cd0c651 100644 --- a/Tranga/MangaConnectors/MangaKatana.cs +++ b/Tranga/MangaConnectors/MangaKatana.cs @@ -186,7 +186,14 @@ private List ParseChaptersFromHtml(Manga manga, string mangaUrl) string? volumeNumber = volumeRex.IsMatch(url) ? volumeRex.Match(url).Groups[1].Value : null; string chapterNumber = chapterNumRex.Match(url).Groups[1].Value; string chapterName = chapterNameRex.Match(fullString).Groups[1].Value; - ret.Add(new Chapter(manga, chapterName, volumeNumber, chapterNumber, url)); + try + { + ret.Add(new Chapter(manga, chapterName, volumeNumber, chapterNumber, url)); + } + catch (Exception e) + { + Log($"Failed to load chapter {chapterNumber}: {e.Message}"); + } } return ret; diff --git a/Tranga/MangaConnectors/MangaLife.cs b/Tranga/MangaConnectors/MangaLife.cs index a01a09d1..66f2d209 100644 --- a/Tranga/MangaConnectors/MangaLife.cs +++ b/Tranga/MangaConnectors/MangaLife.cs @@ -152,7 +152,14 @@ public override Chapter[] GetChapters(Manga manga, string language="en") string chapterNumber = rexMatch.Groups[1].Value; string fullUrl = $"https://manga4life.com{url}"; fullUrl = fullUrl.Replace(Regex.Match(url,"(-page-[0-9])").Value,""); - chapters.Add(new Chapter(manga, "", volumeNumber, chapterNumber, fullUrl)); + try + { + chapters.Add(new Chapter(manga, "", volumeNumber, chapterNumber, fullUrl)); + } + catch (Exception e) + { + Log($"Failed to load chapter {chapterNumber}: {e.Message}"); + } } //Return Chapters ordered by Chapter-Number Log($"Got {chapters.Count} chapters. {manga}"); diff --git a/Tranga/MangaConnectors/Manganato.cs b/Tranga/MangaConnectors/Manganato.cs index c3b337be..7d794142 100644 --- a/Tranga/MangaConnectors/Manganato.cs +++ b/Tranga/MangaConnectors/Manganato.cs @@ -181,7 +181,14 @@ private List ParseChaptersFromHtml(Manga manga, HtmlDocument document) string? volumeNumber = volRex.IsMatch(fullString) ? volRex.Match(fullString).Groups[1].Value : null; string chapterNumber = chapterRex.Match(url).Groups[1].Value; string chapterName = nameRex.Match(fullString).Groups[3].Value; - ret.Add(new Chapter(manga, chapterName, volumeNumber, chapterNumber, url)); + try + { + ret.Add(new Chapter(manga, chapterName, volumeNumber, chapterNumber, url)); + } + catch (Exception e) + { + Log($"Failed to load chapter {chapterNumber}: {e.Message}"); + } } ret.Reverse(); return ret; diff --git a/Tranga/MangaConnectors/Mangasee.cs b/Tranga/MangaConnectors/Mangasee.cs index 78b5271c..f912f6df 100644 --- a/Tranga/MangaConnectors/Mangasee.cs +++ b/Tranga/MangaConnectors/Mangasee.cs @@ -176,7 +176,14 @@ public override Chapter[] GetChapters(Manga manga, string language="en") string chapterNumber = m.Groups[1].Value; string chapterUrl = Regex.Replace(url, @"-page-[0-9]+(\.html)", ".html"); - chapters.Add(new Chapter(manga, "", volumeNumber, chapterNumber, chapterUrl)); + try + { + chapters.Add(new Chapter(manga, "", volumeNumber, chapterNumber, chapterUrl)); + } + catch (Exception e) + { + Log($"Failed to load chapter {chapterNumber}: {e.Message}"); + } } //Return Chapters ordered by Chapter-Number diff --git a/Tranga/MangaConnectors/Mangaworld.cs b/Tranga/MangaConnectors/Mangaworld.cs index 71f5581d..23522b6e 100644 --- a/Tranga/MangaConnectors/Mangaworld.cs +++ b/Tranga/MangaConnectors/Mangaworld.cs @@ -163,7 +163,14 @@ private List ParseChaptersFromHtml(Manga manga, HtmlDocument document) string number = chapterRex.Match(chNode.SelectSingleNode("a").SelectSingleNode("span").InnerText).Groups[1].Value; string url = chNode.SelectSingleNode("a").GetAttributeValue("href", ""); string id = idRex.Match(chNode.SelectSingleNode("a").GetAttributeValue("href", "")).Groups[1].Value; - ret.Add(new Chapter(manga, null, volume, number, url, id)); + try + { + ret.Add(new Chapter(manga, null, volume, number, url, id)); + } + catch (Exception e) + { + Log($"Failed to load chapter {number}: {e.Message}"); + } } } } @@ -174,7 +181,14 @@ private List ParseChaptersFromHtml(Manga manga, HtmlDocument document) string number = chapterRex.Match(chNode.SelectSingleNode("a").SelectSingleNode("span").InnerText).Groups[1].Value; string url = chNode.SelectSingleNode("a").GetAttributeValue("href", ""); string id = idRex.Match(chNode.SelectSingleNode("a").GetAttributeValue("href", "")).Groups[1].Value; - ret.Add(new Chapter(manga, null, null, number, url, id)); + try + { + ret.Add(new Chapter(manga, null, null, number, url, id)); + } + catch (Exception e) + { + Log($"Failed to load chapter {number}: {e.Message}"); + } } } diff --git a/Tranga/MangaConnectors/ManhuaPlus.cs b/Tranga/MangaConnectors/ManhuaPlus.cs index d66fede4..28bbad24 100644 --- a/Tranga/MangaConnectors/ManhuaPlus.cs +++ b/Tranga/MangaConnectors/ManhuaPlus.cs @@ -155,7 +155,14 @@ public override Chapter[] GetChapters(Manga manga, string language="en") string volumeNumber = "1"; string chapterNumber = rexMatch.Groups[1].Value; string fullUrl = url; - chapters.Add(new Chapter(manga, "", volumeNumber, chapterNumber, fullUrl)); + try + { + chapters.Add(new Chapter(manga, "", volumeNumber, chapterNumber, fullUrl)); + } + catch (Exception e) + { + Log($"Failed to load chapter {chapterNumber}: {e.Message}"); + } } //Return Chapters ordered by Chapter-Number Log($"Got {chapters.Count} chapters. {manga}"); From 428d6e13d1bb7b590f6746f05ef46d109365af5b Mon Sep 17 00:00:00 2001 From: Glax Date: Thu, 12 Dec 2024 22:41:28 +0100 Subject: [PATCH 03/34] Fix UpdateJobFile with oldFile: oldFilePath was fullname, not relative --- Tranga/Jobs/JobBoss.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tranga/Jobs/JobBoss.cs b/Tranga/Jobs/JobBoss.cs index 781b978c..b1b38dd8 100644 --- a/Tranga/Jobs/JobBoss.cs +++ b/Tranga/Jobs/JobBoss.cs @@ -197,7 +197,7 @@ private void LoadJobsList(HashSet connectors) internal void UpdateJobFile(Job job, string? oldFile = null) { string newJobFilePath = Path.Join(TrangaSettings.jobsFolderPath, $"{job.id}.json"); - string oldFilePath = Path.Join(TrangaSettings.jobsFolderPath, oldFile??$"{job.id}.json"); + string oldFilePath = oldFile??Path.Join(TrangaSettings.jobsFolderPath, $"{job.id}.json"); //Delete old file if (File.Exists(oldFilePath)) From e3bd7620aa97c13b5d4b0534e5913478c489f424 Mon Sep 17 00:00:00 2001 From: Glax Date: Fri, 13 Dec 2024 18:53:25 +0100 Subject: [PATCH 04/34] Fix #296 AsuraToon AsuraComic does not use Static sites, use Chromium instead. Make Puppeteer spam less logs --- Tranga/MangaConnectors/AsuraToon.cs | 2 +- Tranga/MangaConnectors/ChromiumDownloadClient.cs | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/Tranga/MangaConnectors/AsuraToon.cs b/Tranga/MangaConnectors/AsuraToon.cs index a954a95d..b19334a4 100644 --- a/Tranga/MangaConnectors/AsuraToon.cs +++ b/Tranga/MangaConnectors/AsuraToon.cs @@ -10,7 +10,7 @@ public class AsuraToon : MangaConnector public AsuraToon(GlobalBase clone) : base(clone, "AsuraToon", ["en"]) { - this.downloadClient = new HttpDownloadClient(clone); + this.downloadClient = new ChromiumDownloadClient(clone); } public override Manga[] GetManga(string publicationTitle = "") diff --git a/Tranga/MangaConnectors/ChromiumDownloadClient.cs b/Tranga/MangaConnectors/ChromiumDownloadClient.cs index f48b2ed9..3b5e63de 100644 --- a/Tranga/MangaConnectors/ChromiumDownloadClient.cs +++ b/Tranga/MangaConnectors/ChromiumDownloadClient.cs @@ -43,6 +43,8 @@ public Logger(Logging.Logger? logger) : base(logger) { } public void Log(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func formatter) { + if (logLevel <= LogLevel.Information) + return; logger?.WriteLine("Puppeteer", formatter.Invoke(state, exception)); } From 5a770c8e9f3762add63b4a56cc050140330e46d6 Mon Sep 17 00:00:00 2001 From: Alessandro Benetton Date: Fri, 13 Dec 2024 23:41:54 +0100 Subject: [PATCH 05/34] [feature/weebcentral] Working search --- Tranga/MangaConnectors/WeebCentral.cs | 232 ++++++++++++++++++++++++++ Tranga/Tranga.cs | 1 + 2 files changed, 233 insertions(+) create mode 100644 Tranga/MangaConnectors/WeebCentral.cs diff --git a/Tranga/MangaConnectors/WeebCentral.cs b/Tranga/MangaConnectors/WeebCentral.cs new file mode 100644 index 00000000..3fc6c6d6 --- /dev/null +++ b/Tranga/MangaConnectors/WeebCentral.cs @@ -0,0 +1,232 @@ +using System.Net; +using System.Text.RegularExpressions; +using System.Xml.Linq; +using HtmlAgilityPack; +using Soenneker.Utils.String.NeedlemanWunsch; +using Tranga.Jobs; + +namespace Tranga.MangaConnectors; + +public class Weebcentral : MangaConnector +{ + private readonly string[] _filterWords = + { "a", "the", "of", "as", "to", "no", "for", "on", "with", "be", "and", "in", "wa", "at", "be", "ni" }; + + private readonly string baseURL = "https://weebcentral.com"; + + public Weebcentral(GlobalBase clone) : base(clone, "Weebcentral", ["en"]) + { + downloadClient = new ChromiumDownloadClient(clone); + } + + public override Manga[] GetManga(string publicationTitle = "") + { + Log($"Searching Publications. Term=\"{publicationTitle}\""); + const int limit = 32; //How many values we want returned at once + var offset = 0; //"Page" + var requestUrl = + $"{baseURL}/search/data?limit={limit}&offset={offset}&text={publicationTitle}&sort=Best+Match&order=Ascending&official=Any&display_mode=Minimal%20Display"; + var requestResult = + downloadClient.MakeRequest(requestUrl, RequestType.Default); + if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300 || + requestResult.htmlDocument == null) + { + Log($"Failed to retrieve search: {requestResult.statusCode}"); + return []; + } + + var publications = ParsePublicationsFromHtml(requestResult.htmlDocument); + Log($"Retrieved {publications.Length} publications. Term=\"{publicationTitle}\""); + return publications; + } + + /* + * Title: document.DocumentNode.SelectNodes("/html/body/article/section/div/a[@class='link link-hover']").Select(elem => elem.InnerText).ToList() + * URL: document.DocumentNode.SelectNodes("/html/body/article/a[@class='link link-hover']").Select(elem => elem.GetAttributeValue("href", "")).ToList() + */ + + private Manga[] ParsePublicationsFromHtml(HtmlDocument document) + { + if (document.DocumentNode.SelectNodes("//article") == null) + return Array.Empty(); + + var urls = document.DocumentNode.SelectNodes("/html/body/article/a[@class='link link-hover']") + .Select(elem => elem.GetAttributeValue("href", "")).ToList(); + + HashSet ret = new(); + foreach (var url in urls) + { + var manga = GetMangaFromUrl(url); + if (manga is not null) + ret.Add((Manga)manga); + } + + return ret.ToArray(); + } + + public override Manga? GetMangaFromUrl(string url) + { + Regex publicationIdRex = new(@"https:\/\/weebcentral\.com\/series\/(\w*)\/(.*)"); + var publicationId = publicationIdRex.Match(url).Groups[1].Value; + + var requestResult = downloadClient.MakeRequest(url, RequestType.MangaInfo); + if ((int)requestResult.statusCode < 300 && (int)requestResult.statusCode >= 200 && + requestResult.htmlDocument is not null) + return ParseSinglePublicationFromHtml(requestResult.htmlDocument, publicationId, url); + return null; + } + + private Manga ParseSinglePublicationFromHtml(HtmlDocument document, string publicationId, string websiteUrl) + { + var posterNode = + document.DocumentNode.SelectSingleNode("//section[@class='flex items-center justify-center']/picture/img"); + var posterUrl = posterNode?.GetAttributeValue("src", "") ?? ""; + var coverFileNameInCache = SaveCoverImageToCache(posterUrl, publicationId, RequestType.MangaCover); + + var titleNode = document.DocumentNode.SelectSingleNode("//section/h1"); + var sortName = titleNode?.InnerText ?? "Undefined"; + + HtmlNode[] authorsNodes = + document.DocumentNode.SelectNodes("//ul/li[strong/text() = 'Author(s): ']/span")?.ToArray() ?? []; + var authors = authorsNodes.Select(n => n.InnerText).ToList(); + + HtmlNode[] genreNodes = document.DocumentNode.SelectNodes("//ul/li[strong/text() = 'Tags(s): ']/span")?.ToArray() ?? []; + HashSet tags = genreNodes.Select(n => n.InnerText).ToHashSet(); + + HtmlNode statusNode = document.DocumentNode.SelectSingleNode("//ul/li[strong/text() = 'Status: ']/a"); + var status = statusNode?.InnerText ?? ""; + Log("unable to parse status"); + var releaseStatus = Manga.ReleaseStatusByte.Unreleased; + switch (status.ToLower()) + { + case "cancelled": releaseStatus = Manga.ReleaseStatusByte.Cancelled; break; + case "hiatus": releaseStatus = Manga.ReleaseStatusByte.OnHiatus; break; + case "complete": releaseStatus = Manga.ReleaseStatusByte.Completed; break; + case "ongoing": releaseStatus = Manga.ReleaseStatusByte.Continuing; break; + } + + var yearNode = document.DocumentNode.SelectSingleNode("//ul/li[strong/text() = 'Released: ']/span"); + var year = Convert.ToInt32(yearNode?.InnerText ?? "0"); + + var descriptionNode = document.DocumentNode.SelectSingleNode("//ul/li[strong/text() = 'Description']/p"); + var description = descriptionNode?.InnerText ?? "Undefined"; + + HtmlNode[] altTitleNodes = document.DocumentNode.SelectNodes("//ul/li[strong/text() = 'Associated Name(s)']/ul/li")?.ToArray() ?? []; + Dictionary altTitles = new(), links = new(); + for(int i = 0; i < altTitleNodes.Length; i++) + altTitles.Add(i.ToString(), altTitleNodes[i].InnerText); + + var originalLanguage = ""; + + Manga manga = new(sortName, authors.ToList(), description, altTitles, tags.ToArray(), posterUrl, + coverFileNameInCache, links, + year, originalLanguage, publicationId, releaseStatus, websiteUrl); + AddMangaToCache(manga); + return manga; + } + + private string ToFilteredString(string input) + { + return string.Join(' ', input.ToLower().Split(' ').Where(word => _filterWords.Contains(word) == false)); + } + + private SearchResult[] FilteredResults(string publicationTitle, SearchResult[] unfilteredSearchResults) + { + Dictionary similarity = new(); + foreach (var sr in unfilteredSearchResults) + { + List scores = new(); + var filteredPublicationString = ToFilteredString(publicationTitle); + var filteredSString = ToFilteredString(sr.s); + scores.Add(NeedlemanWunschStringUtil.CalculateSimilarity(filteredSString, filteredPublicationString)); + foreach (var srA in sr.a) + { + var filteredAString = ToFilteredString(srA); + scores.Add(NeedlemanWunschStringUtil.CalculateSimilarity(filteredAString, filteredPublicationString)); + } + + similarity.Add(sr, scores.Sum() / scores.Count); + } + + var ret = similarity.OrderBy(s => s.Value).Take(10).Select(s => s.Key).ToList(); + return ret.ToArray(); + } + + public override Manga? GetMangaFromId(string publicationId) + { + return GetMangaFromUrl($"https://mangasee123.com/manga/{publicationId}"); + } + + public override Chapter[] GetChapters(Manga manga, string language = "en") + { + Log($"Getting chapters {manga}"); + try + { + var doc = XDocument.Load($"https://mangasee123.com/rss/{manga.publicationId}.xml"); + var chapterItems = doc.Descendants("item").ToArray(); + List chapters = new(); + Regex chVolRex = new(@".*chapter-([0-9\.]+)(?:-index-([0-9\.]+))?.*"); + foreach (var chapter in chapterItems) + { + var url = chapter.Descendants("link").First().Value; + var m = chVolRex.Match(url); + var volumeNumber = m.Groups[2].Success ? m.Groups[2].Value : "1"; + var chapterNumber = m.Groups[1].Value; + + var chapterUrl = Regex.Replace(url, @"-page-[0-9]+(\.html)", ".html"); + chapters.Add(new Chapter(manga, "", volumeNumber, chapterNumber, chapterUrl)); + } + + //Return Chapters ordered by Chapter-Number + Log($"Got {chapters.Count} chapters. {manga}"); + return chapters.Order().ToArray(); + } + catch (HttpRequestException e) + { + Log($"Failed to load https://mangasee123.com/rss/{manga.publicationId}.xml \n\r{e}"); + return Array.Empty(); + } + } + + public override HttpStatusCode DownloadChapter(Chapter chapter, ProgressToken? progressToken = null) + { + if (progressToken?.cancellationRequested ?? false) + { + progressToken.Cancel(); + return HttpStatusCode.RequestTimeout; + } + + var chapterParentManga = chapter.parentManga; + if (progressToken?.cancellationRequested ?? false) + { + progressToken.Cancel(); + return HttpStatusCode.RequestTimeout; + } + + Log($"Retrieving chapter-info {chapter} {chapterParentManga}"); + + var requestResult = downloadClient.MakeRequest(chapter.url, RequestType.Default); + if (requestResult.htmlDocument is null) + { + progressToken?.Cancel(); + return HttpStatusCode.RequestTimeout; + } + + var document = requestResult.htmlDocument; + + var gallery = document.DocumentNode.Descendants("div").First(div => div.HasClass("ImageGallery")); + HtmlNode[] images = gallery.Descendants("img").Where(img => img.HasClass("img-fluid")).ToArray(); + List urls = new(); + foreach (var galleryImage in images) + urls.Add(galleryImage.GetAttributeValue("src", "")); + + return DownloadChapterImages(urls.ToArray(), chapter, RequestType.MangaImage, progressToken: progressToken); + } + + private struct SearchResult + { + public string i { get; set; } + public string s { get; set; } + public string[] a { get; set; } + } +} \ No newline at end of file diff --git a/Tranga/Tranga.cs b/Tranga/Tranga.cs index 3209a5ea..65ea58e4 100644 --- a/Tranga/Tranga.cs +++ b/Tranga/Tranga.cs @@ -27,6 +27,7 @@ public Tranga(Logger? logger) : base(logger) new ManhuaPlus(this), new MangaHere(this), new AsuraToon(this), + new Weebcentral(this) }; foreach(DirectoryInfo dir in new DirectoryInfo(Path.GetTempPath()).GetDirectories("trangatemp"))//Cleanup old temp folders dir.Delete(); From 5d9bfc3adfc32bb6afafb5c22df1d941d0a03bbf Mon Sep 17 00:00:00 2001 From: Alessandro Benetton Date: Sat, 14 Dec 2024 00:39:59 +0100 Subject: [PATCH 06/34] [feature/weebcentral] Get chapters --- Tranga/MangaConnectors/WeebCentral.cs | 77 +++++++++++++++------------ 1 file changed, 43 insertions(+), 34 deletions(-) diff --git a/Tranga/MangaConnectors/WeebCentral.cs b/Tranga/MangaConnectors/WeebCentral.cs index 3fc6c6d6..4e25d4fa 100644 --- a/Tranga/MangaConnectors/WeebCentral.cs +++ b/Tranga/MangaConnectors/WeebCentral.cs @@ -40,11 +40,6 @@ public override Manga[] GetManga(string publicationTitle = "") return publications; } - /* - * Title: document.DocumentNode.SelectNodes("/html/body/article/section/div/a[@class='link link-hover']").Select(elem => elem.InnerText).ToList() - * URL: document.DocumentNode.SelectNodes("/html/body/article/a[@class='link link-hover']").Select(elem => elem.GetAttributeValue("href", "")).ToList() - */ - private Manga[] ParsePublicationsFromHtml(HtmlDocument document) { if (document.DocumentNode.SelectNodes("//article") == null) @@ -124,6 +119,11 @@ private Manga ParseSinglePublicationFromHtml(HtmlDocument document, string publi AddMangaToCache(manga); return manga; } + + public override Manga? GetMangaFromId(string publicationId) + { + return GetMangaFromUrl($"https://weebcentral.com/series/{publicationId}"); + } private string ToFilteredString(string input) { @@ -152,40 +152,49 @@ private SearchResult[] FilteredResults(string publicationTitle, SearchResult[] u return ret.ToArray(); } - public override Manga? GetMangaFromId(string publicationId) + public override Chapter[] GetChapters(Manga manga, string language="en") { - return GetMangaFromUrl($"https://mangasee123.com/manga/{publicationId}"); + Log($"Getting chapters {manga}"); + string requestUrl = $"{baseURL}/series/{manga.publicationId}/full-chapter-list"; + RequestResult requestResult = + downloadClient.MakeRequest(requestUrl, RequestType.Default); + if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300) + return Array.Empty(); + + //Return Chapters ordered by Chapter-Number + if (requestResult.htmlDocument is null) + return Array.Empty(); + List chapters = ParseChaptersFromHtml(manga, requestResult.htmlDocument); + Log($"Got {chapters.Count} chapters. {manga}"); + return chapters.Order().ToArray(); } - - public override Chapter[] GetChapters(Manga manga, string language = "en") + + private List ParseChaptersFromHtml(Manga manga, HtmlDocument document) { - Log($"Getting chapters {manga}"); - try - { - var doc = XDocument.Load($"https://mangasee123.com/rss/{manga.publicationId}.xml"); - var chapterItems = doc.Descendants("item").ToArray(); - List chapters = new(); - Regex chVolRex = new(@".*chapter-([0-9\.]+)(?:-index-([0-9\.]+))?.*"); - foreach (var chapter in chapterItems) - { - var url = chapter.Descendants("link").First().Value; - var m = chVolRex.Match(url); - var volumeNumber = m.Groups[2].Success ? m.Groups[2].Value : "1"; - var chapterNumber = m.Groups[1].Value; - - var chapterUrl = Regex.Replace(url, @"-page-[0-9]+(\.html)", ".html"); - chapters.Add(new Chapter(manga, "", volumeNumber, chapterNumber, chapterUrl)); - } + HtmlNode chaptersWrapper = document.DocumentNode.SelectSingleNode("/html/body"); + + Regex chapterRex = new(@".* (\d+)"); + Regex idRex = new(@"https:\/\/weebcentral\.com\/chapters\/(\w*)"); - //Return Chapters ordered by Chapter-Number - Log($"Got {chapters.Count} chapters. {manga}"); - return chapters.Order().ToArray(); - } - catch (HttpRequestException e) + List ret = chaptersWrapper.Descendants("a").Select(elem => { - Log($"Failed to load https://mangasee123.com/rss/{manga.publicationId}.xml \n\r{e}"); - return Array.Empty(); - } + var url = elem.GetAttributeValue("href", "") ?? "Undefined"; + + if (!url.StartsWith("https://") && !url.StartsWith("http://")) return new Chapter(manga, null, null, "-1", "undefined"); + + var idMatch = idRex.Match(url); + var id = (idMatch.Success ? idMatch.Groups[1].Value : null); + + var chapterNode = elem.SelectSingleNode("span[@class='grow flex items-center gap-2']/span")?.InnerText ?? "Undefined"; + + var chapterNumberMatch = chapterRex.Match(chapterNode); + var chapterNumber = (chapterNumberMatch.Success ? chapterNumberMatch.Groups[1].Value : "-1"); + + return new Chapter(manga, null, null, chapterNumber, url, id); + }).Where(elem => elem.chapterNumber != "-1" && elem.url != "undefined").ToList(); + + ret.Reverse(); + return ret; } public override HttpStatusCode DownloadChapter(Chapter chapter, ProgressToken? progressToken = null) From cf4c84a47f1d83bd6bb3841bd9e43341c1df80d0 Mon Sep 17 00:00:00 2001 From: Alessandro Benetton Date: Sat, 14 Dec 2024 00:58:52 +0100 Subject: [PATCH 07/34] [feature/weebcentral] Working download logic --- Tranga/MangaConnectors/WeebCentral.cs | 70 ++++++++++++++------------- 1 file changed, 36 insertions(+), 34 deletions(-) diff --git a/Tranga/MangaConnectors/WeebCentral.cs b/Tranga/MangaConnectors/WeebCentral.cs index 4e25d4fa..0d35b211 100644 --- a/Tranga/MangaConnectors/WeebCentral.cs +++ b/Tranga/MangaConnectors/WeebCentral.cs @@ -1,6 +1,5 @@ using System.Net; using System.Text.RegularExpressions; -using System.Xml.Linq; using HtmlAgilityPack; using Soenneker.Utils.String.NeedlemanWunsch; using Tranga.Jobs; @@ -9,11 +8,11 @@ namespace Tranga.MangaConnectors; public class Weebcentral : MangaConnector { + private readonly string _baseUrl = "https://weebcentral.com"; + private readonly string[] _filterWords = { "a", "the", "of", "as", "to", "no", "for", "on", "with", "be", "and", "in", "wa", "at", "be", "ni" }; - private readonly string baseURL = "https://weebcentral.com"; - public Weebcentral(GlobalBase clone) : base(clone, "Weebcentral", ["en"]) { downloadClient = new ChromiumDownloadClient(clone); @@ -25,7 +24,7 @@ public override Manga[] GetManga(string publicationTitle = "") const int limit = 32; //How many values we want returned at once var offset = 0; //"Page" var requestUrl = - $"{baseURL}/search/data?limit={limit}&offset={offset}&text={publicationTitle}&sort=Best+Match&order=Ascending&official=Any&display_mode=Minimal%20Display"; + $"{_baseUrl}/search/data?limit={limit}&offset={offset}&text={publicationTitle}&sort=Best+Match&order=Ascending&official=Any&display_mode=Minimal%20Display"; var requestResult = downloadClient.MakeRequest(requestUrl, RequestType.Default); if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300 || @@ -37,6 +36,7 @@ public override Manga[] GetManga(string publicationTitle = "") var publications = ParsePublicationsFromHtml(requestResult.htmlDocument); Log($"Retrieved {publications.Length} publications. Term=\"{publicationTitle}\""); + return publications; } @@ -85,10 +85,11 @@ private Manga ParseSinglePublicationFromHtml(HtmlDocument document, string publi document.DocumentNode.SelectNodes("//ul/li[strong/text() = 'Author(s): ']/span")?.ToArray() ?? []; var authors = authorsNodes.Select(n => n.InnerText).ToList(); - HtmlNode[] genreNodes = document.DocumentNode.SelectNodes("//ul/li[strong/text() = 'Tags(s): ']/span")?.ToArray() ?? []; + HtmlNode[] genreNodes = + document.DocumentNode.SelectNodes("//ul/li[strong/text() = 'Tags(s): ']/span")?.ToArray() ?? []; HashSet tags = genreNodes.Select(n => n.InnerText).ToHashSet(); - - HtmlNode statusNode = document.DocumentNode.SelectSingleNode("//ul/li[strong/text() = 'Status: ']/a"); + + var statusNode = document.DocumentNode.SelectSingleNode("//ul/li[strong/text() = 'Status: ']/a"); var status = statusNode?.InnerText ?? ""; Log("unable to parse status"); var releaseStatus = Manga.ReleaseStatusByte.Unreleased; @@ -105,21 +106,22 @@ private Manga ParseSinglePublicationFromHtml(HtmlDocument document, string publi var descriptionNode = document.DocumentNode.SelectSingleNode("//ul/li[strong/text() = 'Description']/p"); var description = descriptionNode?.InnerText ?? "Undefined"; - - HtmlNode[] altTitleNodes = document.DocumentNode.SelectNodes("//ul/li[strong/text() = 'Associated Name(s)']/ul/li")?.ToArray() ?? []; + + HtmlNode[] altTitleNodes = document.DocumentNode + .SelectNodes("//ul/li[strong/text() = 'Associated Name(s)']/ul/li")?.ToArray() ?? []; Dictionary altTitles = new(), links = new(); - for(int i = 0; i < altTitleNodes.Length; i++) + for (var i = 0; i < altTitleNodes.Length; i++) altTitles.Add(i.ToString(), altTitleNodes[i].InnerText); var originalLanguage = ""; - + Manga manga = new(sortName, authors.ToList(), description, altTitles, tags.ToArray(), posterUrl, coverFileNameInCache, links, year, originalLanguage, publicationId, releaseStatus, websiteUrl); AddMangaToCache(manga); return manga; } - + public override Manga? GetMangaFromId(string publicationId) { return GetMangaFromUrl($"https://weebcentral.com/series/{publicationId}"); @@ -152,44 +154,46 @@ private SearchResult[] FilteredResults(string publicationTitle, SearchResult[] u return ret.ToArray(); } - public override Chapter[] GetChapters(Manga manga, string language="en") + public override Chapter[] GetChapters(Manga manga, string language = "en") { Log($"Getting chapters {manga}"); - string requestUrl = $"{baseURL}/series/{manga.publicationId}/full-chapter-list"; - RequestResult requestResult = + var requestUrl = $"{_baseUrl}/series/{manga.publicationId}/full-chapter-list"; + var requestResult = downloadClient.MakeRequest(requestUrl, RequestType.Default); if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300) return Array.Empty(); - + //Return Chapters ordered by Chapter-Number if (requestResult.htmlDocument is null) return Array.Empty(); - List chapters = ParseChaptersFromHtml(manga, requestResult.htmlDocument); + var chapters = ParseChaptersFromHtml(manga, requestResult.htmlDocument); Log($"Got {chapters.Count} chapters. {manga}"); return chapters.Order().ToArray(); } - + private List ParseChaptersFromHtml(Manga manga, HtmlDocument document) { - HtmlNode chaptersWrapper = document.DocumentNode.SelectSingleNode("/html/body"); - + var chaptersWrapper = document.DocumentNode.SelectSingleNode("/html/body"); + Regex chapterRex = new(@".* (\d+)"); Regex idRex = new(@"https:\/\/weebcentral\.com\/chapters\/(\w*)"); - List ret = chaptersWrapper.Descendants("a").Select(elem => + var ret = chaptersWrapper.Descendants("a").Select(elem => { var url = elem.GetAttributeValue("href", "") ?? "Undefined"; - if (!url.StartsWith("https://") && !url.StartsWith("http://")) return new Chapter(manga, null, null, "-1", "undefined"); - + if (!url.StartsWith("https://") && !url.StartsWith("http://")) + return new Chapter(manga, null, null, "-1", "undefined"); + var idMatch = idRex.Match(url); - var id = (idMatch.Success ? idMatch.Groups[1].Value : null); + var id = idMatch.Success ? idMatch.Groups[1].Value : null; + + var chapterNode = elem.SelectSingleNode("span[@class='grow flex items-center gap-2']/span")?.InnerText ?? + "Undefined"; - var chapterNode = elem.SelectSingleNode("span[@class='grow flex items-center gap-2']/span")?.InnerText ?? "Undefined"; - var chapterNumberMatch = chapterRex.Match(chapterNode); - var chapterNumber = (chapterNumberMatch.Success ? chapterNumberMatch.Groups[1].Value : "-1"); - + var chapterNumber = chapterNumberMatch.Success ? chapterNumberMatch.Groups[1].Value : "-1"; + return new Chapter(manga, null, null, chapterNumber, url, id); }).Where(elem => elem.chapterNumber != "-1" && elem.url != "undefined").ToList(); @@ -223,13 +227,11 @@ public override HttpStatusCode DownloadChapter(Chapter chapter, ProgressToken? p var document = requestResult.htmlDocument; - var gallery = document.DocumentNode.Descendants("div").First(div => div.HasClass("ImageGallery")); - HtmlNode[] images = gallery.Descendants("img").Where(img => img.HasClass("img-fluid")).ToArray(); - List urls = new(); - foreach (var galleryImage in images) - urls.Add(galleryImage.GetAttributeValue("src", "")); + var imageNodes = + document.DocumentNode.SelectNodes($"//section[@hx-get='{chapter.url}/images']/img")?.ToArray() ?? []; + var urls = imageNodes.Select(imgNode => imgNode.GetAttributeValue("src", "")).ToArray(); - return DownloadChapterImages(urls.ToArray(), chapter, RequestType.MangaImage, progressToken: progressToken); + return DownloadChapterImages(urls, chapter, RequestType.MangaImage, progressToken: progressToken); } private struct SearchResult From dd988658c034dc09ddfcaf11377a0c3998116701 Mon Sep 17 00:00:00 2001 From: Alessandro Benetton Date: Sat, 14 Dec 2024 16:18:15 +0100 Subject: [PATCH 08/34] [feature/weebcentral] Added Weebcentral to connectors --- Tranga/MangaConnectors/MangaConnectorJsonConverter.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/Tranga/MangaConnectors/MangaConnectorJsonConverter.cs b/Tranga/MangaConnectors/MangaConnectorJsonConverter.cs index 283d7e75..b0cc3dca 100644 --- a/Tranga/MangaConnectors/MangaConnectorJsonConverter.cs +++ b/Tranga/MangaConnectors/MangaConnectorJsonConverter.cs @@ -39,6 +39,7 @@ public override object ReadJson(JsonReader reader, Type objectType, object? exis "ManhuaPlus" => this._connectors.First(c => c is ManhuaPlus), "MangaHere" => this._connectors.First(c => c is MangaHere), "AsuraToon" => this._connectors.First(c => c is AsuraToon), + "Weebcentral" => this._connectors.First(c => c is Weebcentral), _ => throw new UnreachableException($"Could not find Connector with name {connectorName}") }; } From 4c8d9bfaf29ef62f6cd3ce5aaa574bec1162f823 Mon Sep 17 00:00:00 2001 From: Alessandro Benetton Date: Sat, 14 Dec 2024 16:29:43 +0100 Subject: [PATCH 09/34] [feature/weebcentral] Added Weebcentral to readme --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 2aca5783..01514523 100644 --- a/README.md +++ b/README.md @@ -52,6 +52,7 @@ Tranga can download Chapters and Metadata from "Scanlation" sites such as - [Manga4Life](https://manga4life.com) (en) - [ManhuaPlus](https://manhuaplus.org/) (en) - [MangaHere](https://www.mangahere.cc/) (en) (Their covers aren't scrapeable.) +- [Weebcentral](https://weebcentral.com) (en) - ❓ Open an [issue](https://github.com/C9Glax/tranga/issues/new?assignees=&labels=New+Connector&projects=&template=new_connector.yml&title=%5BNew+Connector%5D%3A+) and trigger a library-scan with [Komga](https://komga.org/) and [Kavita](https://www.kavitareader.com/). From 93cfdddd19f260eba05ed77bc78d354ebe2a1148 Mon Sep 17 00:00:00 2001 From: Glax Date: Sat, 14 Dec 2024 17:51:22 +0100 Subject: [PATCH 10/34] Possible fix #300 chromium statup "Failed to launch browser! chrome_crashpad_handler: --database is required" --- Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Dockerfile b/Dockerfile index 55a5f6c1..35eb708a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,6 +5,8 @@ FROM --platform=$TARGETPLATFORM mcr.microsoft.com/dotnet/runtime:$DOTNET AS base WORKDIR /publish ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true ENV PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium +ENV XDG_CONFIG_HOME=/tmp/.chromium +ENV XDG_CACHE_HOME=/tmp/.chromium RUN apt-get update \ && apt-get install -y libx11-6 libx11-xcb1 libatk1.0-0 libgtk-3-0 libcups2 libdrm2 libxkbcommon0 libxcomposite1 libxdamage1 libxrandr2 libgbm1 libpango-1.0-0 libcairo2 libasound2 libxshmfence1 libnss3 chromium \ && apt-get autopurge -y \ From b8c624f3eaba4ee9adf8551e2b2c6a27ae0d07a3 Mon Sep 17 00:00:00 2001 From: Glax Date: Sat, 14 Dec 2024 17:55:20 +0100 Subject: [PATCH 11/34] AsuraToon crash when there is no search-results #296 --- Tranga/MangaConnectors/AsuraToon.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Tranga/MangaConnectors/AsuraToon.cs b/Tranga/MangaConnectors/AsuraToon.cs index b19334a4..ea05fcd3 100644 --- a/Tranga/MangaConnectors/AsuraToon.cs +++ b/Tranga/MangaConnectors/AsuraToon.cs @@ -55,8 +55,8 @@ public override Manga[] GetManga(string publicationTitle = "") private Manga[] ParsePublicationsFromHtml(HtmlDocument document) { HtmlNodeCollection mangaList = document.DocumentNode.SelectNodes("//a[starts-with(@href,'series')]"); - if (mangaList.Count < 1) - return Array.Empty(); + if (mangaList is null || mangaList.Count < 1) + return []; IEnumerable urls = mangaList.Select(a => $"https://asuracomic.net/{a.GetAttributeValue("href", "")}"); From 825b945ad1d507a29a86f81aed4481e20af93716 Mon Sep 17 00:00:00 2001 From: Glax Date: Sat, 14 Dec 2024 18:02:41 +0100 Subject: [PATCH 12/34] AsuraToon Crash on no Artists or Authors Fix #296 --- Tranga/MangaConnectors/AsuraToon.cs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/Tranga/MangaConnectors/AsuraToon.cs b/Tranga/MangaConnectors/AsuraToon.cs index ea05fcd3..8ec52658 100644 --- a/Tranga/MangaConnectors/AsuraToon.cs +++ b/Tranga/MangaConnectors/AsuraToon.cs @@ -102,11 +102,13 @@ private Manga ParseSinglePublicationFromHtml(HtmlDocument document, string publi HtmlNode descriptionNode = document.DocumentNode.SelectSingleNode("//h3[starts-with(text(),'Synopsis')]/../span"); - string description = descriptionNode.InnerText; + string description = descriptionNode?.InnerText??""; HtmlNodeCollection authorNodes = document.DocumentNode.SelectNodes("//h3[text()='Author']/../h3[not(text()='Author' or text()='_')]"); - HtmlNodeCollection artistNodes = document.DocumentNode.SelectNodes("//h3[text()='Artist']/../h3[not(text()='Author' or text()='_')]"); - List authors = authorNodes.Select(a => a.InnerText).Concat(artistNodes.Select(a => a.InnerText)).ToList(); + HtmlNodeCollection artistNodes = document.DocumentNode.SelectNodes("//h3[text()='Artist']/../h3[not(text()='Artist' or text()='_')]"); + IEnumerable authorNames = authorNodes is null ? [] : authorNodes.Select(a => a.InnerText); + IEnumerable artistNames = artistNodes is null ? [] : artistNodes.Select(a => a.InnerText); + List authors = authorNames.Concat(artistNames).ToList(); HtmlNode? firstChapterNode = document.DocumentNode.SelectSingleNode("//a[contains(@href, 'chapter/1')]/../following-sibling::h3"); int? year = int.Parse(firstChapterNode?.InnerText.Split(' ')[^1] ?? "2000"); From 72d9bda0e83632ecca96b657db7f1f0aa28c98d3 Mon Sep 17 00:00:00 2001 From: Alessandro Benetton Date: Sat, 14 Dec 2024 20:44:43 +0100 Subject: [PATCH 13/34] [feature/weebcentral_build_error] fix type in equality check --- Tranga/MangaConnectors/WeebCentral.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tranga/MangaConnectors/WeebCentral.cs b/Tranga/MangaConnectors/WeebCentral.cs index 0d35b211..6c08454a 100644 --- a/Tranga/MangaConnectors/WeebCentral.cs +++ b/Tranga/MangaConnectors/WeebCentral.cs @@ -195,7 +195,7 @@ private List ParseChaptersFromHtml(Manga manga, HtmlDocument document) var chapterNumber = chapterNumberMatch.Success ? chapterNumberMatch.Groups[1].Value : "-1"; return new Chapter(manga, null, null, chapterNumber, url, id); - }).Where(elem => elem.chapterNumber != "-1" && elem.url != "undefined").ToList(); + }).Where(elem => elem.chapterNumber != -1 && elem.url != "undefined").ToList(); ret.Reverse(); return ret; From 6aa8413c40fe9359e754c74f23575bed1becc360 Mon Sep 17 00:00:00 2001 From: Glax Date: Thu, 9 Jan 2025 01:48:13 +0100 Subject: [PATCH 14/34] Fix #311 MangaWorld now requires Javascript --- Tranga/MangaConnectors/Mangaworld.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tranga/MangaConnectors/Mangaworld.cs b/Tranga/MangaConnectors/Mangaworld.cs index 23522b6e..f54876c5 100644 --- a/Tranga/MangaConnectors/Mangaworld.cs +++ b/Tranga/MangaConnectors/Mangaworld.cs @@ -9,7 +9,7 @@ public class Mangaworld: MangaConnector { public Mangaworld(GlobalBase clone) : base(clone, "Mangaworld", ["it"]) { - this.downloadClient = new HttpDownloadClient(clone); + this.downloadClient = new ChromiumDownloadClient(clone); } public override Manga[] GetManga(string publicationTitle = "") From b39dbd5671229dfb0d862038bc7317ba7851f7b6 Mon Sep 17 00:00:00 2001 From: Alessandro Benetton Date: Fri, 10 Jan 2025 22:10:34 +0100 Subject: [PATCH 15/34] [cuttingedge] fix(weebcentral): Fixed regex to capture chapters with decimal (1.5, ..) --- Tranga/MangaConnectors/WeebCentral.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tranga/MangaConnectors/WeebCentral.cs b/Tranga/MangaConnectors/WeebCentral.cs index 6c08454a..842afffe 100644 --- a/Tranga/MangaConnectors/WeebCentral.cs +++ b/Tranga/MangaConnectors/WeebCentral.cs @@ -175,7 +175,7 @@ private List ParseChaptersFromHtml(Manga manga, HtmlDocument document) { var chaptersWrapper = document.DocumentNode.SelectSingleNode("/html/body"); - Regex chapterRex = new(@".* (\d+)"); + Regex chapterRex = new(@"(\d+(?:.\d+)*)"); Regex idRex = new(@"https:\/\/weebcentral\.com\/chapters\/(\w*)"); var ret = chaptersWrapper.Descendants("a").Select(elem => From f532e2ff76f56518e099c1b3af69d02d06671d01 Mon Sep 17 00:00:00 2001 From: Glax Date: Wed, 15 Jan 2025 22:13:50 +0100 Subject: [PATCH 16/34] JobBoss LoadJobsList change: Fix Directory.Exists jobsFolderPath to create new Directory Fix Loading Job fails leading to crash. --- Tranga/Jobs/JobBoss.cs | 35 ++++++++++++++++++++++------------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/Tranga/Jobs/JobBoss.cs b/Tranga/Jobs/JobBoss.cs index b1b38dd8..2fe58a30 100644 --- a/Tranga/Jobs/JobBoss.cs +++ b/Tranga/Jobs/JobBoss.cs @@ -1,4 +1,5 @@ -using System.Runtime.InteropServices; +using System.Diagnostics; +using System.Runtime.InteropServices; using System.Text.RegularExpressions; using Newtonsoft.Json; using Tranga.MangaConnectors; @@ -145,26 +146,24 @@ private void AddJobsToQueue(IEnumerable newJobs) private void LoadJobsList(HashSet connectors) { - Directory.CreateDirectory(TrangaSettings.jobsFolderPath); - if(RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) - File.SetUnixFileMode(TrangaSettings.jobsFolderPath, UserRead | UserWrite | UserExecute | GroupRead | OtherRead); if (!Directory.Exists(TrangaSettings.jobsFolderPath)) //No jobs to load + { + Directory.CreateDirectory(TrangaSettings.jobsFolderPath); + if(RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) + File.SetUnixFileMode(TrangaSettings.jobsFolderPath, UserRead | UserWrite | UserExecute | GroupRead | OtherRead); return; + } //Load json-job-files foreach (FileInfo file in Directory.GetFiles(TrangaSettings.jobsFolderPath, "*.json").Select(f => new FileInfo(f))) { Log($"Adding {file.Name}"); - Job? job = JsonConvert.DeserializeObject(File.ReadAllText(file.FullName), - new JobJsonConverter(this, new MangaConnectorJsonConverter(this, connectors))); - if (job is null) - { - string newName = file.FullName + ".failed"; - Log($"Failed loading file {file.Name}.\nMoving to {newName}"); - File.Move(file.FullName, newName); - } - else + try { + Job? job = JsonConvert.DeserializeObject(File.ReadAllText(file.FullName), + new JobJsonConverter(this, new MangaConnectorJsonConverter(this, connectors))); + if (job is null) throw new NullReferenceException(); + Log($"Adding Job {job}"); if (!AddJob(job, file.FullName)) //If we detect a duplicate, delete the file. { @@ -173,6 +172,16 @@ private void LoadJobsList(HashSet connectors) Log($"Duplicate detected or otherwise not able to add job to list.\nMoved job {job} to {path}"); } } + catch (Exception e) + { + if (e is not UnreachableException or NullReferenceException) + throw; + Log(e.Message); + string newName = file.FullName + ".failed"; + Log($"Failed loading file {file.Name}.\nMoving to {newName}"); + File.Move(file.FullName, newName); + continue; + } } //Connect jobs to parent-jobs and add Publications to cache From 2350c5a04b654508bb36c0352315fcf138987d23 Mon Sep 17 00:00:00 2001 From: Glax Date: Wed, 15 Jan 2025 22:13:58 +0100 Subject: [PATCH 17/34] Remove Mangasee --- .../MangaConnectorJsonConverter.cs | 1 - Tranga/MangaConnectors/Mangasee.cs | 234 ------------------ Tranga/Tranga.cs | 1 - 3 files changed, 236 deletions(-) delete mode 100644 Tranga/MangaConnectors/Mangasee.cs diff --git a/Tranga/MangaConnectors/MangaConnectorJsonConverter.cs b/Tranga/MangaConnectors/MangaConnectorJsonConverter.cs index b0cc3dca..f54dbe39 100644 --- a/Tranga/MangaConnectors/MangaConnectorJsonConverter.cs +++ b/Tranga/MangaConnectors/MangaConnectorJsonConverter.cs @@ -32,7 +32,6 @@ public override object ReadJson(JsonReader reader, Type objectType, object? exis "MangaDex" => this._connectors.First(c => c is MangaDex), "Manganato" => this._connectors.First(c => c is Manganato), "MangaKatana" => this._connectors.First(c => c is MangaKatana), - "Mangasee" => this._connectors.First(c => c is Mangasee), "Mangaworld" => this._connectors.First(c => c is Mangaworld), "Bato" => this._connectors.First(c => c is Bato), "Manga4Life" => this._connectors.First(c => c is MangaLife), diff --git a/Tranga/MangaConnectors/Mangasee.cs b/Tranga/MangaConnectors/Mangasee.cs deleted file mode 100644 index f912f6df..00000000 --- a/Tranga/MangaConnectors/Mangasee.cs +++ /dev/null @@ -1,234 +0,0 @@ -using System.Data; -using System.Net; -using System.Text.RegularExpressions; -using System.Xml.Linq; -using HtmlAgilityPack; -using Newtonsoft.Json; -using Soenneker.Utils.String.NeedlemanWunsch; -using Tranga.Jobs; - -namespace Tranga.MangaConnectors; - -public class Mangasee : MangaConnector -{ - public Mangasee(GlobalBase clone) : base(clone, "Mangasee", ["en"]) - { - this.downloadClient = new ChromiumDownloadClient(clone); - } - - private struct SearchResult - { - public string i { get; set; } - public string s { get; set; } - public string[] a { get; set; } - } - - public override Manga[] GetManga(string publicationTitle = "") - { - Log($"Searching Publications. Term=\"{publicationTitle}\""); - string requestUrl = "https://mangasee123.com/_search.php"; - RequestResult requestResult = - downloadClient.MakeRequest(requestUrl, RequestType.Default); - if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300) - { - Log($"Failed to retrieve search: {requestResult.statusCode}"); - return Array.Empty(); - } - - try - { - SearchResult[] searchResults = JsonConvert.DeserializeObject(requestResult.htmlDocument!.DocumentNode.InnerText) ?? - throw new NoNullAllowedException(); - SearchResult[] filteredResults = FilteredResults(publicationTitle, searchResults); - Log($"Total available manga: {searchResults.Length} Filtered down to: {filteredResults.Length}"); - - - string[] urls = filteredResults.Select(result => $"https://mangasee123.com/manga/{result.i}").ToArray(); - List searchResultManga = new(); - foreach (string url in urls) - { - Manga? newManga = GetMangaFromUrl(url); - if(newManga is { } manga) - searchResultManga.Add(manga); - } - Log($"Retrieved {searchResultManga.Count} publications. Term=\"{publicationTitle}\""); - return searchResultManga.ToArray(); - } - catch (NoNullAllowedException) - { - Log("Failed to retrieve search"); - return Array.Empty(); - } - } - - private readonly string[] _filterWords = {"a", "the", "of", "as", "to", "no", "for", "on", "with", "be", "and", "in", "wa", "at", "be", "ni"}; - private string ToFilteredString(string input) => string.Join(' ', input.ToLower().Split(' ').Where(word => _filterWords.Contains(word) == false)); - private SearchResult[] FilteredResults(string publicationTitle, SearchResult[] unfilteredSearchResults) - { - Dictionary similarity = new(); - foreach (SearchResult sr in unfilteredSearchResults) - { - List scores = new(); - string filteredPublicationString = ToFilteredString(publicationTitle); - string filteredSString = ToFilteredString(sr.s); - scores.Add(NeedlemanWunschStringUtil.CalculateSimilarity(filteredSString, filteredPublicationString)); - foreach (string srA in sr.a) - { - string filteredAString = ToFilteredString(srA); - scores.Add(NeedlemanWunschStringUtil.CalculateSimilarity(filteredAString, filteredPublicationString)); - } - similarity.Add(sr, scores.Sum() / scores.Count); - } - - List ret = similarity.OrderBy(s => s.Value).Take(10).Select(s => s.Key).ToList(); - return ret.ToArray(); - } - - public override Manga? GetMangaFromId(string publicationId) - { - return GetMangaFromUrl($"https://mangasee123.com/manga/{publicationId}"); - } - - public override Manga? GetMangaFromUrl(string url) - { - Regex publicationIdRex = new(@"https:\/\/mangasee123.com\/manga\/(.*)(\/.*)*"); - string publicationId = publicationIdRex.Match(url).Groups[1].Value; - - RequestResult requestResult = this.downloadClient.MakeRequest(url, RequestType.MangaInfo); - if((int)requestResult.statusCode < 300 && (int)requestResult.statusCode >= 200 && requestResult.htmlDocument is not null) - return ParseSinglePublicationFromHtml(requestResult.htmlDocument, publicationId, url); - return null; - } - - private Manga ParseSinglePublicationFromHtml(HtmlDocument document, string publicationId, string websiteUrl) - { - string originalLanguage = "", status = ""; - Dictionary altTitles = new(), links = new(); - HashSet tags = new(); - Manga.ReleaseStatusByte releaseStatus = Manga.ReleaseStatusByte.Unreleased; - - HtmlNode posterNode = document.DocumentNode.SelectSingleNode("//div[@class='BoxBody']//div[@class='row']//img"); - string posterUrl = posterNode.GetAttributeValue("src", ""); - string coverFileNameInCache = SaveCoverImageToCache(posterUrl, publicationId, RequestType.MangaCover); - - HtmlNode titleNode = document.DocumentNode.SelectSingleNode("//div[@class='BoxBody']//div[@class='row']//h1"); - string sortName = titleNode.InnerText; - - HtmlNode[] authorsNodes = document.DocumentNode - .SelectNodes("//div[@class='BoxBody']//div[@class='row']//span[text()='Author(s):']/..").Descendants("a") - .ToArray(); - List authors = new(); - foreach (HtmlNode authorNode in authorsNodes) - authors.Add(authorNode.InnerText); - - HtmlNode[] genreNodes = document.DocumentNode - .SelectNodes("//div[@class='BoxBody']//div[@class='row']//span[text()='Genre(s):']/..").Descendants("a") - .ToArray(); - foreach (HtmlNode genreNode in genreNodes) - tags.Add(genreNode.InnerText); - - HtmlNode yearNode = document.DocumentNode - .SelectNodes("//div[@class='BoxBody']//div[@class='row']//span[text()='Released:']/..").Descendants("a") - .First(); - int year = Convert.ToInt32(yearNode.InnerText); - - HtmlNode[] statusNodes = document.DocumentNode - .SelectNodes("//div[@class='BoxBody']//div[@class='row']//span[text()='Status:']/..").Descendants("a") - .ToArray(); - foreach (HtmlNode statusNode in statusNodes) - if (statusNode.InnerText.Contains("publish", StringComparison.CurrentCultureIgnoreCase)) - status = statusNode.InnerText.Split(' ')[0]; - switch (status.ToLower()) - { - case "cancelled": releaseStatus = Manga.ReleaseStatusByte.Cancelled; break; - case "hiatus": releaseStatus = Manga.ReleaseStatusByte.OnHiatus; break; - case "discontinued": releaseStatus = Manga.ReleaseStatusByte.Cancelled; break; - case "complete": releaseStatus = Manga.ReleaseStatusByte.Completed; break; - case "ongoing": releaseStatus = Manga.ReleaseStatusByte.Continuing; break; - } - - HtmlNode descriptionNode = document.DocumentNode - .SelectNodes("//div[@class='BoxBody']//div[@class='row']//span[text()='Description:']/..") - .Descendants("div").First(); - string description = descriptionNode.InnerText; - - Manga manga = new(sortName, authors.ToList(), description, altTitles, tags.ToArray(), posterUrl, - coverFileNameInCache, links, - year, originalLanguage, publicationId, releaseStatus, websiteUrl: websiteUrl); - AddMangaToCache(manga); - return manga; - } - - public override Chapter[] GetChapters(Manga manga, string language="en") - { - Log($"Getting chapters {manga}"); - try - { - XDocument doc = XDocument.Load($"https://mangasee123.com/rss/{manga.publicationId}.xml"); - XElement[] chapterItems = doc.Descendants("item").ToArray(); - List chapters = new(); - Regex chVolRex = new(@".*chapter-([0-9\.]+)(?:-index-([0-9\.]+))?.*"); - foreach (XElement chapter in chapterItems) - { - string url = chapter.Descendants("link").First().Value; - Match m = chVolRex.Match(url); - string? volumeNumber = m.Groups[2].Success ? m.Groups[2].Value : "1"; - string chapterNumber = m.Groups[1].Value; - - string chapterUrl = Regex.Replace(url, @"-page-[0-9]+(\.html)", ".html"); - try - { - chapters.Add(new Chapter(manga, "", volumeNumber, chapterNumber, chapterUrl)); - } - catch (Exception e) - { - Log($"Failed to load chapter {chapterNumber}: {e.Message}"); - } - } - - //Return Chapters ordered by Chapter-Number - Log($"Got {chapters.Count} chapters. {manga}"); - return chapters.Order().ToArray(); - } - catch (HttpRequestException e) - { - Log($"Failed to load https://mangasee123.com/rss/{manga.publicationId}.xml \n\r{e}"); - return Array.Empty(); - } - } - - public override HttpStatusCode DownloadChapter(Chapter chapter, ProgressToken? progressToken = null) - { - if (progressToken?.cancellationRequested ?? false) - { - progressToken.Cancel(); - return HttpStatusCode.RequestTimeout; - } - - Manga chapterParentManga = chapter.parentManga; - if (progressToken?.cancellationRequested ?? false) - { - progressToken.Cancel(); - return HttpStatusCode.RequestTimeout; - } - - Log($"Retrieving chapter-info {chapter} {chapterParentManga}"); - - RequestResult requestResult = this.downloadClient.MakeRequest(chapter.url, RequestType.Default); - if (requestResult.htmlDocument is null) - { - progressToken?.Cancel(); - return HttpStatusCode.RequestTimeout; - } - - HtmlDocument document = requestResult.htmlDocument; - - HtmlNode gallery = document.DocumentNode.Descendants("div").First(div => div.HasClass("ImageGallery")); - HtmlNode[] images = gallery.Descendants("img").Where(img => img.HasClass("img-fluid")).ToArray(); - List urls = new(); - foreach(HtmlNode galleryImage in images) - urls.Add(galleryImage.GetAttributeValue("src", "")); - - return DownloadChapterImages(urls.ToArray(), chapter, RequestType.MangaImage, progressToken:progressToken); - } -} \ No newline at end of file diff --git a/Tranga/Tranga.cs b/Tranga/Tranga.cs index 65ea58e4..3292e6d8 100644 --- a/Tranga/Tranga.cs +++ b/Tranga/Tranga.cs @@ -18,7 +18,6 @@ public Tranga(Logger? logger) : base(logger) _connectors = new HashSet() { new Manganato(this), - new Mangasee(this), new MangaDex(this), new MangaKatana(this), new Mangaworld(this), From 123a8b06b29fcfa5fdf781ac778c33bdd8fdb3a6 Mon Sep 17 00:00:00 2001 From: Glax Date: Wed, 15 Jan 2025 22:15:33 +0100 Subject: [PATCH 18/34] jobloading errormessage --- Tranga/Jobs/JobBoss.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Tranga/Jobs/JobBoss.cs b/Tranga/Jobs/JobBoss.cs index 2fe58a30..4631a667 100644 --- a/Tranga/Jobs/JobBoss.cs +++ b/Tranga/Jobs/JobBoss.cs @@ -178,7 +178,8 @@ private void LoadJobsList(HashSet connectors) throw; Log(e.Message); string newName = file.FullName + ".failed"; - Log($"Failed loading file {file.Name}.\nMoving to {newName}"); + Log($"Failed loading file {file.Name}.\nMoving to {newName}.\n" + + $"If you think this is a bug, upload contents of the file to the Bugreport!"); File.Move(file.FullName, newName); continue; } From cc32b3dfae4bcbca25440f97cc580db2025e6628 Mon Sep 17 00:00:00 2001 From: Glax Date: Wed, 15 Jan 2025 22:24:55 +0100 Subject: [PATCH 19/34] TrangaSettings Chromium Timeouts --- Tranga/MangaConnectors/ChromiumDownloadClient.cs | 7 ++++--- Tranga/TrangaSettings.cs | 2 ++ 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/Tranga/MangaConnectors/ChromiumDownloadClient.cs b/Tranga/MangaConnectors/ChromiumDownloadClient.cs index 3b5e63de..c4de50ff 100644 --- a/Tranga/MangaConnectors/ChromiumDownloadClient.cs +++ b/Tranga/MangaConnectors/ChromiumDownloadClient.cs @@ -10,7 +10,6 @@ namespace Tranga.MangaConnectors; internal class ChromiumDownloadClient : DownloadClient { private static IBrowser? _browser; - private const int StartTimeoutMs = 10000; private readonly HttpDownloadClient _httpDownloadClient; private static async Task StartBrowser(Logging.Logger? logger = null) @@ -24,7 +23,7 @@ private static async Task StartBrowser(Logging.Logger? logger = null) "--disable-dev-shm-usage", "--disable-setuid-sandbox", "--no-sandbox"}, - Timeout = StartTimeoutMs + Timeout = TrangaSettings.ChromiumStartupTimeoutMs }, new LoggerFactory([new LogProvider(logger)])); } @@ -70,8 +69,10 @@ internal override RequestResult MakeRequestInternal(string url, string? referrer private RequestResult MakeRequestBrowser(string url, string? referrer = null, string? clickButton = null) { + if (_browser is null) + return new RequestResult(HttpStatusCode.InternalServerError, null, Stream.Null); IPage page = _browser.NewPageAsync().Result; - page.DefaultTimeout = 10000; + page.DefaultTimeout = TrangaSettings.ChromiumPageTimeoutMs; IResponse response; try { diff --git a/Tranga/TrangaSettings.cs b/Tranga/TrangaSettings.cs index ccfa9978..3558574b 100644 --- a/Tranga/TrangaSettings.cs +++ b/Tranga/TrangaSettings.cs @@ -35,6 +35,8 @@ public static class TrangaSettings }; public static Dictionary requestLimits { get; set; } = DefaultRequestLimits; + public static int ChromiumStartupTimeoutMs { get; set; } = 30000; + public static int ChromiumPageTimeoutMs { get; set; } = 30000; public static void LoadFromWorkingDirectory(string directory) { From 78a9322036cd756da9c942a4f59cf4d2f2e0ec6c Mon Sep 17 00:00:00 2001 From: Glax Date: Wed, 15 Jan 2025 22:53:39 +0100 Subject: [PATCH 20/34] ChromiumDownloadClient Change WaitUnitlNavigation to Load instead of NetworkIdle --- Tranga/MangaConnectors/ChromiumDownloadClient.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Tranga/MangaConnectors/ChromiumDownloadClient.cs b/Tranga/MangaConnectors/ChromiumDownloadClient.cs index c4de50ff..effa9591 100644 --- a/Tranga/MangaConnectors/ChromiumDownloadClient.cs +++ b/Tranga/MangaConnectors/ChromiumDownloadClient.cs @@ -76,12 +76,12 @@ private RequestResult MakeRequestBrowser(string url, string? referrer = null, st IResponse response; try { - response = page.GoToAsync(url, WaitUntilNavigation.Networkidle0).Result; - Log("Page loaded."); + response = page.GoToAsync(url, WaitUntilNavigation.Load).Result; + Log($"Page loaded. {url}"); } catch (Exception e) { - Log($"Could not load Page:\n{e.Message}"); + Log($"Could not load Page {url}\n{e.Message}"); page.CloseAsync(); return new RequestResult(HttpStatusCode.InternalServerError, null, Stream.Null); } From 268441a47df869a307e899ca634fb90a55b57a0e Mon Sep 17 00:00:00 2001 From: Glax Date: Wed, 15 Jan 2025 23:02:48 +0100 Subject: [PATCH 21/34] Trangasettings Json --- Tranga/TrangaSettings.cs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Tranga/TrangaSettings.cs b/Tranga/TrangaSettings.cs index 3558574b..bb8b54f5 100644 --- a/Tranga/TrangaSettings.cs +++ b/Tranga/TrangaSettings.cs @@ -169,6 +169,8 @@ public static JObject AsJObject() jobj.Add("requestLimits", JToken.FromObject(requestLimits)); jobj.Add("bufferLibraryUpdates", JToken.FromObject(bufferLibraryUpdates)); jobj.Add("bufferNotifications", JToken.FromObject(bufferNotifications)); + jobj.Add("chromiumStartTimeout", JToken.FromObject(ChromiumStartupTimeoutMs)); + jobj.Add("chromiumPageTimeout", JToken.FromObject(ChromiumPageTimeoutMs)); return jobj; } @@ -193,5 +195,9 @@ public static void Deserialize(string serialized) bufferLibraryUpdates = blu.Value()!; if (jobj.TryGetValue("bufferNotifications", out JToken? bn)) bufferNotifications = bn.Value()!; + if (jobj.TryGetValue("chromiumStartTimeout", out JToken? cst)) + ChromiumStartupTimeoutMs = cst.Value(); + if (jobj.TryGetValue("chromiumPageTimeout", out JToken? cpt)) + ChromiumPageTimeoutMs = cpt.Value(); } } \ No newline at end of file From d0b775444d30e46c115dec677d77e6e8ed5b6ac0 Mon Sep 17 00:00:00 2001 From: Glax Date: Wed, 15 Jan 2025 23:14:15 +0100 Subject: [PATCH 22/34] Change Chromium back to WaitUntilNavigation.Networkidle0 --- Tranga/MangaConnectors/ChromiumDownloadClient.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tranga/MangaConnectors/ChromiumDownloadClient.cs b/Tranga/MangaConnectors/ChromiumDownloadClient.cs index effa9591..5c000057 100644 --- a/Tranga/MangaConnectors/ChromiumDownloadClient.cs +++ b/Tranga/MangaConnectors/ChromiumDownloadClient.cs @@ -76,7 +76,7 @@ private RequestResult MakeRequestBrowser(string url, string? referrer = null, st IResponse response; try { - response = page.GoToAsync(url, WaitUntilNavigation.Load).Result; + response = page.GoToAsync(url, WaitUntilNavigation.Networkidle0).Result; Log($"Page loaded. {url}"); } catch (Exception e) From 7921dcb1cbb382afea8ba097040b43cc19a3a963 Mon Sep 17 00:00:00 2001 From: Alessandro Benetton Date: Fri, 24 Jan 2025 21:52:52 +0100 Subject: [PATCH 23/34] [cuttingedge] fix: Change condition for newChapters. Should solve #323 --- Tranga/MangaConnectors/MangaConnector.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tranga/MangaConnectors/MangaConnector.cs b/Tranga/MangaConnectors/MangaConnector.cs index c98bfad2..03afa264 100644 --- a/Tranga/MangaConnectors/MangaConnector.cs +++ b/Tranga/MangaConnectors/MangaConnector.cs @@ -60,7 +60,7 @@ public Chapter[] GetNewChapters(Manga manga, string language = "en") return Array.Empty(); Log($"Checking for duplicates {manga}"); - List newChaptersList = allChapters.Where(nChapter => nChapter.chapterNumber > manga.ignoreChaptersBelow + List newChaptersList = allChapters.Where(nChapter => nChapter.chapterNumber >= manga.ignoreChaptersBelow && !nChapter.CheckChapterIsDownloaded()).ToList(); Log($"{newChaptersList.Count} new chapters. {manga}"); try From f7a285aabd610d07b69ebe90db3d57dd93fafe42 Mon Sep 17 00:00:00 2001 From: Alessandro Benetton Date: Sat, 25 Jan 2025 11:40:00 +0100 Subject: [PATCH 24/34] [cuttingedge] fix: Add escape to Weebcentral regex --- Tranga/MangaConnectors/WeebCentral.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tranga/MangaConnectors/WeebCentral.cs b/Tranga/MangaConnectors/WeebCentral.cs index 842afffe..50d63c47 100644 --- a/Tranga/MangaConnectors/WeebCentral.cs +++ b/Tranga/MangaConnectors/WeebCentral.cs @@ -175,7 +175,7 @@ private List ParseChaptersFromHtml(Manga manga, HtmlDocument document) { var chaptersWrapper = document.DocumentNode.SelectSingleNode("/html/body"); - Regex chapterRex = new(@"(\d+(?:.\d+)*)"); + Regex chapterRex = new(@"(\d+(?:\.\d+)*)"); Regex idRex = new(@"https:\/\/weebcentral\.com\/chapters\/(\w*)"); var ret = chaptersWrapper.Descendants("a").Select(elem => From 0c9e3205c21366458ff95e75896ed01e6501b931 Mon Sep 17 00:00:00 2001 From: Makhuta Date: Thu, 6 Feb 2025 15:37:30 +0100 Subject: [PATCH 25/34] Add Manga Connector - added [Webtoon](https://www.webtoons.com) manga connector - modified/added support for saving covers with refferer --- Tranga/MangaConnectors/MangaConnector.cs | 4 +- .../MangaConnectorJsonConverter.cs | 1 + Tranga/MangaConnectors/Webtoons.cs | 272 ++++++++++++++++++ Tranga/Tranga.cs | 1 + 4 files changed, 276 insertions(+), 2 deletions(-) create mode 100644 Tranga/MangaConnectors/Webtoons.cs diff --git a/Tranga/MangaConnectors/MangaConnector.cs b/Tranga/MangaConnectors/MangaConnector.cs index 03afa264..5529b451 100644 --- a/Tranga/MangaConnectors/MangaConnector.cs +++ b/Tranga/MangaConnectors/MangaConnector.cs @@ -213,7 +213,7 @@ protected HttpStatusCode DownloadChapterImages(string[] imageUrls, Chapter chapt return HttpStatusCode.OK; } - protected string SaveCoverImageToCache(string url, string mangaInternalId, RequestType requestType) + protected string SaveCoverImageToCache(string url, string mangaInternalId, RequestType requestType, string? refferer = null) { Regex urlRex = new (@"https?:\/\/((?:[a-zA-Z0-9-]+\.)+[a-zA-Z0-9]+)\/(?:.+\/)*(.+\.([a-zA-Z]+))"); //https?:\/\/[a-zA-Z0-9-]+\.([a-zA-Z0-9-]+\.[a-zA-Z0-9]+)\/(?:.+\/)*(.+\.([a-zA-Z]+)) for only second level domains @@ -224,7 +224,7 @@ protected string SaveCoverImageToCache(string url, string mangaInternalId, Reque if (File.Exists(saveImagePath)) return saveImagePath; - RequestResult coverResult = downloadClient.MakeRequest(url, requestType); + RequestResult coverResult = downloadClient.MakeRequest(url, requestType, referrer); using MemoryStream ms = new(); coverResult.result.CopyTo(ms); Directory.CreateDirectory(TrangaSettings.coverImageCache); diff --git a/Tranga/MangaConnectors/MangaConnectorJsonConverter.cs b/Tranga/MangaConnectors/MangaConnectorJsonConverter.cs index f54dbe39..1baf08f8 100644 --- a/Tranga/MangaConnectors/MangaConnectorJsonConverter.cs +++ b/Tranga/MangaConnectors/MangaConnectorJsonConverter.cs @@ -39,6 +39,7 @@ public override object ReadJson(JsonReader reader, Type objectType, object? exis "MangaHere" => this._connectors.First(c => c is MangaHere), "AsuraToon" => this._connectors.First(c => c is AsuraToon), "Weebcentral" => this._connectors.First(c => c is Weebcentral), + "Webtoons" => this._connectors.First(c => c is Webtoons), _ => throw new UnreachableException($"Could not find Connector with name {connectorName}") }; } diff --git a/Tranga/MangaConnectors/Webtoons.cs b/Tranga/MangaConnectors/Webtoons.cs new file mode 100644 index 00000000..238bf51b --- /dev/null +++ b/Tranga/MangaConnectors/Webtoons.cs @@ -0,0 +1,272 @@ +using System.Net; +using System.Text.RegularExpressions; +using HtmlAgilityPack; +using Tranga.Jobs; + +namespace Tranga.MangaConnectors; + +public class Webtoons : MangaConnector +{ + + public Webtoons(GlobalBase clone) : base(clone, "Webtoons", ["en"]) + { + this.downloadClient = new HttpDownloadClient(clone); + } + + // Done + public override Manga[] GetManga(string publicationTitle = "") + { + string sanitizedTitle = string.Join(' ', Regex.Matches(publicationTitle, "[A-z]*").Where(m => m.Value.Length > 0)).ToLower(); + Log($"Searching Publications. Term=\"{publicationTitle}\""); + string requestUrl = $"https://www.webtoons.com/en/search?keyword={sanitizedTitle}&searchType=WEBTOON"; + RequestResult requestResult = + downloadClient.MakeRequest(requestUrl, RequestType.Default); + if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300) { + Log($"Failed to retrieve site"); + return Array.Empty(); + } + + if (requestResult.htmlDocument is null) + { + Log($"Failed to retrieve site"); + return Array.Empty(); + } + + Manga[] publications = ParsePublicationsFromHtml(requestResult.htmlDocument); + Log($"Retrieved {publications.Length} publications. Term=\"{publicationTitle}\""); + return publications; + } + + // Done + public override Manga? GetMangaFromId(string publicationId) + { + PublicationManager pb = new PublicationManager(publicationId); + return GetMangaFromUrl($"https://www.webtoons.com/en/{pb.Category}/{pb.Title}/list?title_no={pb.Id}"); + } + + // Done + public override Manga? GetMangaFromUrl(string url) + { + RequestResult requestResult = downloadClient.MakeRequest(url, RequestType.MangaInfo); + if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300) { + return null; + } + if (requestResult.htmlDocument is null) + { + Log($"Failed to retrieve site"); + return null; + } + Regex regex = new Regex(@".*webtoons\.com/en/(?[^/]+)/(?[^/]+)/list\?title_no=(?<id>\d+).*"); + Match match = regex.Match(url); + + if(match.Success) { + PublicationManager pm = new PublicationManager(match.Groups["title"].Value, match.Groups["category"].Value, match.Groups["id"].Value); + return ParseSinglePublicationFromHtml(requestResult.htmlDocument, pm.getPublicationId(), url); + } + Log($"Failed match Regex ID"); + return null; + } + + // Done + private Manga[] ParsePublicationsFromHtml(HtmlDocument document) + { + HtmlNode mangaList = document.DocumentNode.SelectSingleNode("//ul[contains(@class, 'card_lst')]"); + if (!mangaList.ChildNodes.Any(node => node.Name == "li")) { + Log($"Failed to parse publication"); + return Array.Empty<Manga>(); + } + + List<string> urls = document.DocumentNode + .SelectNodes("//ul[contains(@class, 'card_lst')]/li/a") + .Select(node => node.GetAttributeValue("href", "https://www.webtoons.com")) + .ToList(); + + HashSet<Manga> ret = new(); + foreach (string url in urls) + { + Manga? manga = GetMangaFromUrl(url); + if (manga is not null) + ret.Add((Manga)manga); + } + + return ret.ToArray(); + } + + private string capitalizeString(string str = "") { + if(str.Length == 0) return ""; + if(str.Length == 1) return str.ToUpper(); + return char.ToUpper(str[0]) + str.Substring(1).ToLower(); + } + + // Done + private Manga ParseSinglePublicationFromHtml(HtmlDocument document, string publicationId, string websiteUrl) + { + HtmlNode infoNode1 = document.DocumentNode.SelectSingleNode("//*[@id='content']/div[2]/div[1]/div[1]"); + HtmlNode infoNode2 = document.DocumentNode.SelectSingleNode("//*[@id='content']/div[2]/div[2]/div[2]"); + + string sortName = infoNode1.SelectSingleNode(".//h1[contains(@class, 'subj')]").InnerText; + string description = infoNode2.SelectSingleNode(".//p[contains(@class, 'summary')]") + .InnerText.Trim(); + + HtmlNode posterNode = document.DocumentNode.SelectSingleNode("//div[contains(@class, 'detail_body') and contains(@class, 'banner')]"); + + Regex regex = new Regex(@"url\((?<url>.*?)\)"); + Match match = regex.Match(posterNode.GetAttributeValue("style", "")); + + string posterUrl = match.Groups["url"].Value; + string coverFileNameInCache = SaveCoverImageToCache(posterUrl, publicationId, RequestType.MangaCover, websiteUrl); + + string genre = infoNode1.SelectSingleNode(".//h2[contains(@class, 'genre')]") + .InnerText.Trim(); + string[] tags = [ genre ]; + + List<HtmlNode> authorsNodes = infoNode1.SelectSingleNode(".//div[contains(@class, 'author_area')]").Descendants("a").ToList(); + List<string> authors = authorsNodes.Select(node => node.InnerText.Trim()).ToList(); + + string originalLanguage = ""; + + int year = DateTime.Now.Year; + + string status1 = infoNode2.SelectSingleNode(".//p").InnerText; + string status2 = infoNode2.SelectSingleNode(".//p/span").InnerText; + Manga.ReleaseStatusByte releaseStatus = Manga.ReleaseStatusByte.Unreleased; + if(status2.Length == 0 || status1.ToLower() == "completed") { + releaseStatus = Manga.ReleaseStatusByte.Completed; + } else if(status2.ToLower() == "up") { + releaseStatus = Manga.ReleaseStatusByte.Continuing; + } + + Manga manga = new(sortName, authors, description, new Dictionary<string, string>(), tags, posterUrl, coverFileNameInCache, new Dictionary<string, string>(), + year, originalLanguage, publicationId, releaseStatus, websiteUrl: websiteUrl); + AddMangaToCache(manga); + return manga; + } + + // Done + public override Chapter[] GetChapters(Manga manga, string language = "en") + { + PublicationManager pm = new PublicationManager(manga.publicationId); + string requestUrl = $"https://www.webtoons.com/en/{pm.Category}/{pm.Title}/list?title_no={pm.Id}"; + // Leaving this in for verification if the page exists + RequestResult requestResult = + downloadClient.MakeRequest(requestUrl, RequestType.Default); + if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300) + return Array.Empty<Chapter>(); + + // Get number of pages + int pages = requestResult.htmlDocument.DocumentNode.SelectSingleNode("//div[contains(@class, 'paginate')]").ChildNodes.ToArray().Length; + List<Chapter> chapters = new List<Chapter>(); + + for(int page = 1; page <= pages; page++) { + string pageRequestUrl = $"{requestUrl}&page={page}"; + + chapters.AddRange(ParseChaptersFromHtml(manga, pageRequestUrl)); + } + + Log($"Got {chapters.Count} chapters. {manga}"); + return chapters.Order().ToArray(); + } + + // Done + private List<Chapter> ParseChaptersFromHtml(Manga manga, string mangaUrl) + { + RequestResult result = downloadClient.MakeRequest(mangaUrl, RequestType.Default); + if ((int)result.statusCode < 200 || (int)result.statusCode >= 300 || result.htmlDocument is null) + { + Log("Failed to load site"); + return new List<Chapter>(); + } + + List<Chapter> ret = new(); + + foreach (HtmlNode chapterInfo in result.htmlDocument.DocumentNode.SelectNodes("//ul/li[contains(@class, '_episodeItem')]")) + { + HtmlNode infoNode = chapterInfo.SelectSingleNode(".//a"); + string url = infoNode.GetAttributeValue("href", ""); + + string id = chapterInfo.GetAttributeValue("id", ""); + if(id == "") continue; + string? volumeNumber = null; + string chapterNumber = chapterInfo.GetAttributeValue("data-episode-no", ""); + if(chapterNumber == "") continue; + string chapterName = infoNode.SelectSingleNode(".//span[contains(@class, 'subj')]/span").InnerText.Trim(); + ret.Add(new Chapter(manga, chapterName, volumeNumber, chapterNumber, url)); + } + + return ret; + } + + public override HttpStatusCode DownloadChapter(Chapter chapter, ProgressToken? progressToken = null) + { + if (progressToken?.cancellationRequested ?? false) + { + progressToken.Cancel(); + return HttpStatusCode.RequestTimeout; + } + + Manga chapterParentManga = chapter.parentManga; + Log($"Retrieving chapter-info {chapter} {chapterParentManga}"); + string requestUrl = chapter.url; + // Leaving this in to check if the page exists + RequestResult requestResult = + downloadClient.MakeRequest(requestUrl, RequestType.Default); + if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300) + { + progressToken?.Cancel(); + return requestResult.statusCode; + } + + string[] imageUrls = ParseImageUrlsFromHtml(requestUrl); + return DownloadChapterImages(imageUrls, chapter, RequestType.MangaImage, progressToken:progressToken, referrer: requestUrl); + } + + private string[] ParseImageUrlsFromHtml(string mangaUrl) + { + RequestResult requestResult = + downloadClient.MakeRequest(mangaUrl, RequestType.Default); + if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300) + { + return Array.Empty<string>(); + } + if (requestResult.htmlDocument is null) + { + Log($"Failed to retrieve site"); + return Array.Empty<string>(); + } + + return requestResult.htmlDocument.DocumentNode + .SelectNodes("//*[@id='_imageList']/img") + .Select(node => + node.GetAttributeValue("data-url", "")) + .ToArray(); + } +} + +internal class PublicationManager { + public PublicationManager(string title = "", string category = "", string id = "") { + this.Title = title; + this.Category = category; + this.Id = id; + } + + public PublicationManager(string publicationId) { + string[] parts = publicationId.Split("|"); + if(parts.Length == 3) { + this.Title = parts[0]; + this.Category = parts[1]; + this.Id = parts[2]; + } else { + this.Title = ""; + this.Category = ""; + this.Id = ""; + } + } + + public string getPublicationId() { + return $"{this.Title}|{this.Category}|{this.Id}"; + } + + public string Title { get; set; } + public string Category { get; set; } + public string Id { get; set; } +} \ No newline at end of file diff --git a/Tranga/Tranga.cs b/Tranga/Tranga.cs index 3292e6d8..7384a785 100644 --- a/Tranga/Tranga.cs +++ b/Tranga/Tranga.cs @@ -27,6 +27,7 @@ public Tranga(Logger? logger) : base(logger) new MangaHere(this), new AsuraToon(this), new Weebcentral(this) + new Webtoons(this), }; foreach(DirectoryInfo dir in new DirectoryInfo(Path.GetTempPath()).GetDirectories("trangatemp"))//Cleanup old temp folders dir.Delete(); From 7f13d9b1e62eb237883f905010a92c7f6e9c3abe Mon Sep 17 00:00:00 2001 From: Makhuta <matykubla@gmail.com> Date: Thu, 6 Feb 2025 15:39:06 +0100 Subject: [PATCH 26/34] Fix - forgotten comma --- Tranga/Tranga.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tranga/Tranga.cs b/Tranga/Tranga.cs index 7384a785..260f8e86 100644 --- a/Tranga/Tranga.cs +++ b/Tranga/Tranga.cs @@ -26,7 +26,7 @@ public Tranga(Logger? logger) : base(logger) new ManhuaPlus(this), new MangaHere(this), new AsuraToon(this), - new Weebcentral(this) + new Weebcentral(this), new Webtoons(this), }; foreach(DirectoryInfo dir in new DirectoryInfo(Path.GetTempPath()).GetDirectories("trangatemp"))//Cleanup old temp folders From 5a864ab9b7ef477175af005831a2cc5ed16de032 Mon Sep 17 00:00:00 2001 From: Glax <johanna@bernloehr.eu> Date: Sun, 9 Feb 2025 17:27:35 +0100 Subject: [PATCH 27/34] Remove Manga4Life --- .../MangaConnectorJsonConverter.cs | 1 - Tranga/MangaConnectors/MangaLife.cs | 206 ------------------ Tranga/Tranga.cs | 1 - 3 files changed, 208 deletions(-) delete mode 100644 Tranga/MangaConnectors/MangaLife.cs diff --git a/Tranga/MangaConnectors/MangaConnectorJsonConverter.cs b/Tranga/MangaConnectors/MangaConnectorJsonConverter.cs index f54dbe39..eb5c9d80 100644 --- a/Tranga/MangaConnectors/MangaConnectorJsonConverter.cs +++ b/Tranga/MangaConnectors/MangaConnectorJsonConverter.cs @@ -34,7 +34,6 @@ public override object ReadJson(JsonReader reader, Type objectType, object? exis "MangaKatana" => this._connectors.First(c => c is MangaKatana), "Mangaworld" => this._connectors.First(c => c is Mangaworld), "Bato" => this._connectors.First(c => c is Bato), - "Manga4Life" => this._connectors.First(c => c is MangaLife), "ManhuaPlus" => this._connectors.First(c => c is ManhuaPlus), "MangaHere" => this._connectors.First(c => c is MangaHere), "AsuraToon" => this._connectors.First(c => c is AsuraToon), diff --git a/Tranga/MangaConnectors/MangaLife.cs b/Tranga/MangaConnectors/MangaLife.cs deleted file mode 100644 index 66f2d209..00000000 --- a/Tranga/MangaConnectors/MangaLife.cs +++ /dev/null @@ -1,206 +0,0 @@ -using System.Net; -using System.Text.RegularExpressions; -using HtmlAgilityPack; -using Tranga.Jobs; - -namespace Tranga.MangaConnectors; - -public class MangaLife : MangaConnector -{ - public MangaLife(GlobalBase clone) : base(clone, "Manga4Life", ["en"]) - { - this.downloadClient = new ChromiumDownloadClient(clone); - } - - public override Manga[] GetManga(string publicationTitle = "") - { - Log($"Searching Publications. Term=\"{publicationTitle}\""); - string sanitizedTitle = WebUtility.UrlEncode(publicationTitle); - string requestUrl = $"https://manga4life.com/search/?name={sanitizedTitle}"; - RequestResult requestResult = - downloadClient.MakeRequest(requestUrl, RequestType.Default); - if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300) - return Array.Empty<Manga>(); - - if (requestResult.htmlDocument is null) - return Array.Empty<Manga>(); - Manga[] publications = ParsePublicationsFromHtml(requestResult.htmlDocument); - Log($"Retrieved {publications.Length} publications. Term=\"{publicationTitle}\""); - return publications; - } - - public override Manga? GetMangaFromId(string publicationId) - { - return GetMangaFromUrl($"https://manga4life.com/manga/{publicationId}"); - } - - public override Manga? GetMangaFromUrl(string url) - { - Regex publicationIdRex = new(@"https:\/\/(www\.)?manga4life.com\/manga\/(.*)(\/.*)*"); - string publicationId = publicationIdRex.Match(url).Groups[2].Value; - - RequestResult requestResult = this.downloadClient.MakeRequest(url, RequestType.MangaInfo); - if(requestResult.htmlDocument is not null) - return ParseSinglePublicationFromHtml(requestResult.htmlDocument, publicationId, url); - return null; - } - - private Manga[] ParsePublicationsFromHtml(HtmlDocument document) - { - HtmlNode resultsNode = document.DocumentNode.SelectSingleNode("//div[@class='BoxBody']/div[last()]/div[1]/div"); - if (resultsNode.Descendants("div").Count() == 1 && resultsNode.Descendants("div").First().HasClass("NoResults")) - { - Log("No results."); - return Array.Empty<Manga>(); - } - Log($"{resultsNode.SelectNodes("div").Count} items."); - - HashSet<Manga> ret = new(); - - foreach (HtmlNode resultNode in resultsNode.SelectNodes("div")) - { - string url = resultNode.Descendants().First(d => d.HasClass("SeriesName")).GetAttributeValue("href", ""); - Manga? manga = GetMangaFromUrl($"https://manga4life.com{url}"); - if (manga is not null) - ret.Add((Manga)manga); - } - - return ret.ToArray(); - } - - - private Manga ParseSinglePublicationFromHtml(HtmlDocument document, string publicationId, string websiteUrl) - { - string originalLanguage = "", status = ""; - Dictionary<string, string> altTitles = new(), links = new(); - HashSet<string> tags = new(); - Manga.ReleaseStatusByte releaseStatus = Manga.ReleaseStatusByte.Unreleased; - - HtmlNode posterNode = document.DocumentNode.SelectSingleNode("//div[@class='BoxBody']//div[@class='row']//img"); - string posterUrl = posterNode.GetAttributeValue("src", ""); - string coverFileNameInCache = SaveCoverImageToCache(posterUrl, publicationId, RequestType.MangaCover); - - HtmlNode titleNode = document.DocumentNode.SelectSingleNode("//div[@class='BoxBody']//div[@class='row']//h1"); - string sortName = titleNode.InnerText; - - HtmlNode[] authorsNodes = document.DocumentNode - .SelectNodes("//div[@class='BoxBody']//div[@class='row']//span[text()='Author(s):']/..").Descendants("a") - .ToArray(); - List<string> authors = new(); - foreach (HtmlNode authorNode in authorsNodes) - authors.Add(authorNode.InnerText); - - HtmlNode[] genreNodes = document.DocumentNode - .SelectNodes("//div[@class='BoxBody']//div[@class='row']//span[text()='Genre(s):']/..").Descendants("a") - .ToArray(); - foreach (HtmlNode genreNode in genreNodes) - tags.Add(genreNode.InnerText); - - HtmlNode yearNode = document.DocumentNode - .SelectNodes("//div[@class='BoxBody']//div[@class='row']//span[text()='Released:']/..").Descendants("a") - .First(); - int year = Convert.ToInt32(yearNode.InnerText); - - HtmlNode[] statusNodes = document.DocumentNode - .SelectNodes("//div[@class='BoxBody']//div[@class='row']//span[text()='Status:']/..").Descendants("a") - .ToArray(); - foreach (HtmlNode statusNode in statusNodes) - if (statusNode.InnerText.Contains("publish", StringComparison.CurrentCultureIgnoreCase)) - status = statusNode.InnerText.Split(' ')[0]; - switch (status.ToLower()) - { - case "cancelled": releaseStatus = Manga.ReleaseStatusByte.Cancelled; break; - case "hiatus": releaseStatus = Manga.ReleaseStatusByte.OnHiatus; break; - case "discontinued": releaseStatus = Manga.ReleaseStatusByte.Cancelled; break; - case "complete": releaseStatus = Manga.ReleaseStatusByte.Completed; break; - case "ongoing": releaseStatus = Manga.ReleaseStatusByte.Continuing; break; - } - - HtmlNode descriptionNode = document.DocumentNode - .SelectNodes("//div[@class='BoxBody']//div[@class='row']//span[text()='Description:']/..") - .Descendants("div").First(); - string description = descriptionNode.InnerText; - - Manga manga = new(sortName, authors.ToList(), description, altTitles, tags.ToArray(), posterUrl, - coverFileNameInCache, links, year, originalLanguage, publicationId, releaseStatus, websiteUrl: websiteUrl); - AddMangaToCache(manga); - return manga; - } - - public override Chapter[] GetChapters(Manga manga, string language="en") - { - Log($"Getting chapters {manga}"); - RequestResult result = downloadClient.MakeRequest($"https://manga4life.com/manga/{manga.publicationId}", RequestType.Default, clickButton:"[class*='ShowAllChapters']"); - if ((int)result.statusCode < 200 || (int)result.statusCode >= 300 || result.htmlDocument is null) - { - return Array.Empty<Chapter>(); - } - - HtmlNodeCollection chapterNodes = result.htmlDocument.DocumentNode.SelectNodes( - "//a[contains(concat(' ',normalize-space(@class),' '),' ChapterLink ')]"); - string[] urls = chapterNodes.Select(node => node.GetAttributeValue("href", "")).ToArray(); - Regex urlRex = new (@"-chapter-([0-9\\.]+)(-index-([0-9\\.]+))?"); - - List<Chapter> chapters = new(); - foreach (string url in urls) - { - Match rexMatch = urlRex.Match(url); - - string volumeNumber = "1"; - if (rexMatch.Groups[3].Value.Length > 0) - volumeNumber = rexMatch.Groups[3].Value; - string chapterNumber = rexMatch.Groups[1].Value; - string fullUrl = $"https://manga4life.com{url}"; - fullUrl = fullUrl.Replace(Regex.Match(url,"(-page-[0-9])").Value,""); - try - { - chapters.Add(new Chapter(manga, "", volumeNumber, chapterNumber, fullUrl)); - } - catch (Exception e) - { - Log($"Failed to load chapter {chapterNumber}: {e.Message}"); - } - } - //Return Chapters ordered by Chapter-Number - Log($"Got {chapters.Count} chapters. {manga}"); - return chapters.Order().ToArray(); - } - - public override HttpStatusCode DownloadChapter(Chapter chapter, ProgressToken? progressToken = null) - { - if (progressToken?.cancellationRequested ?? false) - { - progressToken.Cancel(); - return HttpStatusCode.RequestTimeout; - } - - Manga chapterParentManga = chapter.parentManga; - if (progressToken?.cancellationRequested ?? false) - { - progressToken.Cancel(); - return HttpStatusCode.RequestTimeout; - } - - Log($"Retrieving chapter-info {chapter} {chapterParentManga}"); - - RequestResult requestResult = this.downloadClient.MakeRequest(chapter.url, RequestType.Default); - if (requestResult.htmlDocument is null) - { - progressToken?.Cancel(); - return HttpStatusCode.RequestTimeout; - } - - HtmlDocument document = requestResult.htmlDocument; - - HtmlNode gallery = document.DocumentNode.Descendants("div").First(div => div.HasClass("ImageGallery")); - HtmlNode[] images = gallery.Descendants("img").Where(img => img.HasClass("img-fluid")).ToArray(); - List<string> urls = new(); - foreach(HtmlNode galleryImage in images) - urls.Add(galleryImage.GetAttributeValue("src", "")); - - string comicInfoPath = Path.GetTempFileName(); - File.WriteAllText(comicInfoPath, chapter.GetComicInfoXmlString()); - - return DownloadChapterImages(urls.ToArray(), chapter, RequestType.MangaImage, progressToken:progressToken); - } -} \ No newline at end of file diff --git a/Tranga/Tranga.cs b/Tranga/Tranga.cs index 3292e6d8..cb20fa05 100644 --- a/Tranga/Tranga.cs +++ b/Tranga/Tranga.cs @@ -22,7 +22,6 @@ public Tranga(Logger? logger) : base(logger) new MangaKatana(this), new Mangaworld(this), new Bato(this), - new MangaLife(this), new ManhuaPlus(this), new MangaHere(this), new AsuraToon(this), From d97da26994f46e3fc55cc7df288ebc3e29458152 Mon Sep 17 00:00:00 2001 From: Glax <johanna@bernloehr.eu> Date: Sun, 9 Feb 2025 17:37:53 +0100 Subject: [PATCH 28/34] spelling error --- Tranga/MangaConnectors/MangaConnector.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tranga/MangaConnectors/MangaConnector.cs b/Tranga/MangaConnectors/MangaConnector.cs index 5529b451..7573cbca 100644 --- a/Tranga/MangaConnectors/MangaConnector.cs +++ b/Tranga/MangaConnectors/MangaConnector.cs @@ -213,7 +213,7 @@ protected HttpStatusCode DownloadChapterImages(string[] imageUrls, Chapter chapt return HttpStatusCode.OK; } - protected string SaveCoverImageToCache(string url, string mangaInternalId, RequestType requestType, string? refferer = null) + protected string SaveCoverImageToCache(string url, string mangaInternalId, RequestType requestType, string? referrer = null) { Regex urlRex = new (@"https?:\/\/((?:[a-zA-Z0-9-]+\.)+[a-zA-Z0-9]+)\/(?:.+\/)*(.+\.([a-zA-Z]+))"); //https?:\/\/[a-zA-Z0-9-]+\.([a-zA-Z0-9-]+\.[a-zA-Z0-9]+)\/(?:.+\/)*(.+\.([a-zA-Z]+)) for only second level domains From edacaaba8a8d55889cb0bcffb8e7ccc13de3944b Mon Sep 17 00:00:00 2001 From: Glax <johanna@bernloehr.eu> Date: Sun, 9 Feb 2025 17:38:54 +0100 Subject: [PATCH 29/34] Update Readme --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index 01514523..3f81f44d 100644 --- a/README.md +++ b/README.md @@ -45,14 +45,13 @@ Tranga can download Chapters and Metadata from "Scanlation" sites such as - [MangaDex.org](https://mangadex.org/) (Multilingual) - [Manganato.com](https://manganato.com/) (en) -- [Mangasee.com](https://mangasee123.com/) (en) - [MangaKatana.com](https://mangakatana.com) (en) - [Mangaworld.bz](https://www.mangaworld.bz/) (it) - [Bato.to](https://bato.to/v3x) (en) -- [Manga4Life](https://manga4life.com) (en) - [ManhuaPlus](https://manhuaplus.org/) (en) - [MangaHere](https://www.mangahere.cc/) (en) (Their covers aren't scrapeable.) - [Weebcentral](https://weebcentral.com) (en) +- [Webtoons](https://www.webtoons.com/en/) - ❓ Open an [issue](https://github.com/C9Glax/tranga/issues/new?assignees=&labels=New+Connector&projects=&template=new_connector.yml&title=%5BNew+Connector%5D%3A+) and trigger a library-scan with [Komga](https://komga.org/) and [Kavita](https://www.kavitareader.com/). From bb8a5148304e6859dca224d6d7509ba6c7eae9c1 Mon Sep 17 00:00:00 2001 From: Glax <johanna@bernloehr.eu> Date: Sun, 9 Feb 2025 17:57:08 +0100 Subject: [PATCH 30/34] Do not create .duplicate files anymore. Just warn in log and delete (or attempt to delete) --- Tranga/Jobs/JobBoss.cs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Tranga/Jobs/JobBoss.cs b/Tranga/Jobs/JobBoss.cs index 4631a667..4ecf2e2c 100644 --- a/Tranga/Jobs/JobBoss.cs +++ b/Tranga/Jobs/JobBoss.cs @@ -167,9 +167,10 @@ private void LoadJobsList(HashSet<MangaConnector> connectors) Log($"Adding Job {job}"); if (!AddJob(job, file.FullName)) //If we detect a duplicate, delete the file. { - string path = string.Concat(file.FullName, ".duplicate"); - file.MoveTo(path); - Log($"Duplicate detected or otherwise not able to add job to list.\nMoved job {job} to {path}"); + //string path = string.Concat(file.FullName, ".duplicate"); + //file.MoveTo(path); + //Log($"Duplicate detected or otherwise not able to add job to list.\nMoved job {job} to {path}"); + Log($"Duplicate detected or otherwise not able to add job to list. Removed the file {file.FullName} {job}"); } } catch (Exception e) From 603e1b41d9c1832771ee53607a77d8d5b27349bf Mon Sep 17 00:00:00 2001 From: Glax <johanna@bernloehr.eu> Date: Sun, 9 Feb 2025 18:37:33 +0100 Subject: [PATCH 31/34] Add Chromium referer header --- Tranga/MangaConnectors/ChromiumDownloadClient.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/Tranga/MangaConnectors/ChromiumDownloadClient.cs b/Tranga/MangaConnectors/ChromiumDownloadClient.cs index 5c000057..86ac8818 100644 --- a/Tranga/MangaConnectors/ChromiumDownloadClient.cs +++ b/Tranga/MangaConnectors/ChromiumDownloadClient.cs @@ -73,6 +73,7 @@ private RequestResult MakeRequestBrowser(string url, string? referrer = null, st return new RequestResult(HttpStatusCode.InternalServerError, null, Stream.Null); IPage page = _browser.NewPageAsync().Result; page.DefaultTimeout = TrangaSettings.ChromiumPageTimeoutMs; + page.SetExtraHttpHeadersAsync(new() { { "Referer", referrer } }); IResponse response; try { From a95cb90561e1582ddb6846f7e2417090d0ef5da0 Mon Sep 17 00:00:00 2001 From: Glax <johanna@bernloehr.eu> Date: Sun, 9 Feb 2025 18:37:41 +0100 Subject: [PATCH 32/34] Update Nuget packages --- Tranga/Tranga.csproj | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Tranga/Tranga.csproj b/Tranga/Tranga.csproj index 1ca1c49f..6447817a 100644 --- a/Tranga/Tranga.csproj +++ b/Tranga/Tranga.csproj @@ -10,9 +10,9 @@ <ItemGroup> <PackageReference Include="GlaxArguments" Version="1.1.0" /> - <PackageReference Include="HtmlAgilityPack" Version="1.11.71" /> + <PackageReference Include="HtmlAgilityPack" Version="1.11.72" /> <PackageReference Include="Newtonsoft.Json" Version="13.0.3" /> - <PackageReference Include="PuppeteerSharp" Version="20.0.5" /> + <PackageReference Include="PuppeteerSharp" Version="20.1.0" /> <PackageReference Include="Soenneker.Utils.String.NeedlemanWunsch" Version="2.1.301" /> </ItemGroup> From e6d40a7b36df8f881fd8119731d34b0223925570 Mon Sep 17 00:00:00 2001 From: Glax <johanna@bernloehr.eu> Date: Sun, 9 Feb 2025 18:37:55 +0100 Subject: [PATCH 33/34] Remove unused code from Weebcentral --- Tranga/MangaConnectors/WeebCentral.cs | 40 ++------------------------- 1 file changed, 3 insertions(+), 37 deletions(-) diff --git a/Tranga/MangaConnectors/WeebCentral.cs b/Tranga/MangaConnectors/WeebCentral.cs index 50d63c47..c5d355f0 100644 --- a/Tranga/MangaConnectors/WeebCentral.cs +++ b/Tranga/MangaConnectors/WeebCentral.cs @@ -43,7 +43,7 @@ public override Manga[] GetManga(string publicationTitle = "") private Manga[] ParsePublicationsFromHtml(HtmlDocument document) { if (document.DocumentNode.SelectNodes("//article") == null) - return Array.Empty<Manga>(); + return []; var urls = document.DocumentNode.SelectNodes("/html/body/article/a[@class='link link-hover']") .Select(elem => elem.GetAttributeValue("href", "")).ToList(); @@ -127,33 +127,6 @@ private Manga ParseSinglePublicationFromHtml(HtmlDocument document, string publi return GetMangaFromUrl($"https://weebcentral.com/series/{publicationId}"); } - private string ToFilteredString(string input) - { - return string.Join(' ', input.ToLower().Split(' ').Where(word => _filterWords.Contains(word) == false)); - } - - private SearchResult[] FilteredResults(string publicationTitle, SearchResult[] unfilteredSearchResults) - { - Dictionary<SearchResult, int> similarity = new(); - foreach (var sr in unfilteredSearchResults) - { - List<int> scores = new(); - var filteredPublicationString = ToFilteredString(publicationTitle); - var filteredSString = ToFilteredString(sr.s); - scores.Add(NeedlemanWunschStringUtil.CalculateSimilarity(filteredSString, filteredPublicationString)); - foreach (var srA in sr.a) - { - var filteredAString = ToFilteredString(srA); - scores.Add(NeedlemanWunschStringUtil.CalculateSimilarity(filteredAString, filteredPublicationString)); - } - - similarity.Add(sr, scores.Sum() / scores.Count); - } - - var ret = similarity.OrderBy(s => s.Value).Take(10).Select(s => s.Key).ToList(); - return ret.ToArray(); - } - public override Chapter[] GetChapters(Manga manga, string language = "en") { Log($"Getting chapters {manga}"); @@ -161,11 +134,11 @@ public override Chapter[] GetChapters(Manga manga, string language = "en") var requestResult = downloadClient.MakeRequest(requestUrl, RequestType.Default); if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300) - return Array.Empty<Chapter>(); + return []; //Return Chapters ordered by Chapter-Number if (requestResult.htmlDocument is null) - return Array.Empty<Chapter>(); + return []; var chapters = ParseChaptersFromHtml(manga, requestResult.htmlDocument); Log($"Got {chapters.Count} chapters. {manga}"); return chapters.Order().ToArray(); @@ -233,11 +206,4 @@ public override HttpStatusCode DownloadChapter(Chapter chapter, ProgressToken? p return DownloadChapterImages(urls, chapter, RequestType.MangaImage, progressToken: progressToken); } - - private struct SearchResult - { - public string i { get; set; } - public string s { get; set; } - public string[] a { get; set; } - } } \ No newline at end of file From 4ad314952352a55bc73fb60c70f9d10582d5f2a6 Mon Sep 17 00:00:00 2001 From: Makhuta <matykubla@gmail.com> Date: Tue, 11 Feb 2025 20:16:30 +0100 Subject: [PATCH 34/34] Fix - fixed when parsing chapters the pages was incorrectly parsed resulting into adding the chapters from the last page multiple times (was still downloading OK but it would try to download the chapters from last page multiple times) --- Tranga/MangaConnectors/Webtoons.cs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Tranga/MangaConnectors/Webtoons.cs b/Tranga/MangaConnectors/Webtoons.cs index 238bf51b..9e78290a 100644 --- a/Tranga/MangaConnectors/Webtoons.cs +++ b/Tranga/MangaConnectors/Webtoons.cs @@ -154,15 +154,16 @@ public override Chapter[] GetChapters(Manga manga, string language = "en") return Array.Empty<Chapter>(); // Get number of pages - int pages = requestResult.htmlDocument.DocumentNode.SelectSingleNode("//div[contains(@class, 'paginate')]").ChildNodes.ToArray().Length; + int pages = requestResult.htmlDocument.DocumentNode + .SelectNodes("//div[contains(@class, 'paginate')]/a") + .ToList() + .Count; List<Chapter> chapters = new List<Chapter>(); for(int page = 1; page <= pages; page++) { string pageRequestUrl = $"{requestUrl}&page={page}"; - chapters.AddRange(ParseChaptersFromHtml(manga, pageRequestUrl)); } - Log($"Got {chapters.Count} chapters. {manga}"); return chapters.Order().ToArray(); }