Merge pull request #342 from C9Glax/cuttingedge

Cuttingedge master merge

Dockerfile

@@ -5,6 +5,8 @@ FROM --platform=$TARGETPLATFORM mcr.microsoft.com/dotnet/runtime:$DOTNET AS base
 WORKDIR /publish
 ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true
 ENV PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium
+ENV XDG_CONFIG_HOME=/tmp/.chromium
+ENV XDG_CACHE_HOME=/tmp/.chromium
 RUN apt-get update \
   && apt-get install -y libx11-6 libx11-xcb1 libatk1.0-0 libgtk-3-0 libcups2 libdrm2 libxkbcommon0 libxcomposite1 libxdamage1 libxrandr2 libgbm1 libpango-1.0-0 libcairo2 libasound2 libxshmfence1 libnss3 chromium \
   && apt-get autopurge -y \

README.md

@@ -45,13 +45,13 @@ Tranga can download Chapters and Metadata from "Scanlation" sites such as
 
 - [MangaDex.org](https://mangadex.org/) (Multilingual)
 - [Manganato.com](https://manganato.com/) (en)
-- [Mangasee.com](https://mangasee123.com/) (en)
 - [MangaKatana.com](https://mangakatana.com) (en)
 - [Mangaworld.bz](https://www.mangaworld.bz/) (it)
 - [Bato.to](https://bato.to/v3x) (en)
-- [Manga4Life](https://manga4life.com) (en)
 - [ManhuaPlus](https://manhuaplus.org/) (en)
 - [MangaHere](https://www.mangahere.cc/) (en) (Their covers aren't scrapeable.)
+- [Weebcentral](https://weebcentral.com) (en)
+- [Webtoons](https://www.webtoons.com/en/)
 - ❓ Open an [issue](https://github.com/C9Glax/tranga/issues/new?assignees=&labels=New+Connector&projects=&template=new_connector.yml&title=%5BNew+Connector%5D%3A+)
 
 and trigger a library-scan with [Komga](https://komga.org/) and [Kavita](https://www.kavitareader.com/).  

Tranga/Chapter.cs

@@ -14,22 +14,28 @@ namespace Tranga;
     // ReSharper disable once MemberCanBePrivate.Global
     public Manga parentManga { get; }
     public string? name { get; }
-    public string volumeNumber { get; }
-    public string chapterNumber { get; }
+    public float volumeNumber { get; }
+    public float chapterNumber { get; }
     public string url { get; }
     // ReSharper disable once MemberCanBePrivate.Global
     public string fileName { get; }
     public string? id { get; }
 
     private static readonly Regex LegalCharacters = new (@"([A-z]*[0-9]* *\.*-*,*\]*\[*'*\'*\)*\(*~*!*)*");
     private static readonly Regex IllegalStrings = new(@"(Vol(ume)?|Ch(apter)?)\.?", RegexOptions.IgnoreCase);
-    private static readonly Regex Digits = new(@"[0-9\.]*");
+
     public Chapter(Manga parentManga, string? name, string? volumeNumber, string chapterNumber, string url, string? id = null)
+        : this(parentManga, name, float.Parse(volumeNumber??"0", GlobalBase.numberFormatDecimalPoint),
+            float.Parse(chapterNumber, GlobalBase.numberFormatDecimalPoint), url, id)
+    {
+    }
+
+    public Chapter(Manga parentManga, string? name, float? volumeNumber, float chapterNumber, string url, string? id = null)
     {
         this.parentManga = parentManga;
         this.name = name;
-        this.volumeNumber = volumeNumber is not null ? string.Concat(Digits.Matches(volumeNumber).Select(x => x.Value)) : "0";
-        this.chapterNumber = string.Concat(Digits.Matches(chapterNumber).Select(x => x.Value));
+        this.volumeNumber = volumeNumber??0;
+        this.chapterNumber = chapterNumber;
         this.url = url;
         this.id = id;
 
@@ -60,26 +66,12 @@ public int CompareTo(object? obj)
     {
         if(obj is not Chapter otherChapter)
             throw new ArgumentException($"{obj} can not be compared to {this}");
-
-        if (float.TryParse(volumeNumber, GlobalBase.numberFormatDecimalPoint, out float volumeNumberFloat) &&
-            float.TryParse(chapterNumber, GlobalBase.numberFormatDecimalPoint, out float chapterNumberFloat) &&
-            float.TryParse(otherChapter.volumeNumber, GlobalBase.numberFormatDecimalPoint,
-                out float otherVolumeNumberFloat) &&
-            float.TryParse(otherChapter.chapterNumber, GlobalBase.numberFormatDecimalPoint,
-                out float otherChapterNumberFloat))
+        return volumeNumber.CompareTo(otherChapter.volumeNumber) switch
         {
-            return volumeNumberFloat.CompareTo(otherVolumeNumberFloat) switch
-            {
-                <0 => -1,
-                >0 => 1,
-                _ => chapterNumberFloat.CompareTo(otherChapterNumberFloat)
-            };
-        }
-        else throw new FormatException($"Value could not be parsed.\n" +
-                                       $"\tVolumeNumber: '{volumeNumber}' ChapterNumber: '{chapterNumber}'\n" +
-                                       $"\tOther-VolumeNumber: '{otherChapter.volumeNumber}' Other-ChapterNumber: '{otherChapter.chapterNumber}'\n" +
-                                       $"\t{this}\n" +
-                                       $"\t{otherChapter}");
+            <0 => -1,
+            >0 => 1,
+            _ => chapterNumber.CompareTo(otherChapter.chapterNumber)
+        };
     }
 
     /// <summary>
@@ -111,9 +103,10 @@ internal bool CheckChapterIsDownloaded()
             {
                 Match m = volChRex.Match(archive.Name);
                 if (m.Groups[1].Success)
-                    return m.Groups[1].Value == t.volumeNumber && m.Groups[2].Value == t.chapterNumber;
+                    return m.Groups[1].Value == t.volumeNumber.ToString(GlobalBase.numberFormatDecimalPoint) &&
+                           m.Groups[2].Value == t.chapterNumber.ToString(GlobalBase.numberFormatDecimalPoint);
                 else
-                    return m.Groups[2].Value == t.chapterNumber;
+                    return m.Groups[2].Value == t.chapterNumber.ToString(GlobalBase.numberFormatDecimalPoint);
             });
         }
 

Tranga/Jobs/DownloadNewChapters.cs

@@ -54,6 +54,6 @@ public override bool Equals(object? obj)
         if (obj is not DownloadNewChapters otherJob)
             return false;
         return otherJob.mangaConnector == this.mangaConnector &&
-               otherJob.manga.Equals(this.manga);
+               otherJob.manga.publicationId == this.manga.publicationId;
     }
 }

Tranga/Jobs/JobBoss.cs

@@ -1,4 +1,5 @@
-using System.Runtime.InteropServices;
+using System.Diagnostics;
+using System.Runtime.InteropServices;
 using System.Text.RegularExpressions;
 using Newtonsoft.Json;
 using Tranga.MangaConnectors;
@@ -70,7 +71,7 @@ public void RemoveJobs(IEnumerable<Job?> jobsToRemove)
                 RemoveJob(job);
     }
 
-    public IEnumerable<Job> GetJobsLike(string? connectorName = null, string? internalId = null, string? chapterNumber = null)
+    public IEnumerable<Job> GetJobsLike(string? connectorName = null, string? internalId = null, float? chapterNumber = null)
     {
         IEnumerable<Job> ret = this.jobs;
         if (connectorName is not null)
@@ -82,7 +83,7 @@ public IEnumerable<Job> GetJobsLike(string? connectorName = null, string? intern
                 if (jjob is not DownloadChapter job)
                     return false;
                 return job.chapter.parentManga.internalId == internalId &&
-                       job.chapter.chapterNumber == chapterNumber;
+                       job.chapter.chapterNumber.Equals(chapterNumber);
             });
         else if (internalId is not null)
             ret = ret.Where(jjob =>
@@ -145,34 +146,44 @@ private void AddJobsToQueue(IEnumerable<Job> newJobs)
 
     private void LoadJobsList(HashSet<MangaConnector> connectors)
     {
-        Directory.CreateDirectory(TrangaSettings.jobsFolderPath);
-        if(RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
-            File.SetUnixFileMode(TrangaSettings.jobsFolderPath, UserRead | UserWrite | UserExecute | GroupRead | OtherRead);
         if (!Directory.Exists(TrangaSettings.jobsFolderPath)) //No jobs to load
+        {
+            Directory.CreateDirectory(TrangaSettings.jobsFolderPath);
+            if(RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
+                File.SetUnixFileMode(TrangaSettings.jobsFolderPath, UserRead | UserWrite | UserExecute | GroupRead | OtherRead);
             return;
+        }
 
         //Load json-job-files
         foreach (FileInfo file in Directory.GetFiles(TrangaSettings.jobsFolderPath, "*.json").Select(f => new FileInfo(f)))
         {
             Log($"Adding {file.Name}");
-            Job? job = JsonConvert.DeserializeObject<Job>(File.ReadAllText(file.FullName),
-                new JobJsonConverter(this, new MangaConnectorJsonConverter(this, connectors)));
-            if (job is null)
-            {
-                string newName = file.FullName + ".failed";
-                Log($"Failed loading file {file.Name}.\nMoving to {newName}");
-                File.Move(file.FullName, newName);
-            }
-            else
+            try
             {
+                Job? job = JsonConvert.DeserializeObject<Job>(File.ReadAllText(file.FullName),
+                    new JobJsonConverter(this, new MangaConnectorJsonConverter(this, connectors)));
+                if (job is null) throw new NullReferenceException();
+
                 Log($"Adding Job {job}");
                 if (!AddJob(job, file.FullName)) //If we detect a duplicate, delete the file.
                 {
-                    string path = string.Concat(file.FullName, ".duplicate");
-                    file.MoveTo(path);
-                    Log($"Duplicate detected or otherwise not able to add job to list.\nMoved job {job} to {path}");
+                    //string path = string.Concat(file.FullName, ".duplicate");
+                    //file.MoveTo(path);
+                    //Log($"Duplicate detected or otherwise not able to add job to list.\nMoved job {job} to {path}");
+                    Log($"Duplicate detected or otherwise not able to add job to list. Removed the file {file.FullName} {job}");
                 }
             }
+            catch (Exception e)
+            {
+                if (e is not UnreachableException or NullReferenceException)
+                    throw;
+                Log(e.Message);
+                string newName = file.FullName + ".failed";
+                Log($"Failed loading file {file.Name}.\nMoving to {newName}.\n" +
+                    $"If you think this is a bug, upload contents of the file to the Bugreport!");
+                File.Move(file.FullName, newName);
+                continue;
+            }
         }
 
         //Connect jobs to parent-jobs and add Publications to cache
@@ -197,7 +208,7 @@ private void LoadJobsList(HashSet<MangaConnector> connectors)
     internal void UpdateJobFile(Job job, string? oldFile = null)
     {
         string newJobFilePath = Path.Join(TrangaSettings.jobsFolderPath, $"{job.id}.json");
-        string oldFilePath = Path.Join(TrangaSettings.jobsFolderPath, oldFile??$"{job.id}.json");
+        string oldFilePath = oldFile??Path.Join(TrangaSettings.jobsFolderPath, $"{job.id}.json");
 
         //Delete old file
         if (File.Exists(oldFilePath))

Tranga/Jobs/UpdateMetadata.cs

@@ -71,6 +71,6 @@ public override bool Equals(object? obj)
         if (obj is not UpdateMetadata otherJob)
             return false;
         return otherJob.mangaConnector == this.mangaConnector &&
-               otherJob.manga.Equals(this.manga);
+               otherJob.manga.publicationId == this.manga.publicationId;
     }
 }

Tranga/MangaConnectors/AsuraToon.cs

@@ -10,7 +10,7 @@ public class AsuraToon : MangaConnector
 
 	public AsuraToon(GlobalBase clone) : base(clone, "AsuraToon", ["en"])
 	{
-		this.downloadClient = new HttpDownloadClient(clone);
+		this.downloadClient = new ChromiumDownloadClient(clone);
 	}
 
 	public override Manga[] GetManga(string publicationTitle = "")
@@ -55,8 +55,8 @@ public override Manga[] GetManga(string publicationTitle = "")
 	private Manga[] ParsePublicationsFromHtml(HtmlDocument document)
 	{
 		HtmlNodeCollection mangaList = document.DocumentNode.SelectNodes("//a[starts-with(@href,'series')]");
-		if (mangaList.Count < 1)
-			return Array.Empty<Manga>();
+		if (mangaList is null || mangaList.Count < 1)
+			return [];
 
 		IEnumerable<string> urls = mangaList.Select(a => $"https://asuracomic.net/{a.GetAttributeValue("href", "")}");
 
@@ -102,11 +102,13 @@ private Manga ParseSinglePublicationFromHtml(HtmlDocument document, string publi
 
 		HtmlNode descriptionNode =
 			document.DocumentNode.SelectSingleNode("//h3[starts-with(text(),'Synopsis')]/../span");
-		string description = descriptionNode.InnerText;
+		string description = descriptionNode?.InnerText??"";
 
 		HtmlNodeCollection authorNodes = document.DocumentNode.SelectNodes("//h3[text()='Author']/../h3[not(text()='Author' or text()='_')]");
-		HtmlNodeCollection artistNodes = document.DocumentNode.SelectNodes("//h3[text()='Artist']/../h3[not(text()='Author' or text()='_')]");
-		List<string> authors = authorNodes.Select(a => a.InnerText).Concat(artistNodes.Select(a => a.InnerText)).ToList();
+		HtmlNodeCollection artistNodes = document.DocumentNode.SelectNodes("//h3[text()='Artist']/../h3[not(text()='Artist' or text()='_')]");
+		IEnumerable<string> authorNames = authorNodes is null ? [] : authorNodes.Select(a => a.InnerText);
+		IEnumerable<string> artistNames = artistNodes is null ? [] : artistNodes.Select(a => a.InnerText);
+		List<string> authors = authorNames.Concat(artistNames).ToList();
 
 		HtmlNode? firstChapterNode = document.DocumentNode.SelectSingleNode("//a[contains(@href, 'chapter/1')]/../following-sibling::h3");
 		int? year = int.Parse(firstChapterNode?.InnerText.Split(' ')[^1] ?? "2000");
@@ -155,7 +157,14 @@ private List<Chapter> ParseChaptersFromHtml(Manga manga, string mangaUrl)
 			string chapterNumber = match.Groups[1].Value;
 			string? chapterName = match.Groups[2].Success && match.Groups[2].Length > 1 ? match.Groups[2].Value : null;
 			string url = $"https://asuracomic.net/series/{chapterUrl}";
-			ret.Add(new Chapter(manga, chapterName, null, chapterNumber, url));
+			try
+			{
+				ret.Add(new Chapter(manga, chapterName, null, chapterNumber, url));
+			}
+			catch (Exception e)
+			{
+				Log($"Failed to load chapter {chapterNumber}: {e.Message}");
+			}
 		}
 
 		return ret;

Tranga/MangaConnectors/Bato.cs

@@ -163,7 +163,14 @@ private List<Chapter> ParseChaptersFromHtml(Manga manga, string mangaUrl)
 			string chapterNumber = match.Groups[3].Value;
 			string chapterName = chapterNumber;
 			string url = $"https://bato.to{chapterUrl}?load=2";
-			ret.Add(new Chapter(manga, chapterName, volumeNumber, chapterNumber, url));
+			try
+			{
+				ret.Add(new Chapter(manga, chapterName, volumeNumber, chapterNumber, url));
+			}
+			catch (Exception e)
+			{
+				Log($"Failed to load chapter {chapterNumber}: {e.Message}");
+			}
 		}
 
 		return ret;

Tranga/MangaConnectors/ChromiumDownloadClient.cs

@@ -10,7 +10,6 @@ namespace Tranga.MangaConnectors;
 internal class ChromiumDownloadClient : DownloadClient
 {
     private static IBrowser? _browser;
-    private const int StartTimeoutMs = 10000;
     private readonly HttpDownloadClient _httpDownloadClient;
 
     private static async Task<IBrowser> StartBrowser(Logging.Logger? logger = null)
@@ -24,7 +23,7 @@ private static async Task<IBrowser> StartBrowser(Logging.Logger? logger = null)
                 "--disable-dev-shm-usage",
                 "--disable-setuid-sandbox",
                 "--no-sandbox"},
-            Timeout = StartTimeoutMs
+            Timeout = TrangaSettings.ChromiumStartupTimeoutMs
         }, new LoggerFactory([new LogProvider(logger)]));
     }
 
@@ -43,6 +42,8 @@ public Logger(Logging.Logger? logger) : base(logger) { }
 
         public void Log<TState>(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func<TState, Exception?, string> formatter)
         {
+            if (logLevel <= LogLevel.Information)
+                return;
             logger?.WriteLine("Puppeteer", formatter.Invoke(state, exception));
         }
 
@@ -68,17 +69,20 @@ internal override RequestResult MakeRequestInternal(string url, string? referrer
 
     private RequestResult MakeRequestBrowser(string url, string? referrer = null, string? clickButton = null)
     {
+        if (_browser is null)
+            return new RequestResult(HttpStatusCode.InternalServerError, null, Stream.Null);
         IPage page = _browser.NewPageAsync().Result;
-        page.DefaultTimeout = 10000;
+        page.DefaultTimeout = TrangaSettings.ChromiumPageTimeoutMs;
+        page.SetExtraHttpHeadersAsync(new() { { "Referer", referrer } });
         IResponse response;
         try
         {
             response = page.GoToAsync(url, WaitUntilNavigation.Networkidle0).Result;
-            Log("Page loaded.");
+            Log($"Page loaded. {url}");
         }
         catch (Exception e)
         {
-            Log($"Could not load Page:\n{e.Message}");
+            Log($"Could not load Page {url}\n{e.Message}");
             page.CloseAsync();
             return new RequestResult(HttpStatusCode.InternalServerError, null, Stream.Null);
         }