Skip to content

Rippers and refactorigns #2102

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 10 commits into
base: main
Choose a base branch
from
Open

Rippers and refactorigns #2102

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
69ec762
pageContainsAlbums only called with the current URL as parameter
zaphodorama Sep 29, 2024
43eb85e
No need to write to sysout
zaphodorama Sep 29, 2024
f82b053
Refactor AbstractHTML/JSON and AlbumRipper to have a common ancestor
zaphodorama Sep 29, 2024
29e2517
Remove URL from the getAlbumTitle function
zaphodorama Sep 29, 2024
8a72442
Helper functions 'collectedCookies' and removeIfStartsWith to RipUtils
zaphodorama Sep 29, 2024
edc7dee
Add helper to collect URLs from the requested page
zaphodorama Sep 29, 2024
3dc9514
Separate method to init threadpool
zaphodorama Sep 29, 2024
3fe22ff
Separate out the 'downloadInBackground' stuff
zaphodorama Sep 29, 2024
09f15c8
Add new message type, which can be fired when the download progress
zaphodorama Sep 29, 2024
fd0e3a8
Implement review suggestions
zaphodorama Aug 8, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion src/main/java/com/rarchives/ripme/App.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -111,7 +111,7 @@ private static void rip(URL url) throws Exception {
entry.url = u;
entry.dir = ripper.getWorkingDir().getAbsolutePath();
try {
entry.title = ripper.getAlbumTitle(ripper.getURL());
entry.title = ripper.getAlbumTitle();
} catch (MalformedURLException ignored) { }
HISTORY.add(entry);
}
Expand Down
202 changes: 9 additions & 193 deletions src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,37 +8,23 @@
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardOpenOption;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.jsoup.nodes.Document;

import com.rarchives.ripme.ui.MainWindow;
import com.rarchives.ripme.ui.RipStatusMessage;
import com.rarchives.ripme.ui.RipStatusMessage.STATUS;
import com.rarchives.ripme.utils.Http;
import com.rarchives.ripme.utils.Utils;
import com.rarchives.ripme.utils.RipUtils;

/**
* Simplified ripper, designed for ripping from sites by parsing HTML.
*/
public abstract class AbstractHTMLRipper extends AbstractRipper {
public abstract class AbstractHTMLRipper extends QueueingRipper {

private static final Logger logger = LogManager.getLogger(AbstractHTMLRipper.class);

private final Map<URL, File> itemsPending = Collections.synchronizedMap(new HashMap<>());
private final Map<URL, Path> itemsCompleted = Collections.synchronizedMap(new HashMap<>());
private final Map<URL, String> itemsErrored = Collections.synchronizedMap(new HashMap<>());
Document cachedFirstPage;

protected AbstractHTMLRipper(URL url) throws IOException {
Expand Down Expand Up @@ -68,6 +54,10 @@ public Document getNextPage(Document doc) throws IOException, URISyntaxException
return null;
}

public List<URL> getURLsFromFirstPage() throws UnsupportedEncodingException, IOException, URISyntaxException {
return RipUtils.toURLList(getURLsFromPage(getCachedFirstPage()));
}

protected abstract List<String> getURLsFromPage(Document page) throws UnsupportedEncodingException, URISyntaxException;

protected List<String> getDescriptionsFromPage(Document doc) throws IOException {
Expand Down Expand Up @@ -115,7 +105,7 @@ protected boolean hasQueueSupport() {
}

// Takes a url and checks if it is for a page of albums
protected boolean pageContainsAlbums(URL url) {
protected boolean pageContainsAlbums() {
return false;
}

Expand All @@ -127,7 +117,7 @@ public void rip() throws IOException, URISyntaxException {
sendUpdate(STATUS.LOADING_RESOURCE, this.url.toExternalForm());
var doc = getCachedFirstPage();

if (hasQueueSupport() && pageContainsAlbums(this.url)) {
if (hasQueueSupport() && pageContainsAlbums()) {
List<String> urls = getAlbumsToQueue(doc);
for (String url : urls) {
MainWindow.addUrlToQueue(url);
Expand Down Expand Up @@ -329,156 +319,6 @@ protected String getPrefix(int index) {
return prefix;
}

/*
* ------ Methods copied from AlbumRipper. ------
* This removes AlbumnRipper's usage from this class.
*/

protected boolean allowDuplicates() {
return false;
}

@Override
/*
Returns total amount of files attempted.
*/
public int getCount() {
return itemsCompleted.size() + itemsErrored.size();
}

@Override
/*
Queues multiple URLs of single images to download from a single Album URL
*/
public boolean addURLToDownload(URL url, Path saveAs, String referrer, Map<String,String> cookies, Boolean getFileExtFromMIME) {
// Only download one file if this is a test.
if (isThisATest() && (itemsCompleted.size() > 0 || itemsErrored.size() > 0)) {
stop();
itemsPending.clear();
return false;
}
if (!allowDuplicates()
&& ( itemsPending.containsKey(url)
|| itemsCompleted.containsKey(url)
|| itemsErrored.containsKey(url) )) {
// Item is already downloaded/downloading, skip it.
logger.info("[!] Skipping " + url + " -- already attempted: " + Utils.removeCWD(saveAs));
return false;
}
if (shouldIgnoreURL(url)) {
sendUpdate(STATUS.DOWNLOAD_SKIP, "Skipping " + url.toExternalForm() + " - ignored extension");
return false;
}
if (Utils.getConfigBoolean("urls_only.save", false)) {
// Output URL to file
Path urlFile = Paths.get(this.workingDir + "/urls.txt");
String text = url.toExternalForm() + System.lineSeparator();
try {
Files.write(urlFile, text.getBytes(StandardCharsets.UTF_8), StandardOpenOption.CREATE, StandardOpenOption.APPEND);
itemsCompleted.put(url, urlFile);
} catch (IOException e) {
logger.error("Error while writing to " + urlFile, e);
}
}
else {
itemsPending.put(url, saveAs.toFile());
DownloadFileThread dft = new DownloadFileThread(url, saveAs.toFile(), this, getFileExtFromMIME);
if (referrer != null) {
dft.setReferrer(referrer);
}
if (cookies != null) {
dft.setCookies(cookies);
}
threadPool.addThread(dft);
}

return true;
}

@Override
public boolean addURLToDownload(URL url, Path saveAs) {
return addURLToDownload(url, saveAs, null, null, false);
}

/**
* Queues image to be downloaded and saved.
* Uses filename from URL to decide filename.
* @param url
* URL to download
* @return
* True on success
*/
protected boolean addURLToDownload(URL url) {
// Use empty prefix and empty subdirectory
return addURLToDownload(url, "", "");
}

@Override
/*
Cleans up & tells user about successful download
*/
public void downloadCompleted(URL url, Path saveAs) {
if (observer == null) {
return;
}
try {
String path = Utils.removeCWD(saveAs);
RipStatusMessage msg = new RipStatusMessage(STATUS.DOWNLOAD_COMPLETE, path);
itemsPending.remove(url);
itemsCompleted.put(url, saveAs);
observer.update(this, msg);

checkIfComplete();
} catch (Exception e) {
logger.error("Exception while updating observer: ", e);
}
}

@Override
/*
* Cleans up & tells user about failed download.
*/
public void downloadErrored(URL url, String reason) {
if (observer == null) {
return;
}
itemsPending.remove(url);
itemsErrored.put(url, reason);
observer.update(this, new RipStatusMessage(STATUS.DOWNLOAD_ERRORED, url + " : " + reason));

checkIfComplete();
}

@Override
/*
Tells user that a single file in the album they wish to download has
already been downloaded in the past.
*/
public void downloadExists(URL url, Path file) {
if (observer == null) {
return;
}

itemsPending.remove(url);
itemsCompleted.put(url, file);
observer.update(this, new RipStatusMessage(STATUS.DOWNLOAD_WARN, url + " already saved as " + file));

checkIfComplete();
}

/**
* Notifies observers and updates state if all files have been ripped.
*/
@Override
protected void checkIfComplete() {
if (observer == null) {
return;
}
if (itemsPending.isEmpty()) {
super.checkIfComplete();
}
}

/**
* Sets directory to save all ripped files to.
* @param url
Expand All @@ -492,7 +332,7 @@ public void setWorkingDir(URL url) throws IOException, URISyntaxException {
if (!path.endsWith(File.separator)) {
path += File.separator;
}
String title = getAlbumTitle(this.url);
String title = getAlbumTitle();
logger.debug("Using album title '" + title + "'");

title = Utils.filesystemSafe(title);
Expand All @@ -509,28 +349,4 @@ public void setWorkingDir(URL url) throws IOException, URISyntaxException {
logger.debug("Set working directory to: " + this.workingDir);
}

/**
* @return
* Integer between 0 and 100 defining the progress of the album rip.
*/
@Override
public int getCompletionPercentage() {
double total = itemsPending.size() + itemsErrored.size() + itemsCompleted.size();
return (int) (100 * ( (total - itemsPending.size()) / total));
}

/**
* @return
* Human-readable information on the status of the current rip.
*/
@Override
public String getStatusText() {
return getCompletionPercentage() +
"% " +
"- Pending: " + itemsPending.size() +
", Completed: " + itemsCompleted.size() +
", Errored: " + itemsErrored.size();
}


}
Loading