Skip to content

Commit

Permalink
Fix usage of deprecated URL constructors
Browse files Browse the repository at this point in the history
  • Loading branch information
@metaprime
metaprime committed Jan 2, 2025
1 parent 3b9352f commit 4e3619a
Show file tree
Hide file tree
Showing 4 changed files with 43 additions and 28 deletions.
55 changes: 33 additions & 22 deletions src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@
* Simplified ripper, designed for ripping from sites by parsing HTML.
*/
public abstract class AbstractHTMLRipper extends AbstractRipper {

private final Map<URL, File> itemsPending = Collections.synchronizedMap(new HashMap<>());
private final Map<URL, Path> itemsCompleted = Collections.synchronizedMap(new HashMap<>());
private final Map<URL, String> itemsErrored = Collections.synchronizedMap(new HashMap<>());
Expand Down Expand Up @@ -60,11 +60,15 @@ protected Document getCachedFirstPage() throws IOException, URISyntaxException {
public Document getNextPage(Document doc) throws IOException, URISyntaxException {
return null;
}
protected abstract List<String> getURLsFromPage(Document page) throws UnsupportedEncodingException;

protected abstract List<String> getURLsFromPage(Document page) throws UnsupportedEncodingException, URISyntaxException;

protected List<String> getDescriptionsFromPage(Document doc) throws IOException {
throw new IOException("getDescriptionsFromPage not implemented"); // Do I do this or make an abstract function?
}

protected abstract void downloadURL(URL url, int index);

protected DownloadThreadPool getThreadPool() {
return null;
}
Expand Down Expand Up @@ -130,7 +134,7 @@ public void rip() throws IOException, URISyntaxException {
List<String> doclocation = new ArrayList<>();

LOGGER.info("Got doc location " + doc.location());

while (doc != null) {

LOGGER.info("Processing a doc...");
Expand Down Expand Up @@ -167,7 +171,7 @@ public void rip() throws IOException, URISyntaxException {
for (String imageURL : imageURLs) {
index += 1;
LOGGER.debug("Found image url #" + index + ": '" + imageURL + "'");
downloadURL(new URL(imageURL), index);
downloadURL(new URI(imageURL).toURL(), index);
if (isStopped() || isThisATest()) {
break;
}
Expand All @@ -182,19 +186,26 @@ public void rip() throws IOException, URISyntaxException {
if (isStopped() || isThisATest()) {
break;
}

textindex += 1;
LOGGER.debug("Getting description from " + textURL);
String[] tempDesc = getDescription(textURL,doc);

if (tempDesc != null) {
if (Utils.getConfigBoolean("file.overwrite", false) || !(new File(
workingDir.getCanonicalPath()
+ ""
+ File.separator
+ getPrefix(index)
+ (tempDesc.length > 1 ? tempDesc[1] : fileNameFromURL(new URL(textURL)))
+ ".txt").exists())) {
URL url = new URI(textURL).toURL();
String filename = fileNameFromURL(url);

boolean fileExists = new File(
workingDir.getCanonicalPath()
+ ""
+ File.separator
+ getPrefix(index)
+ (tempDesc.length > 1 ? tempDesc[1] : filename)
+ ".txt").exists();

if (Utils.getConfigBoolean("file.overwrite", false) || !fileExists) {
LOGGER.debug("Got description from " + textURL);
saveText(new URL(textURL), "", tempDesc[0], textindex, (tempDesc.length > 1 ? tempDesc[1] : fileNameFromURL(new URL(textURL))));
saveText(url, "", tempDesc[0], textindex, (tempDesc.length > 1 ? tempDesc[1] : filename));
sleep(descSleepTime());
} else {
LOGGER.debug("Description from " + textURL + " already exists.");
Expand Down Expand Up @@ -225,12 +236,12 @@ public void rip() throws IOException, URISyntaxException {
}
waitForThreads();
}

/**
* Gets the file name from the URL
* @param url
* @param url
* URL that you want to get the filename from
* @return
* @return
* Filename of the URL
*/
private String fileNameFromURL(URL url) {
Expand All @@ -244,7 +255,7 @@ private String fileNameFromURL(URL url) {
return saveAs;
}
/**
*
*
* @param url
* Target URL
* @param subdirectory
Expand All @@ -253,7 +264,7 @@ private String fileNameFromURL(URL url) {
* Text you want to save
* @param index
* Index in something like an album
* @return
* @return
* True if ripped successfully
* False if failed
*/
Expand Down Expand Up @@ -295,12 +306,12 @@ private boolean saveText(URL url, String subdirectory, String text, int index, S
}
return true;
}

/**
* Gets prefix based on where in the index it is
* @param index
* @param index
* The index in question
* @return
* @return
* Returns prefix for a file. (?)
*/
protected String getPrefix(int index) {
Expand All @@ -313,9 +324,9 @@ protected String getPrefix(int index) {

/*
* ------ Methods copied from AlbumRipper. ------
* This removes AlbumnRipper's usage from this class.
* This removes AlbumnRipper's usage from this class.
*/

protected boolean allowDuplicates() {
return false;
}
Expand Down
3 changes: 2 additions & 1 deletion src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
import java.io.File;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
import java.nio.charset.StandardCharsets;
Expand Down Expand Up @@ -94,7 +95,7 @@ public void rip() throws IOException, URISyntaxException {

index += 1;
LOGGER.debug("Found image url #" + index+ ": " + imageURL);
downloadURL(new URL(imageURL), index);
downloadURL(new URI(imageURL).toURL(), index);
}

if (isStopped() || isThisATest()) {
Expand Down
5 changes: 3 additions & 2 deletions src/main/java/com/rarchives/ripme/ripper/rippers/ChanRipper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
import com.rarchives.ripme.utils.RipUtils;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
import java.util.ArrayList;
Expand Down Expand Up @@ -208,7 +209,7 @@ private boolean isURLBlacklisted(String url) {
return false;
}
@Override
public List<String> getURLsFromPage(Document page) {
public List<String> getURLsFromPage(Document page) throws URISyntaxException {
List<String> imageURLs = new ArrayList<>();
Pattern p; Matcher m;
for (Element link : page.select("a")) {
Expand Down Expand Up @@ -254,7 +255,7 @@ public List<String> getURLsFromPage(Document page) {
//Copied code from RedditRipper, getFilesFromURL should also implement stuff like flickr albums
URL originalURL;
try {
originalURL = new URL(href);
originalURL = new URI(href).toURL();
} catch (MalformedURLException e) {
continue;
}
Expand Down
8 changes: 5 additions & 3 deletions src/main/java/com/rarchives/ripme/ripper/rippers/video/MotherlessVideoRipper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@

import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
import java.util.List;
import java.util.regex.Matcher;
Expand Down Expand Up @@ -51,7 +53,7 @@ public String getGID(URL url) throws MalformedURLException {
}

@Override
public void rip() throws IOException {
public void rip() throws IOException, URISyntaxException {
LOGGER.info(" Retrieving " + this.url);
String html = Http.url(this.url).get().toString();
if (html.contains("__fileurl = '")) {
Expand All @@ -62,7 +64,7 @@ public void rip() throws IOException {
throw new IOException("Could not find video URL at " + url);
}
String vidUrl = vidUrls.get(0);
addURLToDownload(new URL(vidUrl), HOST + "_" + getGID(this.url));
addURLToDownload(new URI(vidUrl).toURL(), HOST + "_" + getGID(this.url));
waitForThreads();
}
}
}

0 comments on commit 4e3619a

Please sign in to comment.