Skip to content

Commit

Permalink
init
Browse files Browse the repository at this point in the history
  • Loading branch information
codingPF committed Feb 17, 2024
1 parent 1cfc733 commit 41003af
Show file tree
Hide file tree
Showing 34 changed files with 25,765 additions and 19 deletions.
25 changes: 14 additions & 11 deletions MServer-Config.yaml
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
#### Server configurations ####

# The maximum amount of cpu threads to be used.
maximumCpuThreads: 10
maximumCpuThreads: 1

# The maximum duration in minutes the server should run.<br>
# If set to 0 the server runs without a time limit.
Expand All @@ -24,15 +24,17 @@ senderIncluded:
#- ARTE_PL
#- ARTE_IT
#- ARTE_ES
#- 3SAT
#- DREISAT
#- FUNK
#- KIKA
#- DW
#- ORF
# - DW
- ORF
#- PHOENIX
#- SRF
- SR
#- SR
#- ZDF

#SRF,SR,PHONIX,ORF,KIKA,DW,3SAT<

# If set the server will be awake after the crawler run and restarts the run after the given amount.
#schedules:
Expand Down Expand Up @@ -133,7 +135,7 @@ topicsSearchEnabled: true
# The maximum amount of sub pages to be crawled.<br>
# Example: If a Sendung overview side has 10 pages with videos for this Sendung and
# the amount set by this is 5 then the crawler crawls pages 1 to 5.
maximumSubpages: 1
maximumSubpages: 5

# The maximum amount of days going to past will be crawled for the "Sendung Verpasst?" section.
maximumDaysForSendungVerpasstSection: 7
Expand All @@ -155,8 +157,7 @@ senderConfigurations:
#10,20,40 ok
maximumSubpages: 0
ORF:
#2,4,8 ok
maximumUrlsPerTask: 40
maximumRequestsPerSecond: 10.0
ARTE_DE:
maximumUrlsPerTask: 1
maximumDaysForSendungVerpasstSectionFuture: 0
Expand All @@ -178,10 +179,12 @@ senderConfigurations:
maximumRequestsPerSecond: 10.0
FUNK:
maximumUrlsPerTask: 99
DW:
maximumSubpages: 0
SR:
DREISAT:
maximumSubpages: 5
maximumDaysForSendungVerpasstSection: 60
PHOENIX:
maximumSubpages: 500


# configure string variables
crawlerApiParams:
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,111 @@
package de.mediathekview.mserver.base.utils;

import java.util.ArrayList;
import java.util.Set;

import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;

import de.mediathekview.mlib.daten.Film;
import de.mediathekview.mlib.daten.Filmlist;
import de.mediathekview.mlib.daten.Resolution;
import de.mediathekview.mserver.crawler.kika.KikaApiCrawler;

public class FilmlistDebugHelper {
private static final Logger LOG = LogManager.getLogger(FilmlistDebugHelper.class);



public static Film getFilmFromSet(Set<Film> base, Film searchFilm) {
for (Film e : base) {
if (e.equals(searchFilm)) {
return e;
}
}
return null;
}

public static void printFilmlistForSender(Filmlist list, String sender, boolean printFullDetails) {
list.getFilms().values().stream().forEach( e -> {
if (e.getSenderName().equalsIgnoreCase(sender)) {
if (printFullDetails) {
LOG.debug("{} {} {}", e.getTitel(), e.getThema(), e);
} else {
LOG.debug("{} {}", e.getTitel(), e.getThema());
}
}
});
}

public static void printFilmlistForThema(Filmlist list, String thema, boolean printFullDetails) {
list.getFilms().values().stream().forEach( e -> {
if (e.getThema().equalsIgnoreCase(thema)) {
if (printFullDetails) {
LOG.debug("{} {} {} {}", e.getSenderName(), e.getTitel(), e.getThema(), e);
} else {
LOG.debug("{} {} {}", e.getSenderName(), e.getTitel(), e.getThema());
}
}
});
}




public static void compareFilmlist(Filmlist aFilmlist, Filmlist bFilmlist) {
ArrayList<Film> bFilms = new ArrayList<>(bFilmlist.getFilms().values());
aFilmlist.getFilms().values().forEach( f -> {
if (bFilms.indexOf(f) == -1) {
LOG.info("Missing Film in source list");
LOG.info(f.toString());
} else {
Film expectedFilm = bFilms.get(bFilms.indexOf(f));
compare(f, expectedFilm);
}
});
ArrayList<Film> aFilms = new ArrayList<>(aFilmlist.getFilms().values());
bFilms.forEach( f -> {
if (aFilms.indexOf(f) == -1) {
LOG.info("Missing Film in target list");
LOG.info(f.toString());
}
});
}

private static void compare(Film aFilm, Film bFilm) {
String error = "";
if (!aFilm.getSenderName().equalsIgnoreCase(bFilm.getSenderName())) {
error = "Incorrect Sender";
} else if (!aFilm.getTitel().equalsIgnoreCase(bFilm.getTitel())){
error = "Incorrect Title";
} else if (!aFilm.getThema().equalsIgnoreCase(bFilm.getThema())){
error = "Incorrect Topic";
} else if (!aFilm.getDuration().equals(bFilm.getDuration())){
error = "Incorrect Duration";
} else if (!aFilm.getBeschreibung().equalsIgnoreCase(bFilm.getBeschreibung())){
error = "Incorrect Description";
} else if (!aFilm.getWebsite().toString().equalsIgnoreCase(bFilm.getWebsite().toString())){
error = "Incorrect website";
} else if (!aFilm.getTime().equals(bFilm.getTime())){
error = "Incorrect Time";
} else if (!aFilm.getSubtitles().equals(bFilm.getSubtitles())){
error = "Incorrect subtitle";
} else if (bFilm.getUrl(Resolution.SMALL) != null &&
!aFilm.getUrl(Resolution.SMALL).toString().equalsIgnoreCase(bFilm.getUrl(Resolution.SMALL).toString())) {
error = "URL SMALL";
} else if (bFilm.getUrl(Resolution.NORMAL) != null &&
!aFilm.getUrl(Resolution.NORMAL).toString().equalsIgnoreCase(bFilm.getUrl(Resolution.NORMAL).toString())) {
error = "URL NORMAL";
} else if (bFilm.getUrl(Resolution.HD) != null &&
!aFilm.getUrl(Resolution.HD).toString().equalsIgnoreCase(bFilm.getUrl(Resolution.HD).toString())) {
error = "URL HD";
}
//
if (error != "") {
LOG.info(error);
LOG.info(aFilm.toString());
LOG.info(bFilm.toString());
}
}

}
15 changes: 13 additions & 2 deletions src/main/java/de/mediathekview/mserver/base/utils/JsonUtils.java
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,18 @@ public static Optional<Integer> getAttributeAsInt(final JsonObject jsonObject, f
}

public static Optional<String> getElementValueAsString(final JsonElement aJsonElement, final String... aElementIds) {
Optional<String> rs = Optional.empty();
Optional<JsonElement> rs = JsonUtils.getElement(aJsonElement, aElementIds);
if (rs.isPresent()) {
return Optional.of(rs.get().getAsString());
}
return Optional.empty();
}

public static Optional<JsonElement> getElement(final JsonElement aJsonElement, final String... aElementIds) {
Optional<JsonElement> rs = Optional.empty();
if (aElementIds == null || aElementIds.length == 0) {
return rs;
}
JsonObject aJsonObject = aJsonElement.getAsJsonObject();
for (int i = 0; i < aElementIds.length-1; i++) {
String elementId = aElementIds[i];
Expand All @@ -91,7 +102,7 @@ public static Optional<String> getElementValueAsString(final JsonElement aJsonEl
//
String elementId = aElementIds[aElementIds.length-1];
if (aJsonObject != null && aJsonObject.has(elementId) && !aJsonObject.get(elementId).isJsonNull()) {
rs = Optional.of(aJsonObject.get(elementId).getAsString());
rs = Optional.of(aJsonObject.get(elementId));
}
//
return rs;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
import de.mediathekview.mserver.crawler.funk.FunkCrawler;
import de.mediathekview.mserver.crawler.kika.KikaApiCrawler;
import de.mediathekview.mserver.crawler.orf.OrfCrawler;
import de.mediathekview.mserver.crawler.orfon.OrfOnCrawler;
import de.mediathekview.mserver.crawler.phoenix.PhoenixCrawler;
import de.mediathekview.mserver.crawler.sr.SrCrawler;
import de.mediathekview.mserver.crawler.srf.SrfCrawler;
Expand Down Expand Up @@ -519,8 +520,10 @@ private void initializeCrawler(final MServerConfigManager rootConfig) {
new KikaApiCrawler(forkJoinPool, messageListeners, progressListeners, rootConfig));
crawlerMap.put(
Sender.DW, new DwCrawler(forkJoinPool, messageListeners, progressListeners, rootConfig));
//crawlerMap.put(
// Sender.ORF, new OrfCrawler(forkJoinPool, messageListeners, progressListeners, rootConfig));
crawlerMap.put(
Sender.ORF, new OrfCrawler(forkJoinPool, messageListeners, progressListeners, rootConfig));
Sender.ORF, new OrfOnCrawler(forkJoinPool, messageListeners, progressListeners, rootConfig));
crawlerMap.put(
Sender.PHOENIX,
new PhoenixCrawler(forkJoinPool, messageListeners, progressListeners, rootConfig));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -145,7 +145,8 @@ private static Resolution getQualityForNumber(final int qualityIndicator) {
case 1 -> Resolution.SMALL;
case 3, 4 -> Resolution.HD;
case 5 -> Resolution.UHD;
case 2, default -> Resolution.NORMAL;
case 2 -> Resolution.NORMAL;
default -> Resolution.NORMAL;
};
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ public NexxCloudSessionInitiationTask(final AbstractCrawler aCrawler) {
@Override
public Long call() {
final Gson gson = createGson();

/*
try(final Client client = createClient()) {
final WebTarget target =
client.target(FunkUrls.NEXX_CLOUD_SESSION_INIT.getAsString(crawler.getRuntimeConfig()));
Expand All @@ -55,7 +55,7 @@ public Long call() {
response.getStatus());
}
}
}
}*/
return null;
}

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
package de.mediathekview.mserver.crawler.orfon;

import java.util.ArrayList;
import java.util.List;
import java.util.Objects;

import de.mediathekview.mserver.crawler.basic.CrawlerUrlDTO;

public class OrfOnBreadCrumsUrlDTO extends CrawlerUrlDTO {
private List<String> breadCrums = new ArrayList<>();

public OrfOnBreadCrumsUrlDTO(String breadCrum, String aUrl) {
super(aUrl);
setBreadCrums(List.of(breadCrum));
}
public OrfOnBreadCrumsUrlDTO(List<String> breadCrums, String aUrl) {
super(aUrl);
setBreadCrums(breadCrums);
}

public List<String> getBreadCrums() {
return breadCrums;
}

public void setBreadCrums(List<String> breadCrums) {
this.breadCrums = breadCrums;
}

public void setBreadCrumsPath(List<String> breadCrums) {
breadCrums.addAll(getBreadCrums());
setBreadCrums(breadCrums);
}

public boolean addBreadCrum(String value) {
if (!breadCrums.contains(value)) {
breadCrums.add(value);
return true;
}
return false;
}

@Override
public boolean equals(final Object obj) {
if (obj == null || getClass() != obj.getClass()) {
return false;
}

if (super.equals(obj)) {
return breadCrums.containsAll(((OrfOnBreadCrumsUrlDTO)obj).breadCrums);
}

return false;
}

@Override
public int hashCode() {
return Objects.hash(super.hashCode(), this.breadCrums);
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
package de.mediathekview.mserver.crawler.orfon;

public final class OrfOnConstants {
//
public static final String HOST = "https://api-tvthek.orf.at/api/v4.3";
//
public static final String SCHEDULE = HOST + "/schedule";
//
public static final String AZ = HOST + "/profiles/lettergroup";
public static final int PAGE_SIZE = 200;
//
public static final String HISTORY = HOST + "/history";
//
public static final String EPISODE = HOST + "/episode";
//
public static final String bearer = "Basic b3JmX29uX3Y0MzpqRlJzYk5QRmlQU3h1d25MYllEZkNMVU41WU5aMjhtdA==";
//
private OrfOnConstants() {}
//
}
Loading

0 comments on commit 41003af

Please sign in to comment.