Skip to content

Commit

Permalink
Merge branch 'release/3.0.2'
Browse files Browse the repository at this point in the history 
Fixed: ZDF-Crawler "Abmeldung" nicht zuverlässig

closed #85
  • Loading branch information
@alex1702
alex1702 committed Mar 29, 2017
2 parents 0834ab3 + 8319e6b commit 1e8e341
Show file tree
Hide file tree
Showing 3 changed files with 70 additions and 109 deletions.
2 changes: 1 addition & 1 deletion build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ apply from: "${project.rootDir}/gradle/eclipse.gradle"
sourceCompatibility = 1.8
targetCompatibility = 1.8
group = 'de.mediathekview'
version = '3.0.1'
version = '3.0.2'

def jarName = 'MServer.jar'
def mainClass = 'mServer.Main'
Expand Down
41 changes: 10 additions & 31 deletions src/main/java/mServer/crawler/FilmeSuchen.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,42 +19,20 @@
*/
package mServer.crawler;

import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.LinkedList;

import javax.swing.event.EventListenerList;

import org.apache.commons.lang3.time.FastDateFormat;

import de.mediathekview.mlib.Config;
import de.mediathekview.mlib.Const;
import de.mediathekview.mlib.daten.ListeFilme;
import de.mediathekview.mlib.filmesuchen.ListenerFilmeLaden;
import de.mediathekview.mlib.filmesuchen.ListenerFilmeLadenEvent;
import de.mediathekview.mlib.tool.Log;
import mServer.crawler.sender.Mediathek3Sat;
import mServer.crawler.sender.MediathekArd;
import mServer.crawler.sender.MediathekArte_de;
import mServer.crawler.sender.MediathekArte_fr;
import mServer.crawler.sender.MediathekBr;
import mServer.crawler.sender.MediathekDw;
import mServer.crawler.sender.MediathekHr;
import mServer.crawler.sender.MediathekKika;
import mServer.crawler.sender.MediathekMdr;
import mServer.crawler.sender.MediathekNdr;
import mServer.crawler.sender.MediathekOrf;
import mServer.crawler.sender.MediathekPhoenix;
import mServer.crawler.sender.MediathekRbb;
import mServer.crawler.sender.MediathekReader;
import mServer.crawler.sender.MediathekSr;
import mServer.crawler.sender.MediathekSrf;
import mServer.crawler.sender.MediathekSrfPod;
import mServer.crawler.sender.MediathekSwr;
import mServer.crawler.sender.MediathekWdr;
import mServer.crawler.sender.MediathekZdf;
import mServer.crawler.sender.MediathekZdfTivi;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.LinkedList;
import javax.swing.event.EventListenerList;
import mServer.crawler.CrawlerConfig;
import mServer.crawler.sender.*;
import org.apache.commons.lang3.time.FastDateFormat;

/**
* ###########################################################################################################
Expand Down Expand Up @@ -145,7 +123,8 @@ public void updateSender(String[] nameSender, ListeFilme listeFilme) {
for (String s : nameSender) {
if (reader.checkNameSenderFilmliste(s)) {
starten = true;
reader.start();
new Thread(reader).start();
//reader.start();
}
}
}
Expand Down
136 changes: 59 additions & 77 deletions src/main/java/mServer/crawler/sender/MediathekZdf.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,41 +19,30 @@
*/
package mServer.crawler.sender;

import java.util.Collection;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.ForkJoinPool;
import java.util.concurrent.Phaser;
import java.util.concurrent.RecursiveAction;
import java.util.concurrent.TimeUnit;

import de.mediathekview.mlib.Config;
import de.mediathekview.mlib.Const;
import de.mediathekview.mlib.daten.DatenFilm;
import de.mediathekview.mlib.tool.Log;
import etm.core.configuration.EtmManager;
import etm.core.monitor.EtmPoint;
import java.util.Collection;
import java.util.concurrent.*;
import mServer.crawler.CrawlerTool;
import mServer.crawler.FilmeSuchen;
import mServer.crawler.RunSender;
import mServer.crawler.sender.newsearch.DownloadDTO;
import mServer.crawler.sender.newsearch.GeoLocations;
import mServer.crawler.sender.newsearch.Qualities;
import mServer.crawler.sender.newsearch.VideoDTO;
import mServer.crawler.sender.newsearch.ZDFSearchTask;
import mServer.crawler.sender.newsearch.ZdfDatenFilm;
import mServer.crawler.sender.newsearch.*;

public class MediathekZdf extends MediathekReader
{
public class MediathekZdf extends MediathekReader {

public final static String SENDERNAME = Const.ZDF;
// public static final String URL_PATTERN_SENDUNG_VERPASST = "https://www.zdf.de/sendung-verpasst?airtimeDate=%s";
// public static final String[] KATEGORIE_ENDS = {"A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L", "M", "N", "O", "P", "Q", "R", "S", "T", "U", "V", "W", "X", "Y", "Z", "0+-+9"};
// public static final String KATEGORIEN_URL_PATTERN = "https://www.zdf.de/sendungen-a-z/?group=%s";
private final ForkJoinPool forkJoinPool = new ForkJoinPool(Runtime.getRuntime().availableProcessors() * 4);
//private final ForkJoinPool forkJoinPool = new ForkJoinPool(Runtime.getRuntime().availableProcessors() * 4);
private ForkJoinPool forkJoinPool;
// private final MSStringBuilder seite = new MSStringBuilder(Const.STRING_BUFFER_START_BUFFER);

public MediathekZdf(FilmeSuchen ssearch, int startPrio)
{
public MediathekZdf(FilmeSuchen ssearch, int startPrio) {
super(ssearch, SENDERNAME, 0 /* threads */, 150 /* urlWarten */, startPrio);
setName("MediathekZdf");
}
Expand All @@ -64,10 +53,11 @@ public MediathekZdf(FilmeSuchen ssearch, int startPrio)
public void addToList() {
meldungStart();
meldungAddThread();

int days = CrawlerTool.loadLongMax() ? 300 : 20;

final ZDFSearchTask newTask = new ZDFSearchTask(days);
forkJoinPool = new ForkJoinPool(Runtime.getRuntime().availableProcessors() * 4);
forkJoinPool.execute(newTask);
Collection<VideoDTO> filmList = newTask.join();

Expand All @@ -91,8 +81,9 @@ public void addToList() {
wasInterrupted = true;
phaser.forceTermination();
shutdownAndAwaitTermination(forkJoinPool, 5, TimeUnit.SECONDS);
} else
} else {
TimeUnit.SECONDS.sleep(1);
}
} catch (InterruptedException ignored) {
}
}
Expand All @@ -101,10 +92,11 @@ public void addToList() {
shutdownAndAwaitTermination(forkJoinPool, 60, TimeUnit.SECONDS);

perfPoint.collect();
if (wasInterrupted)
if (wasInterrupted) {
Log.sysLog("VideoDTO conversion interrupted.");
else
} else {
Log.sysLog("convert VideoDTO to DatenFilm finished.");
}

meldungThreadUndFertig();
}
Expand All @@ -114,8 +106,9 @@ void shutdownAndAwaitTermination(ExecutorService pool, long delay, TimeUnit dela
try {
if (!pool.awaitTermination(delay, delayUnit)) {
pool.shutdownNow();
if (!pool.awaitTermination(delay, delayUnit))
if (!pool.awaitTermination(delay, delayUnit)) {
Log.sysLog("Pool did not terminate");
}
}
} catch (InterruptedException ie) {
pool.shutdownNow();
Expand All @@ -125,6 +118,7 @@ void shutdownAndAwaitTermination(ExecutorService pool, long delay, TimeUnit dela

@SuppressWarnings("serial")
private class VideoDtoDatenFilmConverterAction extends RecursiveAction {

private final VideoDTO video;

public VideoDtoDatenFilmConverterAction(VideoDTO aVideoDTO) {
Expand All @@ -134,42 +128,39 @@ public VideoDtoDatenFilmConverterAction(VideoDTO aVideoDTO) {

@Override
protected void compute() {
if(video != null) {
try {
DownloadDTO download = video.getDownloadDto();

DatenFilm film = new ZdfDatenFilm(SENDERNAME, video.getTopic(), video.getWebsiteUrl() /*urlThema*/,
video.getTitle(), download.getUrl(Qualities.NORMAL), "" /*urlRtmp*/,
video.getDate(), video.getTime(), video.getDuration(), video.getDescription());
urlTauschen(film, video.getWebsiteUrl(), mlibFilmeSuchen);

//don´t use addFilm here
if (mlibFilmeSuchen.listeFilmeNeu.addFilmVomSender(film)) {
// dann ist er neu
FilmeSuchen.listeSenderLaufen.inc(film.arr[DatenFilm.FILM_SENDER], RunSender.Count.FILME);
}

if (!download.getUrl(Qualities.HD).isEmpty())
{
CrawlerTool.addUrlHd(film, download.getUrl(Qualities.HD), "");
}
if (!download.getUrl(Qualities.SMALL).isEmpty())
{
CrawlerTool.addUrlKlein(film, download.getUrl(Qualities.SMALL), "");
}
if (!download.getSubTitleUrl().isEmpty())
{
CrawlerTool.addUrlSubtitle(film, download.getSubTitleUrl());
}
if(download.getGeoLocation() != GeoLocations.GEO_NONE) {
film.arr[DatenFilm.FILM_GEO] = download.getGeoLocation().getDescription();
}
} catch (Exception ex) {
Log.errorLog(496583211, ex, "add film failed: " + video.getWebsiteUrl());
}
if (video != null) {
try {
DownloadDTO download = video.getDownloadDto();

DatenFilm film = new ZdfDatenFilm(SENDERNAME, video.getTopic(), video.getWebsiteUrl() /*urlThema*/,
video.getTitle(), download.getUrl(Qualities.NORMAL), "" /*urlRtmp*/,
video.getDate(), video.getTime(), video.getDuration(), video.getDescription());
urlTauschen(film, video.getWebsiteUrl(), mlibFilmeSuchen);

//don´t use addFilm here
if (mlibFilmeSuchen.listeFilmeNeu.addFilmVomSender(film)) {
// dann ist er neu
FilmeSuchen.listeSenderLaufen.inc(film.arr[DatenFilm.FILM_SENDER], RunSender.Count.FILME);
}

if (!download.getUrl(Qualities.HD).isEmpty()) {
CrawlerTool.addUrlHd(film, download.getUrl(Qualities.HD), "");
}
if (!download.getUrl(Qualities.SMALL).isEmpty()) {
CrawlerTool.addUrlKlein(film, download.getUrl(Qualities.SMALL), "");
}
if (!download.getSubTitleUrl().isEmpty()) {
CrawlerTool.addUrlSubtitle(film, download.getSubTitleUrl());
}
if (download.getGeoLocation() != GeoLocations.GEO_NONE) {
film.arr[DatenFilm.FILM_GEO] = download.getGeoLocation().getDescription();
}
} catch (Exception ex) {
Log.errorLog(496583211, ex, "add film failed: " + video.getWebsiteUrl());
}
}
phaser.arrive();
}
phaser.arriveAndDeregister();
}
}

private static void updateHdStatus(DatenFilm film, String urlSeite) {
Expand Down Expand Up @@ -213,39 +204,30 @@ public static void urlTauschen(DatenFilm film, String urlSeite, FilmeSuchen mSFi
updateHdStatus(film, urlSeite);
}

private static void changeUrl(String from, String to, DatenFilm film, String urlSeite, FilmeSuchen mSFilmeSuchen)
{
if (film.arr[DatenFilm.FILM_URL].endsWith(from))
{
private static void changeUrl(String from, String to, DatenFilm film, String urlSeite, FilmeSuchen mSFilmeSuchen) {
if (film.arr[DatenFilm.FILM_URL].endsWith(from)) {
String url = film.arr[DatenFilm.FILM_URL].substring(0, film.arr[DatenFilm.FILM_URL].lastIndexOf(from)) + to;
String l = mSFilmeSuchen.listeFilmeAlt.getFileSizeUrl(url);
// zum Testen immer machen!!
if (!l.isEmpty())
{
if (!l.isEmpty()) {
film.arr[DatenFilm.FILM_GROESSE] = l;
film.arr[DatenFilm.FILM_URL] = url;
} else if (urlExists(url))
{
} else if (urlExists(url)) {
// dann wars wohl nur ein "403er"
film.arr[DatenFilm.FILM_URL] = url;
} else
{
} else {
Log.errorLog(945120369, "urlTauschen: " + urlSeite);
}
}
}

private static void updateHd(String from, String to, DatenFilm film, String urlSeite)
{
if (film.arr[DatenFilm.FILM_URL_HD].isEmpty() && film.arr[DatenFilm.FILM_URL].endsWith(from))
{
private static void updateHd(String from, String to, DatenFilm film, String urlSeite) {
if (film.arr[DatenFilm.FILM_URL_HD].isEmpty() && film.arr[DatenFilm.FILM_URL].endsWith(from)) {
String url = film.arr[DatenFilm.FILM_URL].substring(0, film.arr[DatenFilm.FILM_URL].lastIndexOf(from)) + to;
// zum Testen immer machen!!
if (urlExists(url))
{
if (urlExists(url)) {
CrawlerTool.addUrlHd(film, url, "");
} else
{
} else {
Log.errorLog(945120147, "urlTauschen: " + urlSeite);
}
}
Expand Down

0 comments on commit 1e8e341

Please sign in to comment.