Skip to content

Commit

Permalink
Merge branch 'hotfix/3.1.9'
Browse files Browse the repository at this point in the history 
fixed: ARTE: keine aktuellen Filme mehr

closed #191
  • Loading branch information
@alex1702
alex1702 committed Jul 6, 2017
2 parents 41aabd9 + e5ba8ae commit 0e2ca6b
Show file tree
Hide file tree
Showing 4 changed files with 116 additions and 51 deletions.
2 changes: 1 addition & 1 deletion build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ apply from: "${project.rootDir}/gradle/eclipse.gradle"
sourceCompatibility = 1.8
targetCompatibility = 1.8
group = 'de.mediathekview'
version = '3.1.8'
version = '3.1.9'

def jarName = 'MServer.jar'
def mainClass = 'mServer.Main'
Expand Down
3 changes: 2 additions & 1 deletion src/main/java/mServer/crawler/sender/arte/ArteDatenFilmDeserializer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@

public class ArteDatenFilmDeserializer implements JsonDeserializer<ListeFilme>
{
private static final String JSON_ELEMENT_VIDEOS = "videos";
private static final Logger LOG = LogManager.getLogger(ArteDatenFilmDeserializer.class);

private final String langCode;
Expand All @@ -37,7 +38,7 @@ public ListeFilme deserialize(JsonElement aJsonElement, Type aType, JsonDeserial
ListeFilme listeFilme = new ListeFilme();

Collection<Future<DatenFilm>> futureFilme = new ArrayList<>();
for (JsonElement jsonElement : aJsonElement.getAsJsonArray())
for (JsonElement jsonElement : aJsonElement.getAsJsonObject().get(JSON_ELEMENT_VIDEOS).getAsJsonArray())
{
ExecutorService executor = Executors.newCachedThreadPool();
futureFilme.add(executor.submit(new ArteJsonObjectToDatenFilmCallable(jsonElement.getAsJsonObject(), langCode, senderName)));
Expand Down
133 changes: 88 additions & 45 deletions src/main/java/mServer/crawler/sender/arte/ArteJsonObjectToDatenFilmCallable.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,28 +4,28 @@
import java.time.LocalTime;
import java.util.concurrent.Callable;

import mServer.crawler.CrawlerTool;
import mServer.crawler.sender.newsearch.Qualities;
import okhttp3.OkHttpClient;
import okhttp3.Request;
import okhttp3.Response;

import org.apache.commons.lang3.time.FastDateFormat;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;

import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import com.google.gson.JsonElement;
import com.google.gson.JsonNull;
import com.google.gson.JsonObject;

import de.mediathekview.mlib.daten.DatenFilm;
import de.mediathekview.mlib.tool.MVHttpClient;
import mServer.crawler.CrawlerTool;
import mServer.crawler.sender.newsearch.Qualities;
import mServer.tool.MserverDatumZeit;
import org.apache.commons.lang3.time.FastDateFormat;
import okhttp3.OkHttpClient;
import okhttp3.Request;
import okhttp3.Response;

public class ArteJsonObjectToDatenFilmCallable implements Callable<DatenFilm>
{
private static final Logger LOG = LogManager.getLogger(ArteJsonObjectToDatenFilmCallable.class);
private static final String JSON_OBJECT_KEY_PROGRAM = "program";
private static final String JSON_ELEMENT_KEY_CATEGORY = "category";
private static final String JSON_ELEMENT_KEY_SUBCATEGORY = "subcategory";
private static final String JSON_ELEMENT_KEY_NAME = "name";
Expand All @@ -34,7 +34,10 @@ public class ArteJsonObjectToDatenFilmCallable implements Callable<DatenFilm>
private static final String JSON_ELEMENT_KEY_URL = "url";
private static final String JSON_ELEMENT_KEY_PROGRAM_ID = "programId";
private static final String ARTE_VIDEO_INFORMATION_URL_PATTERN = "https://api.arte.tv/api/player/v1/config/%s/%s?platform=ARTE_NEXT";
private static final String ARTE_VIDEO_INFORMATION_URL_PATTERN_2 = "https://api.arte.tv/api/opa/v3/programs/%s/%s"; // Für broadcastBeginRounded
private static final String JSON_ELEMENT_KEY_SHORT_DESCRIPTION = "shortDescription";
private static final String JSON_ELEMENT_BROADCAST_ELTERNKNOTEN_1 = "programs";
private static final String JSON_ELEMENT_BROADCAST_ELTERNKNOTEN_2 = "broadcastProgrammings";
private static final String JSON_ELEMENT_BROADCAST = "broadcastBeginRounded";

private final JsonObject jsonObject;
Expand All @@ -52,52 +55,92 @@ public ArteJsonObjectToDatenFilmCallable(JsonObject aJsonObjec, String aLangCode
@Override
public DatenFilm call() {
DatenFilm film = null;
if(jsonObject != null && jsonObject.has(JSON_OBJECT_KEY_PROGRAM))
try {
if(isValidProgramObject(jsonObject))
{
JsonObject programObject = jsonObject.get(JSON_OBJECT_KEY_PROGRAM).getAsJsonObject();
if(isValidProgramObject(programObject))
String titel = getTitle(jsonObject);
String thema = getSubject(jsonObject);

String beschreibung = getElementValue(jsonObject, JSON_ELEMENT_KEY_SHORT_DESCRIPTION);

String urlWeb = getElementValue(jsonObject, JSON_ELEMENT_KEY_URL);

//https://api.arte.tv/api/player/v1/config/[language:de/fr]/[programId]?platform=ARTE_NEXT
String programId = getElementValue(jsonObject, JSON_ELEMENT_KEY_PROGRAM_ID);
String videosUrl = String.format(ARTE_VIDEO_INFORMATION_URL_PATTERN, langCode, programId);

Gson gson = new GsonBuilder().registerTypeAdapter(ArteVideoDTO.class, new ArteVideoDeserializer()).create();

try(Response responseVideoDetails = executeRequest(videosUrl))
{
String titel = getTitle(programObject);
String thema = getSubject(programObject);

String beschreibung = getElementValue(programObject, JSON_ELEMENT_KEY_SHORT_DESCRIPTION);

String urlWeb = getElementValue(programObject, JSON_ELEMENT_KEY_URL);

//https://api.arte.tv/api/player/v1/config/[language:de/fr]/[programId]
String programId = getElementValue(programObject, JSON_ELEMENT_KEY_PROGRAM_ID);
String videosUrl = String.format(ARTE_VIDEO_INFORMATION_URL_PATTERN, langCode, programId);

Gson gson = new GsonBuilder().registerTypeAdapter(ArteVideoDTO.class, new ArteVideoDeserializer()).create();

try(Response responseVideoDetails = executeRequest(videosUrl))
if(responseVideoDetails.isSuccessful())
{
if(responseVideoDetails.isSuccessful())
ArteVideoDTO video = gson.fromJson(responseVideoDetails.body().string(), ArteVideoDTO.class);

//The duration as time so it can be formatted and co.
LocalTime durationAsTime = durationAsTime(video.getDurationInSeconds());

if (video.getVideoUrls().containsKey(Qualities.NORMAL))
{
ArteVideoDTO video = gson.fromJson(responseVideoDetails.body().string(), ArteVideoDTO.class);
String broadcastBegin = "";

OkHttpClient httpClient = MVHttpClient.getInstance().getHttpClient();
//https://api.arte.tv/api/opa/v3/programs/[language:de/fr]/[programId]
String videosUrlVideoDetails2 = String.format(ARTE_VIDEO_INFORMATION_URL_PATTERN_2, langCode, programId);
Request request = new Request.Builder()
.addHeader(MediathekArte_de.AUTH_HEADER, MediathekArte_de.AUTH_TOKEN)
.url(videosUrlVideoDetails2).build();

//The duration as time so it can be formatted and co.
LocalTime durationAsTime = durationAsTime(video.getDurationInSeconds());

if (video.getVideoUrls().containsKey(Qualities.NORMAL))
try(Response responseVideoDetails2 = httpClient.newCall(request).execute())
{
String broadcastBegin = "";

if(jsonObject.has(JSON_ELEMENT_BROADCAST)) {
broadcastBegin = jsonObject.get(JSON_ELEMENT_BROADCAST).getAsString();
if(responseVideoDetails2.isSuccessful())
{
/*
* Grobe Struktur des Json's:
* {
* "meta": {}
* "programs": [{ (JSON_ELEMENT_BROADCAST_ELTERNKNOTEN_1)
"broadcastProgrammings": [{ (JSON_ELEMENT_BROADCAST_ELTERNKNOTEN_2)
"broadcastBeginRounded": "2016-07-06T02:40:00Z", (JSON_ELEMENT_BROADCAST)
]}
]}
* }
*/
JsonObject jsonObjectVideoDetails2 = gson.fromJson(responseVideoDetails2.body().string(), JsonObject.class);
if(jsonObjectVideoDetails2.isJsonObject() &&
jsonObjectVideoDetails2.get(JSON_ELEMENT_BROADCAST_ELTERNKNOTEN_1).getAsJsonArray().size() > 0 &&
jsonObjectVideoDetails2.get(JSON_ELEMENT_BROADCAST_ELTERNKNOTEN_1).getAsJsonArray().get(0).getAsJsonObject()
.get(JSON_ELEMENT_BROADCAST_ELTERNKNOTEN_2).getAsJsonArray().size() > 0 &&
jsonObjectVideoDetails2.get(JSON_ELEMENT_BROADCAST_ELTERNKNOTEN_1).getAsJsonArray().get(0).getAsJsonObject()
.get(JSON_ELEMENT_BROADCAST_ELTERNKNOTEN_2).getAsJsonArray().get(0).getAsJsonObject()
.has(JSON_ELEMENT_BROADCAST))
{
JsonElement jsonBegin = jsonObjectVideoDetails2
.get(JSON_ELEMENT_BROADCAST_ELTERNKNOTEN_1).getAsJsonArray().get(0).getAsJsonObject()
.get(JSON_ELEMENT_BROADCAST_ELTERNKNOTEN_2).getAsJsonArray().get(0).getAsJsonObject()
.get(JSON_ELEMENT_BROADCAST);
if(jsonBegin != JsonNull.INSTANCE) {
broadcastBegin = jsonBegin.getAsString();
}
}

film = createFilm(thema, urlWeb, titel, video, broadcastBegin, durationAsTime, beschreibung);
} else {
LOG.debug(String.format("Keine \"normale\" Video URL für den Film \"%s\" mit der URL \"%s\". Video Details URL:\"%s\" ", titel, urlWeb, videosUrl));
}
}
} catch (IOException ioException)
{
LOG.error("Beim laden der Informationen eines Filmes für Arte kam es zu Verbindungsproblemen.", ioException);
}
}

film = createFilm(thema, urlWeb, titel, video, broadcastBegin, durationAsTime, beschreibung);
} else {
LOG.debug(String.format("Keine \"normale\" Video URL für den Film \"%s\" mit der URL \"%s\". Video Details URL:\"%s\" ", titel, urlWeb, videosUrl));
}
}
} catch (IOException ioException)
{
LOG.error("Beim laden der Informationen eines Filmes für Arte kam es zu Verbindungsproblemen.", ioException);
}
}
} catch(Exception e) {
e.printStackTrace();
LOG.error(e);
}

return film;
}

Expand Down Expand Up @@ -176,4 +219,4 @@ private LocalTime durationAsTime(long aDurationInSeconds)

return localTime;
}
}
}
29 changes: 25 additions & 4 deletions src/main/java/mServer/crawler/sender/arte/MediathekArte_de.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,10 +46,29 @@

public class MediathekArte_de extends MediathekReader
{
/*
* Informationen zu den ARTE-URLs:
* {} sind nur Makierungen, dass es Platzhalter sind, sie gehören nicht zur URL.
*
* Allgemeine URL eines Films: (050169-002-A = ID des Films); (die-spur-der-steine = Titel)
* http://www.arte.tv/de/videos/{050169-002-A}/{die-spur-der-steine}
*
* Alle Sendungen: (Deutsch = DE; Französisch = FR)
* https://api.arte.tv/api/opa/v3/videos?channel={DE}
*
* Informationen zum Film: (050169-002-A = ID des Films); (de für deutsch / fr für französisch)
* https://api.arte.tv/api/player/v1/config/{de}/{050169-002-A}?platform=ARTE_NEXT
*
* Zweite Quelle für Informationen zum Film: (050169-002-A = ID des Films); (de für deutsch / fr für französisch)
* https://api.arte.tv/api/opa/v3/programs/{de}/{050169-002-A}
*
*/
private static final Logger LOG = LogManager.getLogger(MediathekArte_de.class);
private final static String SENDERNAME = Const.ARTE_DE;
private static final String ARTE_API_TAG_URL_PATTERN = "http://www.arte.tv/guide/api/api/program/%s/scheduled/%s";
private static final DateTimeFormatter ARTE_API_DATEFORMATTER = DateTimeFormatter.ofPattern("yy-MM-dd");
private static final String ARTE_API_TAG_URL_PATTERN = "https://api.arte.tv/api/opa/v3/videos?channel=%s&arteSchedulingDay=%s";
private static final DateTimeFormatter ARTE_API_DATEFORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd");
public static final String AUTH_HEADER = "Authorization";
public static final String AUTH_TOKEN = "Bearer Nzc1Yjc1ZjJkYjk1NWFhN2I2MWEwMmRlMzAzNjI5NmU3NWU3ODg4ODJjOWMxNTMxYzEzZGRjYjg2ZGE4MmIwOA";
protected String LANG_CODE = "de";
protected String URL_CONCERT = "http://concert.arte.tv/de/videos/all";
protected String URL_CONCERT_NOT_CONTAIN = "-STF";
Expand Down Expand Up @@ -105,7 +124,7 @@ private void addConcert() {
private void addTage() {
// http://www.arte.tv/guide/de/plus7/videos?day=-2&page=1&isLoading=true&sort=newest&country=DE
for (int i = 0; i <= 14; ++i) {
String u = String.format(ARTE_API_TAG_URL_PATTERN,LANG_CODE,LocalDate.now().minusDays(i).format(ARTE_API_DATEFORMATTER));
String u = String.format(ARTE_API_TAG_URL_PATTERN,LANG_CODE.toUpperCase(),LocalDate.now().minusDays(i).format(ARTE_API_DATEFORMATTER));
listeThemen.add(new String[]{u});
}
}
Expand Down Expand Up @@ -280,7 +299,9 @@ private void addFilmeForTag(String aUrl) {

MVHttpClient mvhttpClient = MVHttpClient.getInstance();
OkHttpClient httpClient = mvhttpClient.getHttpClient();
Request request = new Request.Builder().url(aUrl).build();
Request request = new Request.Builder()
.addHeader(AUTH_HEADER, AUTH_TOKEN)
.url(aUrl).build();
try
{
Response response = httpClient.newCall(request).execute();
Expand Down

0 comments on commit 0e2ca6b

Please sign in to comment.