Merge branch 'hotfix/2.7.2'

fixed: WDR: Sendung Lokalzeit unvollständig
fixed: ZDF: Requests liefern 403

closed #74
closed #84

build.gradle

@@ -9,7 +9,7 @@ import org.apache.tools.ant.filters.ReplaceTokens
 sourceCompatibility = 1.8
 targetCompatibility = 1.8
 group = 'de.mediathekview'
-version = '2.7.0'
+version = '2.7.2'
 
 def jarName = 'MServer.jar'
 def mainClass = 'mServer.Main'

src/main/java/mServer/crawler/sender/MediathekWdr.java

@@ -267,7 +267,7 @@ private void sendungsSeitenSuchen2(String strUrl, String th) {
             String url;
             String titel;
             String dauer;
-            String datum;
+            String datum = "";
             String thema;
             long duration = 0;
 
@@ -297,6 +297,44 @@ private void sendungsSeitenSuchen2(String strUrl, String th) {
                 sendungsSeitenSuchenNeu(strUrl, sendungsSeite2, thema);
             }
 
+            //Lokalzeit
+            // <div class="teaser video">
+            pos = 0;
+            while (!Config.getStop() && (pos = sendungsSeite2.indexOf("<div class=\"teaser video\">", pos)) != -1) {
+                pos += MUSTER_URL.length();
+                url = sendungsSeite2.extract("<a href=\"/mediathek/video/sendungen/", "\"", pos);
+                if (!url.equals("")) {
+                    url = "http://www1.wdr.de/mediathek/video/sendungen/" + url;
+
+                    titel = sendungsSeite2.extract("</span>", "<", pos).trim();
+                    titel = titel.replace("\n", "");
+
+//                    datum = sendungsSeite2.extract("<p class=\"subtitle\">", "|", pos).trim();
+//                    if (datum.length() != 10) {
+//                        datum = "";
+//                    }
+                    dauer = sendungsSeite2.extract("<span class=\"hidden\">L&auml;nge: </span>", "<", pos).trim();
+                    try {
+                        if (!dauer.equals("")) {
+                            String[] parts = dauer.split(":");
+                            duration = 0;
+                            long power = 1;
+                            for (int i = parts.length - 1; i >= 0; i--) {
+                                duration += Long.parseLong(parts[i]) * power;
+                                power *= 60;
+                            }
+                        }
+                    } catch (Exception ex) {
+                        Log.errorLog(915263654, ex, strUrl);
+                    }
+
+                    //weiter gehts
+                    addFilm1(thema, titel, url, duration, datum);
+                } else {
+                    Log.errorLog(731201247, "keine Url" + strUrl);
+                }
+            }
+
             pos = 0;
             while (!Config.getStop() && (pos = sendungsSeite2.indexOf(MUSTER_URL, pos)) != -1) {
                 pos += MUSTER_URL.length();
@@ -307,10 +345,10 @@ private void sendungsSeitenSuchen2(String strUrl, String th) {
                     titel = sendungsSeite2.extract("<span class=\"hidden\">Video:</span>", "<", pos).trim();
                     titel = titel.replace("\n", "");
 
-                    datum = sendungsSeite2.extract("<p class=\"programInfo\">", "|", pos).trim();
-                    if (datum.length() != 8) {
-                        datum = "";
-                    }
+//                    datum = sendungsSeite2.extract("<p class=\"programInfo\">", "|", pos).trim();
+//                    if (datum.length() != 8) {
+//                        datum = "";
+//                    }
                     dauer = sendungsSeite2.extract("<span class=\"hidden\">L&auml;nge: </span>", "<", pos).trim();
                     try {
                         if (!dauer.equals("")) {

src/main/java/mServer/crawler/sender/newsearch/ZDFClient.java

@@ -14,8 +14,8 @@
 /**
  * jersey client of ZDF
  */
-public class ZDFClient {
-    
+public class ZDFClient
+{
     private static final String ZDF_SEARCH_URL = "https://api.zdf.de/search/documents";
     private static final String HEADER_ACCESS_CONTROL_REQUEST_HEADERS = "Access-Control-Request-Headers";
     private static final String HEADER_ACCESS_CONTROL_REQUEST_METHOD = "access-control-request-method";
@@ -28,60 +28,101 @@ public class ZDFClient {
     private static final String HOST = "api.zdf.de";
     private static final String ORIGIN = "https://www.zdf.de";
     private static final String USER_AGENT = "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:50.0) Gecko/20100101 Firefox/50.0";
-    private static final String API_TOKEN = "Bearer f4ba81fa117681c42383194a7103251db2981962";
-
+    private static final String API_TOKEN_SEARCH = "Bearer f4ba81fa117681c42383194a7103251db2981962";
+    private static final String API_TOKEN_PATTERN = "Bearer %s";
+
     private final Client client;
     private final Gson gson;
-
-    public ZDFClient() {
-        client = Client.create();        
+
+    public ZDFClient()
+    {
+        client = Client.create();
         gson = new Gson();
     }
-
-    public WebResource createSearchResource() {
+
+    public WebResource createSearchResource()
+    {
         return createResource(ZDF_SEARCH_URL);
     }
-
-    public WebResource createResource(String aUrl) {
+
+    public WebResource createResource(String aUrl)
+    {
         return client.resource(aUrl);
     }
-
-    public JsonObject execute(WebResource webResource) {
+
+    public JsonObject execute(WebResource webResource, ZDFClientMode aMode)
+    {
+        String apiToken = loadApiToken(aMode);
+
         ClientResponse response = webResource.header(HEADER_ACCESS_CONTROL_REQUEST_HEADERS, ACCESS_CONTROL_API_AUTH)
-                    .header(HEADER_ACCESS_CONTROL_REQUEST_METHOD, ACCESS_CONTROL_REQUEST_METHOD_GET)
-                    .header(HEADER_API_AUTH, API_TOKEN)
-                    .header(HEADER_HOST, HOST)
-                    .header(HEADER_ORIGIN, ORIGIN)
-                    .header(HEADER_USER_AGENT, USER_AGENT)
-                    .get(ClientResponse.class);
+                .header(HEADER_ACCESS_CONTROL_REQUEST_METHOD, ACCESS_CONTROL_REQUEST_METHOD_GET)
+                .header(HEADER_API_AUTH, apiToken)
+                .header(HEADER_HOST, HOST)
+                .header(HEADER_ORIGIN, ORIGIN)
+                .header(HEADER_USER_AGENT, USER_AGENT)
+                .get(ClientResponse.class);
 
         Log.sysLog("Lade Seite: " + webResource.getURI());
-
-        if (response.getStatus() == 200) {
+
+        if (response.getStatus() == 200)
+        {
             return handleOk(response);
-        } else {
+        } else
+        {
             Log.errorLog(496583258, "Lade Seite " + webResource.getURI() + " fehlgeschlagen: " + response.getStatus());
             increment(RunSender.Count.FEHLER);
             return null;
         }
     }
-
-    private JsonObject handleOk(ClientResponse response) {
+
+    private String loadApiToken(final ZDFClientMode aMode)
+    {
+        String apiToken;
+        switch (aMode)
+        {
+            case SEARCH:
+                apiToken = API_TOKEN_SEARCH;
+                break;
+            case VIDEO:
+                apiToken = loadApiTokenVideos();
+                break;
+            default:
+                apiToken = loadApiTokenVideos();
+                break;
+        }
+        return apiToken;
+    }
+
+    private String loadApiTokenVideos()
+    {
+        return String.format(API_TOKEN_PATTERN, ZDFConfigurationLoader.getInstance().loadConfig().getApiToken());
+    }
+
+    private JsonObject handleOk(ClientResponse response)
+    {
         increment(RunSender.Count.ANZAHL);
 
         String jsonOutput = response.getEntity(String.class);
-        
+
         long bytes = jsonOutput.length();
         increment(RunSender.Count.SUM_DATA_BYTE, bytes);
         increment(RunSender.Count.SUM_TRAFFIC_BYTE, bytes);
 
-        return gson.fromJson(jsonOutput, JsonObject.class);        
+        return gson.fromJson(jsonOutput, JsonObject.class);
     }
-
-    private void increment(RunSender.Count count) {
+
+    private void increment(RunSender.Count count)
+    {
         FilmeSuchen.listeSenderLaufen.inc(Const.ZDF, count);
     }
-    private void increment(RunSender.Count count, long value) {
+
+    private void increment(RunSender.Count count, long value)
+    {
         FilmeSuchen.listeSenderLaufen.inc(Const.ZDF, count, value);
     }
+
+    enum ZDFClientMode
+    {
+        SEARCH, VIDEO;
+    }
 }

src/main/java/mServer/crawler/sender/newsearch/ZDFConfigurationDTO.java

@@ -0,0 +1,48 @@
+package mServer.crawler.sender.newsearch;
+
+import java.io.Serializable;
+
+/**
+ * A Data-Transfer-Object to transfer the ZDF configuration information.
+ */
+public class ZDFConfigurationDTO implements Serializable
+{
+    private static final long serialVersionUID = -445386435734116784L;
+    private String apiToken;
+
+    /**
+     * @param aApiToken The ZDF api authentication token.
+     */
+    public ZDFConfigurationDTO(final String aApiToken)
+    {
+        setApiToken(aApiToken);
+    }
+
+    public String getApiToken()
+    {
+        return apiToken;
+    }
+
+    public void setApiToken(final String aApiToken)
+    {
+        apiToken = aApiToken;
+    }
+
+    @Override
+    public boolean equals(final Object o)
+    {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+
+        final ZDFConfigurationDTO that = (ZDFConfigurationDTO) o;
+
+        return getApiToken() != null ? getApiToken().equals(that.getApiToken()) : that.getApiToken() == null;
+    }
+
+    @Override
+    public int hashCode()
+    {
+        return getApiToken() != null ? getApiToken().hashCode() : 0;
+    }
+
+}

src/main/java/mServer/crawler/sender/newsearch/ZDFConfigurationDTODeserializer.java

@@ -0,0 +1,33 @@
+package mServer.crawler.sender.newsearch;
+
+import com.google.gson.*;
+import mSearch.tool.Log;
+
+import java.lang.reflect.Type;
+
+/**
+ * A JSON deserializer to gather the needed information for a {@link ZDFConfigurationDTO}.
+ */
+public class ZDFConfigurationDTODeserializer implements JsonDeserializer<ZDFConfigurationDTO>
+{
+
+    public static final String JSON_ELEMENT_API_TOKEN = "apiToken";
+
+    @Override
+    public ZDFConfigurationDTO deserialize(final JsonElement aJsonElement, final Type aTypeOfT, final JsonDeserializationContext aJsonDeserializationContext) throws JsonParseException
+    {
+        ZDFConfigurationDTO dto = null;
+        try {
+            JsonObject targetObject = aJsonElement.getAsJsonObject();
+            JsonElement apiTokenElement = targetObject.get(JSON_ELEMENT_API_TOKEN);
+
+            String apiToken = apiTokenElement.getAsString();
+
+            dto = new ZDFConfigurationDTO(apiToken);
+        } catch (Exception ex) {
+            Log.errorLog(496583255, ex);
+        }       
+
+        return dto;
+    }
+}

src/main/java/mServer/crawler/sender/newsearch/ZDFConfigurationLoader.java

@@ -0,0 +1,60 @@
+package mServer.crawler.sender.newsearch;
+
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+import com.sun.jersey.api.client.Client;
+import com.sun.jersey.api.client.ClientResponse;
+import com.sun.jersey.api.client.WebResource;
+import mSearch.tool.Log;
+
+/**
+ * A simple singelton to read the ZDF configuration just once per runtime.
+ */
+public class ZDFConfigurationLoader
+{
+    public static final String ZDF_CONFIGURATION_URL = "https://www.zdf.de/ZDFplayer/configs/zdf/zdf2016/configuration.json";
+    private static final String USER_AGENT = "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:50.0) Gecko/20100101 Firefox/50.0";
+    private static final String HEADER_USER_AGENT = "user-agent";
+    private static ZDFConfigurationLoader instance;
+
+    private ZDFConfigurationDTO config;
+
+    private ZDFConfigurationLoader()
+    {
+        config = null;
+    }
+
+    public static ZDFConfigurationLoader getInstance()
+    {
+        if (instance == null)
+        {
+            instance = new ZDFConfigurationLoader();
+        }
+        return instance;
+    }
+
+    public ZDFConfigurationDTO loadConfig()
+    {
+        if (config == null)
+        {
+            WebResource webResource = Client.create().resource(ZDF_CONFIGURATION_URL);
+            ClientResponse response = webResource
+                    .header(HEADER_USER_AGENT, USER_AGENT)
+                    .get(ClientResponse.class);
+
+            if (response.getStatus() == 200)
+            {
+                Gson gson = new GsonBuilder()
+                        .registerTypeAdapter(ZDFConfigurationDTO.class, new ZDFConfigurationDTODeserializer())
+                        .create();
+                config = gson.fromJson(response.getEntity(String.class), ZDFConfigurationDTO.class);
+            } else
+            {
+                Log.errorLog(496583428, "Lade der Config Seite " + webResource.getURI() + " fehlgeschlagen: " + response.getStatus());
+                config = new ZDFConfigurationDTO("");
+            }
+
+        }
+        return config;
+    }
+}

src/main/java/mServer/crawler/sender/newsearch/ZDFEntryTask.java

@@ -40,14 +40,14 @@ protected VideoDTO compute() {
                 // read film details
                 String infoUrl = zdfEntryDTO.getEntryGeneralInformationUrl();
                 WebResource webResourceInfo = client.createResource(infoUrl);
-                JsonObject baseObjectInfo = client.execute(webResourceInfo);
+                JsonObject baseObjectInfo = client.execute(webResourceInfo, ZDFClient.ZDFClientMode.VIDEO);
                 if(baseObjectInfo != null) {
                     dto = gson.fromJson(baseObjectInfo, VideoDTO.class);
                     if(dto != null) {
                         // read download details
                         String downloadUrl = zdfEntryDTO.getEntryDownloadInformationUrl();
                         WebResource webResourceDownload = client.createResource(downloadUrl);
-                        JsonObject baseObjectDownload = client.execute(webResourceDownload);
+                        JsonObject baseObjectDownload = client.execute(webResourceDownload, ZDFClient.ZDFClientMode.VIDEO);
                         if(baseObjectDownload != null) {
                             DownloadDTO downloadDto = gson.fromJson(baseObjectDownload, DownloadDTO.class);
                             dto.setDownloadDto(downloadDto);

src/main/java/mServer/crawler/sender/newsearch/ZDFSearchTask.java

@@ -97,7 +97,7 @@ private JsonObject loadPage() {
 
         Log.sysLog("Lade Seite: " + webResource.getURI());
 
-        return client.execute(webResource);        
+        return client.execute(webResource, ZDFClient.ZDFClientMode.SEARCH);
     }
 
 }