Merge branch 'hotfix/2.7.6'

fixed: ZDF Crawler hat Probleme mit ZDFNeo

closed #117

build.gradle

@@ -9,7 +9,7 @@ import org.apache.tools.ant.filters.ReplaceTokens
 sourceCompatibility = 1.8
 targetCompatibility = 1.8
 group = 'de.mediathekview'
-version = '2.7.5'
+version = '2.7.6'
 
 def jarName = 'MServer.jar'
 def mainClass = 'mServer.Main'

src/main/java/mServer/crawler/sender/newsearch/ZDFEntryDTODeserializer.java

@@ -3,37 +3,69 @@
 import com.google.gson.*;
 
 import java.lang.reflect.Type;
+
 import mSearch.tool.Log;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 
 /**
  * A JSON deserializer to gather the needed information for a {@link ZDFEntryDTO}.
  */
 public class ZDFEntryDTODeserializer implements JsonDeserializer<ZDFEntryDTO>
 {
-
-    public static final String JSON_ELEMENT_DOWNLOAD_INFORMATION_URL = "http://zdf.de/rels/streams/ptmd-template";
-    public static final String JSON_OBJ_ELEMENT_TARGET = "http://zdf.de/rels/target";
-    public static final String JSON_OBJ_ELEMENT_MAIN_VIDEO_CONTENT = "mainVideoContent";
-    public static final String JSON_ELEMENT_GENERAL_INFORMATION_URL = "canonical";
+    private static final Logger LOG = LogManager.getLogger(ZDFEntryDTODeserializer.class);
+    private static final String JSON_ELEMENT_DOWNLOAD_INFORMATION_URL = "http://zdf.de/rels/streams/ptmd-template";
+    private static final String JSON_OBJ_ELEMENT_TARGET = "http://zdf.de/rels/target";
+    private static final String JSON_OBJ_ELEMENT_MAIN_VIDEO_CONTENT = "mainVideoContent";
+    private static final String JSON_ELEMENT_GENERAL_INFORMATION_URL = "canonical";
+    public static final String JSON_OBJ_VIDEO_PAGE_TEASER = "http://zdf.de/rels/content/video-page-teaser";
+    public static final String PLACEHOLDER_PLAYER_ID = "{playerId}";
+    public static final String PLAYER_ID = "ngplayer_2_3";
 
     @Override
     public ZDFEntryDTO deserialize(final JsonElement aJsonElement, final Type aTypeOfT, final JsonDeserializationContext aJsonDeserializationContext) throws JsonParseException
     {
         ZDFEntryDTO dto = null;
-        try {
+        try
+        {
             JsonObject targetObject = aJsonElement.getAsJsonObject().getAsJsonObject(JSON_OBJ_ELEMENT_TARGET);
-            JsonObject mainVideoContentObject = targetObject.getAsJsonObject(JSON_OBJ_ELEMENT_MAIN_VIDEO_CONTENT).getAsJsonObject(JSON_OBJ_ELEMENT_TARGET);
-            final JsonElement entryGeneralInformationUrlElement = targetObject.get(JSON_ELEMENT_GENERAL_INFORMATION_URL);
-            final JsonElement entryDownloadInformationUrlElement = mainVideoContentObject.get(JSON_ELEMENT_DOWNLOAD_INFORMATION_URL);
-
-            String downloadUrl = entryDownloadInformationUrlElement.getAsString()
-                    .replace("{playerId}", "ngplayer_2_3");
+            if (null == targetObject)
+            {
+                LOG.error("Can't find an JSON Target Object Element for Entry.");
+                LOG.debug("Entry: " + aJsonElement.toString());
+            } else
+            {
+                JsonObject mainVideoContentObject;
+                if (!targetObject.has(JSON_OBJ_ELEMENT_MAIN_VIDEO_CONTENT) && targetObject.has(JSON_OBJ_VIDEO_PAGE_TEASER) && targetObject.getAsJsonObject(JSON_OBJ_VIDEO_PAGE_TEASER).has(JSON_OBJ_ELEMENT_MAIN_VIDEO_CONTENT))
+                {
+                    targetObject = targetObject.getAsJsonObject(JSON_OBJ_VIDEO_PAGE_TEASER);
+                }
 
-            dto = new ZDFEntryDTO(entryGeneralInformationUrlElement.getAsString(), downloadUrl);
-        } catch (Exception ex) {
+                mainVideoContentObject = targetObject.getAsJsonObject(JSON_OBJ_ELEMENT_MAIN_VIDEO_CONTENT);
+                if (mainVideoContentObject != null)
+                {
+                    dto = buildZDFEntryDTO(targetObject, mainVideoContentObject);
+                }
+            }
+        } catch (Exception ex)
+        {
             Log.errorLog(496583255, ex);
-        }       
+            LOG.debug("Entry: " + aJsonElement.toString());
+        }
+
 
         return dto;
     }
+
+    private ZDFEntryDTO buildZDFEntryDTO(JsonObject aTargetObject, final JsonObject aMainVideoContentObject)
+    {
+        JsonObject elementTargetObject = aMainVideoContentObject.getAsJsonObject(JSON_OBJ_ELEMENT_TARGET);
+        final JsonElement entryGeneralInformationUrlElement = aTargetObject.get(JSON_ELEMENT_GENERAL_INFORMATION_URL);
+        final JsonElement entryDownloadInformationUrlElement = elementTargetObject.get(JSON_ELEMENT_DOWNLOAD_INFORMATION_URL);
+
+        String downloadUrl = entryDownloadInformationUrlElement.getAsString()
+                .replace(PLACEHOLDER_PLAYER_ID, PLAYER_ID);
+
+        return new ZDFEntryDTO(entryGeneralInformationUrlElement.getAsString(), downloadUrl);
+    }
 }

src/main/java/mServer/crawler/sender/newsearch/ZDFEntryTask.java

@@ -7,59 +7,69 @@
 import com.sun.jersey.api.client.WebResource;
 
 import java.util.concurrent.RecursiveTask;
+
 import mSearch.Config;
 import mSearch.tool.Log;
 
 /**
  * Searches all information required for a film
  */
-public class ZDFEntryTask extends RecursiveTask<VideoDTO> {
+public class ZDFEntryTask extends RecursiveTask<VideoDTO>
+{
 
     private static final long serialVersionUID = 1L;
-    
+
     private final ZDFClient client;
     private final ZDFEntryDTO zdfEntryDTO;
     private final Gson gson;
-
-    public ZDFEntryTask(ZDFEntryDTO aEntryDto) {
+
+    public ZDFEntryTask(ZDFEntryDTO aEntryDto)
+    {
         client = new ZDFClient();
-        zdfEntryDTO = aEntryDto;                
+        zdfEntryDTO = aEntryDto;
         gson = new GsonBuilder()
                 .registerTypeAdapter(VideoDTO.class, new ZDFVideoDTODeserializer())
                 .registerTypeAdapter(DownloadDTO.class, new ZDFDownloadDTODeserializer())
                 .create();
     }
-    
+
     @Override
-    protected VideoDTO compute() {
+    protected VideoDTO compute()
+    {
 
         VideoDTO dto = null;
 
-        if(!Config.getStop()) {
-            try {
+        if (!Config.getStop())
+        {
+            try
+            {
                 // read film details
                 String infoUrl = zdfEntryDTO.getEntryGeneralInformationUrl();
                 WebResource webResourceInfo = client.createResource(infoUrl);
                 JsonObject baseObjectInfo = client.execute(webResourceInfo, ZDFClient.ZDFClientMode.VIDEO);
-                if(baseObjectInfo != null) {
+                if (baseObjectInfo != null)
+                {
                     dto = gson.fromJson(baseObjectInfo, VideoDTO.class);
-                    if(dto != null) {
+                    if (dto != null)
+                    {
                         // read download details
                         String downloadUrl = zdfEntryDTO.getEntryDownloadInformationUrl();
                         WebResource webResourceDownload = client.createResource(downloadUrl);
                         JsonObject baseObjectDownload = client.execute(webResourceDownload, ZDFClient.ZDFClientMode.VIDEO);
-                        if(baseObjectDownload != null) {
+                        if (baseObjectDownload != null)
+                        {
                             DownloadDTO downloadDto = gson.fromJson(baseObjectDownload, DownloadDTO.class);
                             dto.setDownloadDto(downloadDto);
                         }
                     }
                 }
-            } catch (Exception ex) {
+            } catch (Exception ex)
+            {
                 Log.errorLog(496583202, ex, "Exception parsing " + zdfEntryDTO.getEntryGeneralInformationUrl());
                 dto = null;
             }
         }
-        
+
         return dto;
-    }    
+    }
 }

src/main/java/mServer/crawler/sender/newsearch/ZDFSearchTask.java

@@ -16,8 +16,6 @@ public class ZDFSearchTask extends RecursiveTask<Collection<VideoDTO>>
     private static final String PROPERTY_HAS_VIDEO = "hasVideo";
     private static final String PROPERTY_SEARCHPARAM_Q = "q";
     private static final String SEARCH_ALL = "*";
-    private static final String PROPERTY_TYPES = "types";
-    private static final String TYPE_PAGE_VIDEO = "page-video";
     private static final String PROPERTY_SORT_ORDER = "sortOrder";
     private static final String SORT_ORDER_DESC = "desc";
     private static final String PROPERTY_DATE_FROM = "from";
@@ -88,7 +86,6 @@ private JsonObject loadPage() {
         WebResource webResource = client.createSearchResource()
                     .queryParam(PROPERTY_HAS_VIDEO, Boolean.TRUE.toString())
                     .queryParam(PROPERTY_SEARCHPARAM_Q, SEARCH_ALL)
-                    .queryParam(PROPERTY_TYPES, TYPE_PAGE_VIDEO)
                     .queryParam(PROPERTY_SORT_ORDER, SORT_ORDER_DESC)
                     .queryParam(PROPERTY_DATE_FROM, today.minusDays(days).format(DATE_TIME_FORMAT))
                     .queryParam(PROPERTY_DATE_TO, today.plusMonths(1).format(DATE_TIME_FORMAT))

src/main/resources/log4j2.xml

@@ -0,0 +1,20 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<Configuration status="WARN">
+    <Appenders>
+        <Console name="Console" target="SYSTEM_OUT">
+            <PatternLayout pattern="%d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n"/>
+        </Console>
+        <!--For logging to file:
+        <File name="File" fileName="logs/error.log" value="warn">
+            <PatternLayout>
+                <Pattern>%d %p %c{1.} [%t] %m%n</Pattern>
+            </PatternLayout>
+        </File>-->
+    </Appenders>
+    <Loggers>
+        <Root level="info"><!-- for debug set to "debug" -->
+            <AppenderRef ref="Console"/>
+            <!--For logging to file: <AppenderRef ref="File"/>-->
+        </Root>
+    </Loggers>
+</Configuration>