fix 992-01-990-989 and remove java preview feature

codingPF · codingPF · commit e8466162cfe6 · 2024-05-24T21:15:01.000+02:00
diff --git a/src/main/java/de/mediathekview/mserver/crawler/ard/ArdConstants.java b/src/main/java/de/mediathekview/mserver/crawler/ard/ArdConstants.java
@@ -3,6 +3,7 @@
 import java.util.HashMap;
 import java.util.Map;
 
+import de.mediathekview.mlib.daten.Resolution;
 import de.mediathekview.mlib.daten.Sender;
 
 public class ArdConstants {
@@ -67,6 +68,17 @@ public class ArdConstants {
   public static final String WEBSITE_URL = "https://www.ardmediathek.de/video/%s";
 
   public static final String BASE_URL_SUBTITLES = "https://classic.ardmediathek.de";
+  
+  // provide the same as master until crawler release
+  public static Resolution getResolutionFromWidth(final int width) {
+    if (width > 1280) {
+      return Resolution.HD;
+    } else if (width > 640) {
+      return Resolution.NORMAL;
+    } else {
+      return Resolution.SMALL;  
+    }
+  }
 
   private ArdConstants() {}
 }
diff --git a/src/main/java/de/mediathekview/mserver/crawler/ard/json/ArdFilmDeserializer.java b/src/main/java/de/mediathekview/mserver/crawler/ard/json/ArdFilmDeserializer.java
@@ -7,6 +7,7 @@
 import de.mediathekview.mlib.daten.Sender;
 import de.mediathekview.mserver.base.utils.GeoLocationGuesser;
 import de.mediathekview.mserver.base.utils.JsonUtils;
+import de.mediathekview.mserver.base.utils.UrlUtils;
 import de.mediathekview.mserver.crawler.ard.ArdConstants;
 import de.mediathekview.mserver.crawler.ard.ArdFilmDto;
 import de.mediathekview.mserver.crawler.ard.ArdFilmInfoDto;
@@ -57,13 +58,16 @@ public class ArdFilmDeserializer implements JsonDeserializer<List<ArdFilmDto>> {
   private static final String ATTRIBUTE_RESOLUTION_H = "maxHResolutionPx";
   private static final String ATTRIBUTE_MIME = "mimeType";
   private static final String ATTRIBUTE_KIND = "kind";
+  private static final String ATTRIBUTE_ADUIO_LANG = "languageCode";
 
   private static final String MARKER_VIDEO_MP4 = "video/mp4"; 
   private static final String MARKER_VIDEO_STANDARD = "standard";
   private static final String MARKER_VIDEO_CATEGORY_MAIN = "main";
   private static final String MARKER_VIDEO_CATEGORY_MPEG = "application/vnd.apple.mpegurl";
   private static final String MARKER_VIDEO_AD = "audio-description";
-  private static final String MARKER_VIDEO_DGS = "sign-language";  
+  private static final String MARKER_VIDEO_DGS = "sign-language";
+  private static final String MARKER_VIDEO_OV = "OV";
+  private static final String MARKER_VIDEO_DE = "deu";
 
   private final ArdVideoInfoJsonDeserializer videoDeserializer;
   private final AbstractCrawler crawler;
@@ -292,7 +296,6 @@ private Film createFilm(
             duration == null ? Duration.ofSeconds(0) : duration);
 
     Optional.ofNullable(description).ifPresent(film::setBeschreibung);
-
     
     film.setGeoLocations(GeoLocationGuesser.getGeoLocations(Sender.ARD, videoInfo.getDefaultVideoUrl()));
     
@@ -354,12 +357,19 @@ private void addDGSUrls(final Film film, final Map<Resolution, String> videoUrls
   private Optional<ArdVideoInfoDto> parseVideos(final JsonObject playerPageObject, final String title) {
     ArdVideoInfoDto allVideoUrls = new ArdVideoInfoDto();
     //
-    final Optional<Map<Resolution, String>> videoInfoStandard = parseVideoUrls(playerPageObject, MARKER_VIDEO_CATEGORY_MAIN, MARKER_VIDEO_STANDARD, MARKER_VIDEO_MP4);
-    final Optional<Map<Resolution, String>> videoInfoAdaptive = parseVideoUrls(playerPageObject, MARKER_VIDEO_CATEGORY_MAIN, MARKER_VIDEO_STANDARD, MARKER_VIDEO_CATEGORY_MPEG);
-    final Optional<Map<Resolution, String>> videoInfoAD = parseVideoUrls(playerPageObject, MARKER_VIDEO_CATEGORY_MAIN, MARKER_VIDEO_AD, MARKER_VIDEO_MP4);
-    final Optional<Map<Resolution, String>> videoInfoDGS = parseVideoUrls(playerPageObject, MARKER_VIDEO_DGS, MARKER_VIDEO_STANDARD, MARKER_VIDEO_MP4);
+    final Optional<Map<Resolution, String>> videoInfoStandard = parseVideoUrls(playerPageObject, MARKER_VIDEO_CATEGORY_MAIN, MARKER_VIDEO_STANDARD, MARKER_VIDEO_MP4, MARKER_VIDEO_DE);
+    final Optional<Map<Resolution, String>> videoInfoAdaptive = parseVideoUrls(playerPageObject, MARKER_VIDEO_CATEGORY_MAIN, MARKER_VIDEO_STANDARD, MARKER_VIDEO_CATEGORY_MPEG, MARKER_VIDEO_DE);
+    final Optional<Map<Resolution, String>> videoInfoAD = parseVideoUrls(playerPageObject, MARKER_VIDEO_CATEGORY_MAIN, MARKER_VIDEO_AD, MARKER_VIDEO_MP4, MARKER_VIDEO_DE);
+    final Optional<Map<Resolution, String>> videoInfoDGS = parseVideoUrls(playerPageObject, MARKER_VIDEO_DGS, MARKER_VIDEO_STANDARD, MARKER_VIDEO_MP4, MARKER_VIDEO_DE);
+    final Optional<Map<Resolution, String>> videoInfoOV = parseVideoUrls(playerPageObject, MARKER_VIDEO_CATEGORY_MAIN, MARKER_VIDEO_STANDARD, MARKER_VIDEO_MP4, MARKER_VIDEO_OV);
     final Optional<Set<String>> subtitles = prepareSubtitleUrl(playerPageObject);
     //
+    // OV is a single film but also included in the standard film
+    if ((title.toUpperCase().contains("(OV)") || title.toUpperCase().contains("(ORIGINALVERSION)")) && videoInfoOV.isPresent()) {
+      allVideoUrls.putAll(videoInfoOV.get());
+      return Optional.of(allVideoUrls);
+    }
+    //
     if (subtitles.isPresent()) {
       allVideoUrls.setSubtitleUrl(subtitles);
     }
@@ -385,16 +395,38 @@ private Optional<ArdVideoInfoDto> parseVideos(final JsonObject playerPageObject,
     }    
     return Optional.of(allVideoUrls);
   }
+  
+  private Optional<Map<Resolution, String>> parseVideoUrls(final JsonObject playerPageObject, String streamType, String aduioType, String mimeType, String language) {
+    Optional<Map<Integer, String>> urls = parseVideoUrlMap(playerPageObject, streamType, aduioType, mimeType, language);
+    if (urls.isEmpty()) {
+      return Optional.empty();
+    }
+    Map<Resolution, String> videoInfo = new EnumMap<>(Resolution.class);
+    for (Map.Entry<Integer, String> entry : urls.get().entrySet()) {
+      Resolution resolution = ArdConstants.getResolutionFromWidth(entry.getKey());
+      if(!videoInfo.containsKey(resolution)) {
+        videoInfo.put(resolution, entry.getValue());
+      }
+    }
+    // issue if we do not have normal res
+    // TODO: FIXME
+    if (!videoInfo.containsKey(Resolution.NORMAL)) {
+      Resolution anyResolution = videoInfo.keySet().stream().findFirst().get();
+      videoInfo.put(Resolution.NORMAL, videoInfo.get(anyResolution));
+      videoInfo.remove(anyResolution);
+    }
+    return Optional.of(videoInfo);
+  }
 
-  private Optional<Map<Resolution, String>> parseVideoUrls(final JsonObject playerPageObject, String streamType, String aduioType, String mimeType) {
+  private Optional<Map<Integer, String>> parseVideoUrlMap(final JsonObject playerPageObject, String streamType, String aduioType, String mimeType, String language) {
     final Optional<JsonObject> mediaCollectionObject = getMediaCollectionObject(playerPageObject);
     if (mediaCollectionObject.isEmpty())
       return Optional.empty();
     final Optional<JsonElement> streams = JsonUtils.getElement(mediaCollectionObject.get(), ELEMENT_STREAMS);
     if (streams.isEmpty() || !streams.get().isJsonArray() || (streams.get().getAsJsonArray().size() == 0))
       return Optional.empty();
     //
-    Map<Resolution, String> videoInfo = new EnumMap<>(Resolution.class);
+    Map<Integer, String> videoInfo = new TreeMap<>(Comparator.reverseOrder());
     for (JsonElement streamsCategory : streams.get().getAsJsonArray()) {
       final Optional<String> streamKind = JsonUtils.getElementValueAsString(streamsCategory, ATTRIBUTE_KIND);
       final Optional<JsonElement> media = JsonUtils.getElement(streamsCategory, ELEMENT_MEDIA);
@@ -409,9 +441,9 @@ private Optional<Map<Resolution, String>> parseVideoUrls(final JsonObject player
               Optional<String> kind = JsonUtils.getElementValueAsString(audios.get().getAsJsonArray().get(0), ATTRIBUTE_KIND);
               Optional<String> resh = JsonUtils.getElementValueAsString(video, ATTRIBUTE_RESOLUTION_H);
               Optional<String> url = JsonUtils.getElementValueAsString(video, ATTRIBUTE_URL);
-              if (url.isPresent() && resh.isPresent() && kind.isPresent() && kind.get().equalsIgnoreCase(aduioType)) {
-                Resolution resolution = Resolution.getResolutionFromWidth(Integer.parseInt(resh.get()));
-                videoInfo.put(resolution, url.get());
+              Optional<String> languageCode = JsonUtils.getElementValueAsString(audios.get().getAsJsonArray().get(0), ATTRIBUTE_ADUIO_LANG);
+              if (url.isPresent() && resh.isPresent() && kind.isPresent() && kind.get().equalsIgnoreCase(aduioType) && languageCode.orElse("").equalsIgnoreCase(language)) {
+                videoInfo.put(Integer.parseInt(resh.get()), UrlUtils.removeParameters(url.get()));
               }
             }
           }
@@ -421,12 +453,6 @@ private Optional<Map<Resolution, String>> parseVideoUrls(final JsonObject player
     if (videoInfo.size() == 0) {
       return Optional.empty();
     }
-    // TODO: FIX ME
-    if (!videoInfo.containsKey(Resolution.NORMAL)) {
-      Resolution r = videoInfo.keySet().stream().findFirst().get();
-      videoInfo.put(Resolution.NORMAL, videoInfo.get(r));
-      videoInfo.remove(r);
-    }
     return Optional.of(videoInfo);
   }
 }
diff --git a/src/main/java/de/mediathekview/mserver/crawler/ard/json/ArdMediaArrayToDownloadUrlsConverter.java b/src/main/java/de/mediathekview/mserver/crawler/ard/json/ArdMediaArrayToDownloadUrlsConverter.java
@@ -145,7 +145,8 @@ private static Resolution getQualityForNumber(final int qualityIndicator) {
       case 1 -> Resolution.SMALL;
       case 3, 4 -> Resolution.HD;
       case 5 -> Resolution.UHD;
-      case 2, default -> Resolution.NORMAL;
+      case 2 -> Resolution.NORMAL;
+      default -> Resolution.NORMAL;
     };
   }
 
diff --git a/src/main/java/de/mediathekview/mserver/crawler/funk/tasks/NexxCloudSessionInitiationTask.java b/src/main/java/de/mediathekview/mserver/crawler/funk/tasks/NexxCloudSessionInitiationTask.java
@@ -37,24 +37,22 @@ public NexxCloudSessionInitiationTask(final AbstractCrawler aCrawler) {
   public Long call() {
     final Gson gson = createGson();
 
-    try(final Client client = createClient()) {
-      final WebTarget target =
-          client.target(FunkUrls.NEXX_CLOUD_SESSION_INIT.getAsString(crawler.getRuntimeConfig()));
+    final Client client = createClient();
+    final WebTarget target =
+        client.target(FunkUrls.NEXX_CLOUD_SESSION_INIT.getAsString(crawler.getRuntimeConfig()));
 
-      final MultivaluedHashMap<String, String> formData = createForm();
+    final MultivaluedHashMap<String, String> formData = createForm();
 
-      try(final Response response = target.request().post(Entity.form(formData)))
-      {
-        if (response.getStatus() == 201) {
-          final String jsonOutput = response.readEntity(String.class);
-          return gson.fromJson(jsonOutput, Long.class);
-        } else {
-          crawler.printErrorMessage();
-          LOG.fatal(
-              "A HTTP error {} occurred when initialising the Nexx cloud session.",
-              response.getStatus());
-        }
-      }
+    final Response response = target.request().post(Entity.form(formData));
+
+    if (response.getStatus() == 201) {
+      final String jsonOutput = response.readEntity(String.class);
+      return gson.fromJson(jsonOutput, Long.class);
+    } else {
+      crawler.printErrorMessage();
+      LOG.fatal(
+          "A HTTP error {} occurred when initialising the Nexx cloud session.",
+          response.getStatus());
     }
     return null;
   }
diff --git a/src/main/java/de/mediathekview/mserver/crawler/zdf/json/ZdfFilmDetailDeserializer.java b/src/main/java/de/mediathekview/mserver/crawler/zdf/json/ZdfFilmDetailDeserializer.java
@@ -303,6 +303,9 @@ private Optional<String> formatEpisodeTitle(Optional<Integer> season, Optional<I
     if (season.isPresent()) {
       result += String.format("S%02d", season.get());
     }
+    if (season.isPresent() && episode.isPresent()) {
+      result += "/";
+    }
     if (episode.isPresent()) {
       result += String.format("E%02d", episode.get());
     }
diff --git a/src/test/java/de/mediathekview/mserver/crawler/ard/json/ArdFilmDeserializerTest.java b/src/test/java/de/mediathekview/mserver/crawler/ard/json/ArdFilmDeserializerTest.java
@@ -258,7 +258,7 @@ public static Collection<Object[]> data() {
             /*DGSsmall */ "",
             /*DGSnormal */ "",
             /*DGShd */ "",
-            /*sub*/ "https://api.ardmediathek.de/player-service/subtitle/webvtt/urn:ard:subtitle:0567b031db73e4b9.vtt",
+            /*sub*/ "",
             /*hd*/ GeoLocations.GEO_DE,
             /*related*/ new ArdFilmInfoDto[0],
             /*sender*/ Optional.of(Sender.ONE),
@@ -346,6 +346,48 @@ public static Collection<Object[]> data() {
             /*hd*/ GeoLocations.GEO_DE,
             /*related*/ new ArdFilmInfoDto[0],
             /*sender*/ Optional.of(Sender.SWR),
+          }, 
+          {
+            /*jsonFile*/ "/ard/ard_item_ignore_OV.json",
+            /*topic*/ "You shall not lie - Tödliche Geheimnisse",
+            /*title*/ "Folge 4: Der Verrat (S01/E04)",
+            /*description*/ "Zwischen Macarena und Iván, der jetzt von ihrer Schwangerschaft weiß, kommt es zu einer dramatischen Auseinandersetzung auf See. Erschöpft schwimmt der 18-Jährige an Land und beschuldigt sie, ihn vom Boot gestoßen zu haben. Die Lehrerin muss nun nicht nur die Folgen ihres Seitensprungs fürchten, sondern auch eine Anzeige wegen versuchten Totschlags.  +++    Sechsteilige Thrillerserie, Spanien 2021\n.....",
+            /*date*/ LocalDateTime.parse("2022-10-15T02:50"),
+            /*duration*/ Duration.parse("PT42M18S"),
+            /*small*/ "https://pd-videos.daserste.de/de/2022/10/11/1cb6c5d8-c0f4-4868-ac42-2a06b7a3381f/JOB_461836_sendeton_640x360-50p-1200kbit.mp4",
+            /*normal*/ "https://pd-videos.daserste.de/de/2022/10/11/1cb6c5d8-c0f4-4868-ac42-2a06b7a3381f/JOB_461836_sendeton_960x540-50p-1600kbit.mp4",
+            /*hd*/ "https://pd-videos.daserste.de/de/2022/10/11/1cb6c5d8-c0f4-4868-ac42-2a06b7a3381f/JOB_461836_sendeton_1280x720-50p-3200kbit.mp4",
+            /*ADsmall*/ "https://pd-videos.daserste.de/de/2022/10/11/1cb6c5d8-c0f4-4868-ac42-2a06b7a3381f/JOB_461837_internationalerton_640x360-50p-1200kbit.mp4",
+            /*ADnormal*/ "https://pd-videos.daserste.de/de/2022/10/11/1cb6c5d8-c0f4-4868-ac42-2a06b7a3381f/JOB_461837_internationalerton_960x540-50p-1600kbit.mp4",
+            /*ADhd*/ "https://pd-videos.daserste.de/de/2022/10/11/1cb6c5d8-c0f4-4868-ac42-2a06b7a3381f/JOB_461837_internationalerton_1280x720-50p-3200kbit.mp4",
+            /*DGSsmall */ "",
+            /*DGSnormal */ "",
+            /*DGShd */ "",
+            /*sub*/ "https://api.ardmediathek.de/player-service/subtitle/webvtt/urn:ard:subtitle:efab8bf55007171e.vtt",
+            /*hd*/ GeoLocations.GEO_DE,
+            /*related*/ new ArdFilmInfoDto[0],
+            /*sender*/ Optional.of(Sender.ARD),
+          },
+          {
+            /*jsonFile*/ "/ard/ard_item_OV.json",
+            /*topic*/ "Murdoch Mysteries",
+            /*title*/ "Folge 12: Der küssende Bandit (S04/E12) - (Originalversion)",
+            /*description*/ "Während sich Dr. Ogden auf ihre Hochzeit vorbereitet, muss Murdoch versuchen, den 'Küssenden Banditen' aufzuhalten, einen umstrittenen Bankräuber, der schnell zum Volkshelden aufsteigt.",
+            /*date*/ LocalDateTime.parse("2024-04-24T22:50"),
+            /*duration*/ Duration.parse("PT45M44S"),
+            /*small*/ "https://wdrmedien-a.akamaihd.net/medp/ondemand/de/fsk12/310/3103788/3103788_57016296.mp4",
+            /*normal*/ "https://wdrmedien-a.akamaihd.net/medp/ondemand/de/fsk12/310/3103788/3103788_57016297.mp4",
+            /*hd*/ "https://wdrmedien-a.akamaihd.net/medp/ondemand/de/fsk12/310/3103788/3103788_57016298.mp4",
+            /*ADsmall*/ "",
+            /*ADnormal*/ "",
+            /*ADhd*/ "",
+            /*DGSsmall */ "",
+            /*DGSnormal */ "",
+            /*DGShd */ "",
+            /*sub*/ "",
+            /*hd*/ GeoLocations.GEO_DE,
+            /*related*/ new ArdFilmInfoDto[0],
+            /*sender*/ Optional.of(Sender.ONE),
           }
         });
   }
diff --git a/src/test/java/de/mediathekview/mserver/crawler/zdf/json/ZdfFilmDetailDeserializerTest.java b/src/test/java/de/mediathekview/mserver/crawler/zdf/json/ZdfFilmDetailDeserializerTest.java
@@ -162,7 +162,7 @@ public static Collection<Object[]> data() {
             "/zdf/zdf_film_details_with_episodes.json",
             Sender.ZDF,
             "The Rookie",
-            "Der Prozess (S05E01)",
+            "Der Prozess (S05/E01)",
             LocalDateTime.of(2024, 4, 11, 20, 15, 0),
             Duration.ofMinutes(41).plusSeconds(6),
             "Officer John Nolan sieht sich wieder mit der Serienmörderin Rosalind Dyer konfrontiert. Vor Beginn der Gerichtsverhandlung gelingt ihr die Flucht und sie muss erneut gefasst werden.",
diff --git a/src/test/resources/ard/ard_item_OV.json b/src/test/resources/ard/ard_item_OV.json
diff --git a/src/test/resources/ard/ard_item_ignore_OV.json b/src/test/resources/ard/ard_item_ignore_OV.json

Original file line number	Diff line number	Diff line change
`@@ -145,7 +145,8 @@ private static Resolution getQualityForNumber(final int qualityIndicator) {`
`145`	`145`	`case 1 -> Resolution.SMALL;`
`146`	`146`	`case 3, 4 -> Resolution.HD;`
`147`	`147`	`case 5 -> Resolution.UHD;`
`148`		`- case 2, default -> Resolution.NORMAL;`
	`148`	`+ case 2 -> Resolution.NORMAL;`
	`149`	`+ default -> Resolution.NORMAL;`
`149`	`150`	`};`
`150`	`151`	`}`
`151`	`152`
Original file line number	Diff line number	Diff line change
`@@ -303,6 +303,9 @@ private Optional<String> formatEpisodeTitle(Optional<Integer> season, Optional<I`
`303`	`303`	`if (season.isPresent()) {`
`304`	`304`	`result += String.format("S%02d", season.get());`
`305`	`305`	`}`
	`306`	`+ if (season.isPresent() && episode.isPresent()) {`
	`307`	`+ result += "/";`
	`308`	`+ }`
`306`	`309`	`if (episode.isPresent()) {`
`307`	`310`	`result += String.format("E%02d", episode.get());`
`308`	`311`	`}`