integrate mlib (#1091)

* integrate mlib

* fix deprecation

* remove deprecated use of new URL

* fix oboslete funk crawler api param

* remove deprecated function

* fix java doc

* fix sonar

* fix sonar

* refactor mlib-messages

* cleanup

* remove mlib in docs

Author: pidoubleyou

ARCHITECTURE_OVERVIEW.adoc

@@ -88,10 +88,6 @@ Starts all configured crawlers, collects the results and start the filmlist writ
 
 Contains basic logic needed in every crawler like the work splitting.
 
-=== MLib
-
-An internal library which contains the model objects of MediathekView.
-
 === Crawler
 
 The heart of MServer crawlers for the public broadcaster media libraries.

MServer-Config.yaml

@@ -188,9 +188,8 @@ senderConfigurations:
 
 
 # configure string variables
-crawlerApiParams:
-  FUNK_REQUEST_TOKEN: 137782e774d7cadc93dcbffbbde0ce9c
-  
+#crawlerApiParams:
+
 #### COPY ####
 copySettings:
   # En- / disables FTP

docs/architecture/images/Architectural_Overview.svg

@@ -61,7 +61,9 @@
 
     <assertj.version>3.27.4</assertj.version>
     <byte-buddy.version>1.17.7</byte-buddy.version>
+    <commons-compress.version>1.28.0</commons-compress.version>
     <commons-net.version>3.12.0</commons-net.version>
+    <commons-lang.version>3.19.0</commons-lang.version>
     <commons-text.version>1.14.0</commons-text.version>
     <exec-maven-plugin.version>3.5.1</exec-maven-plugin.version>
     <hamcrest.version>3.0</hamcrest.version>
@@ -71,6 +73,7 @@
     <jersey.version>3.1.11</jersey.version>
     <javax-jaxb.version>4.0.2</javax-jaxb.version>
     <log4j2.version>2.25.1</log4j2.version>
+    <jetbrains-annotations.version>22.0.0</jetbrains-annotations.version>
     <junit.version>5.13.4</junit.version>
     <license-maven-plugin.version>2.0.0</license-maven-plugin.version>
     <maven-assembly-plugin.version>3.4.2</maven-assembly-plugin.version>
@@ -86,7 +89,10 @@
     <maven-surefire-plugin.version>3.0.0-M7</maven-surefire-plugin.version>
     <mockito.version>5.19.0</mockito.version>
     <nexus-staging-maven-plugin.version>1.6.13</nexus-staging-maven-plugin.version>
+    <okhttp3.version>4.10.0</okhttp3.version>
     <wiremock.version>2.35.2</wiremock.version>
+    <xz.version>1.10</xz.version>
+    <yacl4j-core.version>0.9.2</yacl4j-core.version>
     <docker-maven-plugin.version>0.40.2</docker-maven-plugin.version>
     <sortpom-maven-plugin.version>3.2.0</sortpom-maven-plugin.version>
   </properties>
@@ -118,11 +124,38 @@
   </dependencyManagement>
 
   <dependencies>
-    <!-- MLib -->
+
+    <dependency>
+      <groupId>org.apache.commons</groupId>
+      <artifactId>commons-compress</artifactId>
+      <version>${commons-compress.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.commons</groupId>
+      <artifactId>commons-lang3</artifactId>
+      <version>${commons-lang.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.commons</groupId>
+      <artifactId>commons-text</artifactId>
+      <version>${commons-text.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.tukaani</groupId>
+      <artifactId>xz</artifactId>
+      <version>${xz.version}</version>
+    </dependency>
+
     <dependency>
-      <groupId>de.mediathekview</groupId>
-      <artifactId>MLib</artifactId>
-      <version>4.0.1-SNAPSHOT</version>
+      <groupId>com.github.fabriziocucci</groupId>
+      <artifactId>yacl4j-core</artifactId>
+      <version>${yacl4j-core.version}</version>
+    </dependency>
+
+    <dependency>
+      <groupId>com.squareup.okhttp3</groupId>
+      <artifactId>okhttp</artifactId>
+      <version>${okhttp3.version}</version>
     </dependency>
 
     <!-- Google Guava for rate limiting on Arte -->
@@ -145,6 +178,10 @@
       <version>${jsoup.version}</version>
     </dependency>
 
+    <dependency>
+      <groupId>org.glassfish.jersey.core</groupId>
+      <artifactId>jersey-client</artifactId>
+    </dependency>
     <dependency>
       <groupId>org.glassfish.jersey.core</groupId>
       <artifactId>jersey-common</artifactId>
@@ -169,7 +206,13 @@
       <artifactId>log4j-core</artifactId>
     </dependency>
 
-    <!-- Unit Tests -->
+    <dependency>
+      <groupId>org.jetbrains</groupId>
+      <artifactId>annotations</artifactId>
+      <version>${jetbrains-annotations.version}</version>
+    </dependency>
+
+      <!-- Unit Tests -->
     <dependency>
       <groupId>org.junit.jupiter</groupId>
       <artifactId>junit-jupiter-api</artifactId>

pom.xml

@@ -0,0 +1,156 @@
+package de.mediathekview.mserver.base.compression;
+
+import java.io.*;
+import java.nio.ByteBuffer;
+import java.nio.channels.Channels;
+import java.nio.channels.ReadableByteChannel;
+import java.nio.channels.WritableByteChannel;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.zip.GZIPInputStream;
+import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream;
+import org.apache.commons.compress.compressors.bzip2.BZip2CompressorOutputStream;
+import org.apache.commons.compress.compressors.xz.XZCompressorInputStream;
+import org.apache.commons.compress.compressors.xz.XZCompressorOutputStream;
+import org.tukaani.xz.LZMA2Options;
+
+/** A util class to work with compressed files. */
+public class CompressionManager {
+  private static CompressionManager instance;
+
+  private CompressionManager() {
+    super();
+  }
+
+  public static CompressionManager getInstance() {
+    if (instance == null) {
+      instance = new CompressionManager();
+    }
+    return instance;
+  }
+
+  public OutputStream compress(
+      final CompressionType aCompressionType, final OutputStream aOutputStream) throws IOException {
+    switch (aCompressionType) {
+      case XZ:
+        return XZCompressorOutputStream.builder().setOutputStream(new BufferedOutputStream(aOutputStream, 512000)).setLzma2Options(new LZMA2Options(9)).get();
+      case GZIP:
+        return new MediathekviewGZIPOutputStrean(new BufferedOutputStream(aOutputStream, 512000));
+      case BZIP:
+        // This uses already the best compression.
+        return new BZip2CompressorOutputStream(new BufferedOutputStream(aOutputStream, 512000));
+      default:
+        throw new IllegalArgumentException(
+            String.format("The type \"%s\" isn't supported.", aCompressionType.name()));
+    }
+  }
+
+  /**
+   * Compresses a file and uses the old name and appends the file appender based on the {@link
+   * CompressionType}.
+   *
+   * @param aCompressionType The type in which should be compressed.
+   * @param aSourceFile The file to compress.
+   * @throws IOException Throws an IOException when the file to be compressed does not exists.
+   */
+  public void compress(final CompressionType aCompressionType, final Path aSourceFile)
+      throws IOException {
+    compress(aCompressionType, aSourceFile, aSourceFile);
+  }
+
+  /**
+   * Compresses a file.
+   *
+   * @param aCompressionType The type in which should be compressed.
+   * @param aSourceFile The file to compress.
+   * @param aTargetPath The path for the compressed file.
+   * @throws IOException Throws an IOException when the file to be compressed does not exists.
+   */
+  public void compress(
+      final CompressionType aCompressionType, final Path aSourceFile, final Path aTargetPath)
+      throws IOException {
+    final Path targetPath =
+        aTargetPath.getFileName().toString().endsWith(aCompressionType.getFileEnding())
+            ? aTargetPath
+            : aTargetPath.resolveSibling(
+                aTargetPath.getFileName().toString() + aCompressionType.getFileEnding());
+
+    try (final BufferedInputStream input = new BufferedInputStream(Files.newInputStream(aSourceFile), 512000);
+         final OutputStream output = compress(aCompressionType, Files.newOutputStream(targetPath))) {
+      fastChannelCopy(Channels.newChannel(input), Channels.newChannel(output));
+    }
+  }
+
+  /**
+   * Uncompresses a {@link InputStream}.
+   *
+   * @param aCompressionType The type in which should be compressed.
+   * @param aInputStream The {@link InputStream} to uncompress.
+   * @return A uncompressed {@link InputStream}
+   * @throws IOException Throws an IOException when the file to be decompressed does not exists.
+   */
+  public InputStream decompress(
+      final CompressionType aCompressionType, final InputStream aInputStream) throws IOException {
+    switch (aCompressionType) {
+      case XZ:
+        return new XZCompressorInputStream(aInputStream);
+      case GZIP:
+        return new GZIPInputStream(aInputStream);
+      case BZIP:
+        return new BZip2CompressorInputStream(aInputStream);
+      default:
+        throw new IllegalArgumentException(
+            String.format("The type \"%s\" isn't supported.", aCompressionType.name()));
+    }
+  }
+
+  /**
+   * Decompresses a file and uses the old name and removes the file appender based on the {@link
+   * CompressionType}.
+   *
+   * @param aCompressionType The type in which should be compressed.
+   * @param aSourceFile The source File to be decompressed
+   * @throws IOException Throws an IOException when the file to be decompressed does not exists.
+   */
+  public void decompress(final CompressionType aCompressionType, final Path aSourceFile)
+      throws IOException {
+    decompress(
+        aCompressionType,
+        aSourceFile,
+        aSourceFile.resolveSibling(
+            aSourceFile.getFileName().toString().replace(aCompressionType.getFileEnding(), "")));
+  }
+
+  /**
+   * Decompresses a file.
+   *
+   * @param aCompressionType The type in which should be compressed.
+   * @param aSourceFile The file to decompress.
+   * @param aTargetPath The path for the decompressed file.
+   * @throws IOException Throws an IOException when the file to be decompressed does not exists.
+   */
+  public void decompress(
+      final CompressionType aCompressionType, final Path aSourceFile, final Path aTargetPath)
+      throws IOException {
+    try (final InputStream input = decompress(aCompressionType, Files.newInputStream(aSourceFile));
+         final OutputStream output = new BufferedOutputStream(Files.newOutputStream(aTargetPath))) {
+      fastChannelCopy(Channels.newChannel(input), Channels.newChannel(output));
+    }
+  }
+
+  private void fastChannelCopy(final ReadableByteChannel src, final WritableByteChannel dest)
+      throws IOException {
+    final ByteBuffer buffer = ByteBuffer.allocateDirect(64 * 1024);
+    while (src.read(buffer) != -1) {
+      buffer.flip();
+      dest.write(buffer);
+      buffer.compact();
+    }
+
+    buffer.flip();
+
+    while (buffer.hasRemaining()) {
+      dest.write(buffer);
+    }
+  }
+}

src/main/java/de/mediathekview/mserver/base/compression/CompressionManager.java

@@ -0,0 +1,17 @@
+package de.mediathekview.mserver.base.compression;
+
+public enum CompressionType {
+  XZ(".xz"),
+  GZIP(".gz"),
+  BZIP(".bz");
+
+  private final String fileEnding;
+
+  CompressionType(final String aFileEnding) {
+    fileEnding = aFileEnding;
+  }
+
+  public String getFileEnding() {
+    return fileEnding;
+  }
+}

src/main/java/de/mediathekview/mserver/base/compression/CompressionType.java

@@ -0,0 +1,13 @@
+package de.mediathekview.mserver.base.compression;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.zip.Deflater;
+import java.util.zip.GZIPOutputStream;
+
+class MediathekviewGZIPOutputStrean extends GZIPOutputStream {
+  MediathekviewGZIPOutputStrean(final OutputStream out) throws IOException {
+    super(out);
+    def.setLevel(Deflater.BEST_COMPRESSION);
+  }
+}

src/main/java/de/mediathekview/mserver/base/compression/MediathekviewGZIPOutputStrean.java

@@ -0,0 +1,8 @@
+package de.mediathekview.mserver.base.config;
+
+/**
+ * A Data-Transfer-Object which holds the actual configuration.
+ */
+public interface ConfigDTO
+{
+}