support libsvm export

Author: shuttie

README.md

@@ -18,15 +18,15 @@ human-friendly API. Currently, is under an active development.
 
 libLTR is published to maven-central for scala 2.12 and 2.13, so for SBT, add this snippet to `build.sbt`:
 ```scala
-libraryDependencies += "io.github.metarank" %% "ltrlib" % "0.1.15"
+libraryDependencies += "io.github.metarank" %% "ltrlib" % "0.1.16"
 ```
 
 For maven:
 ```xml
 <dependency>
   <groupId>io.github.metarank</groupId>
   <artifactId>ltrlib_2.13</artifactId>
-  <version>0.1.15</version>
+  <version>0.1.16</version>
 </dependency>
 ```
 ## Usage

build.sbt

@@ -2,9 +2,9 @@ import Deps._
 
 name := "ltrlib"
 
-version := "0.1.15"
+version := "0.1.16"
 
-scalaVersion := "2.12.17"
+scalaVersion := "2.13.10"
 
 crossScalaVersions := List("2.13.10", "2.12.17")
 
@@ -14,7 +14,7 @@ Test / logBuffered := false
 
 Test / parallelExecution := false
 
-scalacOptions ++= Seq("-feature", "-deprecation", "-target:jvm-1.8")
+scalacOptions ++= Seq("-feature", "-deprecation", "-release:8")
 
 libraryDependencies ++= Seq(
   "org.scalatest"          %% "scalatest"               % scalatestVersion % Test,
@@ -28,7 +28,7 @@ libraryDependencies ++= Seq(
   "io.github.metarank"     %% "cfor"                    % "0.2",
   "io.github.metarank"      % "lightgbm4j"              % "3.3.2-2",
   "io.github.metarank"      % "xgboost-java"            % "1.6.1-2",
-  "com.opencsv"             % "opencsv"                 % "5.7.0",
+  "com.opencsv"             % "opencsv"                 % "5.7.1",
   "org.scala-lang.modules" %% "scala-collection-compat" % "2.8.1"
 )
 

project/build.properties

@@ -1 +1 @@
-sbt.version = 1.7.2
+sbt.version = 1.7.3

src/main/scala/io/github/metarank/ltrlib/output/CSVOutputFormat.scala

@@ -1,19 +1,19 @@
 package io.github.metarank.ltrlib.output
 
-import com.opencsv.CSVWriter
+import com.opencsv.{CSVWriter, CSVWriterBuilder}
 import io.github.metarank.ltrlib.model.{Dataset, DatasetDescriptor, Feature, Query}
 
 import java.io.{OutputStream, OutputStreamWriter}
 
 object CSVOutputFormat extends OutputFormat {
-  def write(stream: OutputStream, data: Dataset) = {
+  def write(stream: OutputStream, data: Dataset, header: Boolean) = {
     val writer = new CSVWriter(new OutputStreamWriter(stream))
-    writer.writeNext(writeHeader(data.desc).toArray)
+    if (header) writer.writeNext(writeHeader(data.desc).toArray, false)
     for {
       query <- data.groups
       line  <- writeGroup(query)
     } {
-      writer.writeNext(line.toArray)
+      writer.writeNext(line.toArray, false)
     }
     writer.close()
     logger.debug(s"wrote ${data.groups.size} groups to CSV file")

src/main/scala/io/github/metarank/ltrlib/output/LibSVMOutputFormat.scala

@@ -0,0 +1,33 @@
+package io.github.metarank.ltrlib.output
+
+import io.github.metarank.ltrlib.model.Dataset
+
+import java.io.OutputStream
+
+object LibSVMOutputFormat extends OutputFormat {
+  def write(data: OutputStream, ds: Dataset) = {
+    for {
+      query <- ds.groups
+      rowid <- 0 until query.rows
+    } {
+      val row  = query.getRow(rowid).zipWithIndex.filter(_._1 != 0).map(x => s"${x._2}:${x._1}")
+      val line = s"${query.labels(rowid)} qid:${query.group} ${row.mkString(" ")}\n"
+      data.write(line.getBytes())
+    }
+  }
+
+  def write(data: OutputStream, groups: OutputStream, ds: Dataset) = {
+    for {
+      query <- ds.groups
+    } {
+      groups.write(s"${query.rows}\n".getBytes())
+      for {
+        rowid <- 0 until query.rows
+      } {
+        val row  = query.getRow(rowid).zipWithIndex.filter(_._1 != 0).map(x => s"${x._2}:${x._1}")
+        val line = s"${query.labels(rowid)} ${row.mkString(" ")}\n"
+        data.write(line.getBytes())
+      }
+    }
+  }
+}

src/test/scala/io/github/metarank/ltrlib/output/CSVOutputFormatTest.scala

@@ -12,21 +12,21 @@ class CSVOutputFormatTest extends AnyFlatSpec with Matchers {
     val desc = DatasetDescriptor(List(SingularFeature("f1"), SingularFeature("f2")))
     val ds   = Dataset(desc, List(Query(desc, List(LabeledItem(1, 1, Array(1.0, 2.0))))))
     val out  = new ByteArrayOutputStream()
-    CSVOutputFormat.write(out, ds)
+    CSVOutputFormat.write(out, ds, true)
     val str = new String(out.toByteArray)
-    str shouldBe """"label","group","f1","f2"
-                   |"1.0","1","1.0","2.0"
+    str shouldBe """label,group,f1,f2
+                   |1.0,1,1.0,2.0
                    |""".stripMargin
   }
 
   it should "export CSV with vectors" in {
     val desc = DatasetDescriptor(List(SingularFeature("f1"), VectorFeature("f2", 2)))
     val ds   = Dataset(desc, List(Query(desc, List(LabeledItem(1, 1, Array(1.0, 2.0, 3.0))))))
     val out  = new ByteArrayOutputStream()
-    CSVOutputFormat.write(out, ds)
+    CSVOutputFormat.write(out, ds, true)
     val str = new String(out.toByteArray)
-    str shouldBe """"label","group","f1","f2_0","f2_1"
-                   |"1.0","1","1.0","2.0","3.0"
+    str shouldBe """label,group,f1,f2_0,f2_1
+                   |1.0,1,1.0,2.0,3.0
                    |""".stripMargin
   }
 }

src/test/scala/io/github/metarank/ltrlib/output/LibSVMOutputFormatTest.scala

@@ -0,0 +1,45 @@
+package io.github.metarank.ltrlib.output
+
+import io.github.metarank.ltrlib.model.{Dataset, DatasetDescriptor, LabeledItem, Query}
+import io.github.metarank.ltrlib.model.Feature.SingularFeature
+import org.scalatest.flatspec.AnyFlatSpec
+import org.scalatest.matchers.should.Matchers
+
+import java.io.ByteArrayOutputStream
+import scala.collection.immutable.List
+
+class LibSVMOutputFormatTest extends AnyFlatSpec with Matchers {
+  val desc = DatasetDescriptor(List(SingularFeature("f1"), SingularFeature("f2")))
+  val ds = Dataset(
+    desc,
+    List(
+      Query(desc, List(LabeledItem(1, 1, Array(1.0, 2.0)), LabeledItem(0, 1, Array(0.0, 0.0)))),
+      Query(desc, List(LabeledItem(1, 2, Array(1.0, 2.0))))
+    )
+  )
+
+  it should "export with qid label" in {
+    val out = new ByteArrayOutputStream()
+    LibSVMOutputFormat.write(out, ds)
+    val str = new String(out.toByteArray)
+    str shouldBe
+      """1.0 qid:1 0:1.0 1:2.0
+        |0.0 qid:1 
+        |1.0 qid:2 0:1.0 1:2.0
+        |""".stripMargin
+  }
+
+  it should "export with separate groups file" in {
+    val data   = new ByteArrayOutputStream()
+    val groups = new ByteArrayOutputStream()
+    LibSVMOutputFormat.write(data, groups, ds)
+    val str  = new String(data.toByteArray)
+    val gstr = new String(groups.toByteArray)
+    str shouldBe
+      """1.0 0:1.0 1:2.0
+        |0.0 
+        |1.0 0:1.0 1:2.0
+        |""".stripMargin
+    gstr shouldBe "2\n1\n"
+  }
+}