Skip to content

Commit

Permalink
[query] Remove jackson string length restriction in Spark/Local backends
Browse files Browse the repository at this point in the history
Resolves #14650
  • Loading branch information
chrisvittal committed Aug 5, 2024
1 parent 84dd26c commit fd10c0b
Show file tree
Hide file tree
Showing 3 changed files with 30 additions and 0 deletions.
12 changes: 12 additions & 0 deletions hail/src/main/scala/is/hail/backend/local/LocalBackend.scala
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ import scala.reflect.ClassTag

import java.io.PrintWriter

import com.fasterxml.jackson.core.StreamReadConstraints
import com.google.common.util.concurrent.MoreExecutors
import org.apache.hadoop
import org.json4s._
Expand All @@ -46,6 +47,17 @@ object LocalBackend {
skipLoggingConfiguration: Boolean = false,
): LocalBackend = synchronized {
require(theLocalBackend == null)
// From https://github.com/hail-is/hail/issues/14580 :
// IR can get quite big, especially as it can contain an arbitrary
// amount of encoded literals from the user's python session. This
// was a (controversial) restriction imposed by Jackson and should be lifted.
//
// We remove this restriction _here_ (as opposed to anywhere else) because
// this is the first call into the JVM we control as part of initializing
// hail for the local backend
StreamReadConstraints.overrideDefaultStreamReadConstraints(
StreamReadConstraints.builder().maxStringLength(Integer.MAX_VALUE).build()
)

if (!skipLoggingConfiguration)
HailContext.configureLogging(logFile, quiet, append)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -465,6 +465,12 @@ object ServiceBackendAPI {

implicit val formats: Formats = DefaultFormats

// From https://github.com/hail-is/hail/issues/14580 :
// IR can get quite big, especially as it can contain an arbitrary
// amount of encoded literals from the user's python session. This
// was a (controversial) restriction imposed by Jackson and should be lifted.
//
// We remove this restriction.
StreamReadConstraints.overrideDefaultStreamReadConstraints(
StreamReadConstraints.builder().maxStringLength(Integer.MAX_VALUE).build()
);
Expand Down
12 changes: 12 additions & 0 deletions hail/src/main/scala/is/hail/backend/spark/SparkBackend.scala
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ import scala.util.control.NonFatal

import java.io.{Closeable, PrintWriter}

import com.fasterxml.jackson.core.StreamReadConstraints
import org.apache.hadoop
import org.apache.hadoop.conf.Configuration
import org.apache.spark._
Expand Down Expand Up @@ -257,6 +258,17 @@ object SparkBackend {
gcsRequesterPaysBuckets: String = null,
): SparkBackend = synchronized {
require(theSparkBackend == null)
// From https://github.com/hail-is/hail/issues/14580 :
// IR can get quite big, especially as it can contain an arbitrary
// amount of encoded literals from the user's python session. This
// was a (controversial) restriction imposed by Jackson and should be lifted.
//
// We remove this restriction _here_ (as opposed to anywhere else) because
// this is the first call into the JVM we control as part of initializing
// hail for the spark backend
StreamReadConstraints.overrideDefaultStreamReadConstraints(
StreamReadConstraints.builder().maxStringLength(Integer.MAX_VALUE).build()
)

if (!skipLoggingConfiguration)
HailContext.configureLogging(logFile, quiet, append)
Expand Down

0 comments on commit fd10c0b

Please sign in to comment.