[query] Lift backend state into {Service|Py4J}BackendApi

hail/python/hail/backend/local_backend.py

@@ -74,7 +74,6 @@ def __init__(
         hail_package = getattr(self._gateway.jvm, 'is').hail
 
         jbackend = hail_package.backend.local.LocalBackend.apply(
-            tmpdir,
             log,
             True,
             append,

hail/python/hail/backend/py4j_backend.py

@@ -170,8 +170,15 @@ def parse(node):
 
 class Py4JBackend(Backend):
     @abc.abstractmethod
-    def __init__(self, jvm: JVMView, jbackend: JavaObject, jhc: JavaObject):
-        super(Py4JBackend, self).__init__()
+    def __init__(
+        self,
+        jvm: JVMView,
+        jbackend: JavaObject,
+        jhc: JavaObject,
+        tmpdir: str,
+        remote_tmpdir: str,
+    ):
+        super().__init__()
         import base64
 
         def decode_bytearray(encoded):
@@ -184,12 +191,11 @@ def decode_bytearray(encoded):
         self._jvm = jvm
         self._hail_package = getattr(self._jvm, 'is').hail
         self._utils_package_object = scala_package_object(self._hail_package.utils)
-        self._jbackend = jbackend
         self._jhc = jhc
 
-        self._backend_server = self._hail_package.backend.BackendServer(self._jbackend)
-        self._backend_server_port: int = self._backend_server.port()
-        self._backend_server.start()
+        self._jbackend = self._hail_package.backend.api.P4jBackendApi(jbackend)
+        self._jhttp_server = self._jbackend.pyHttpServer()
+        self._backend_server_port: int = self._jbackend.HttpServer.port()
         self._requests_session = requests.Session()
 
         # This has to go after creating the SparkSession. Unclear why.
@@ -289,7 +295,7 @@ def _to_java_blockmatrix_ir(self, ir):
         return self._parse_blockmatrix_ir(self._render_ir(ir))
 
     def stop(self):
-        self._backend_server.close()
+        self._jhttp_server.close()
         self._jbackend.close()
         self._jhc.stop()
         self._jhc = None

hail/src/main/scala/is/hail/backend/Backend.scala

@@ -9,7 +9,6 @@ import is.hail.io.fs.FS
 import is.hail.types.RTable
 import is.hail.types.encoded.EType
 import is.hail.types.physical.PTuple
-import is.hail.utils.ExecutionTimer.Timings
 import is.hail.utils.fatal
 
 import scala.reflect.ClassTag
@@ -54,6 +53,7 @@ trait BackendContext {
 }
 
 abstract class Backend extends Closeable {
+
   // From https://github.com/hail-is/hail/issues/14580 :
   //   IR can get quite big, especially as it can contain an arbitrary
   //   amount of encoded literals from the user's python session. This
@@ -119,7 +119,7 @@ abstract class Backend extends Closeable {
   def tableToTableStage(ctx: ExecuteContext, inputIR: TableIR, analyses: LoweringAnalyses)
     : TableStage
 
-  def withExecuteContext[T](f: ExecuteContext => T)(implicit E: Enclosing): (T, Timings)
-
   def execute(ctx: ExecuteContext, ir: IR): Either[Unit, (PTuple, Long)]
+
+  def backendContext(ctx: ExecuteContext): BackendContext
 }

hail/src/main/scala/is/hail/backend/BackendRpc.scala

@@ -244,8 +244,4 @@ trait HttpLikeBackendRpc[A] extends BackendRpc {
           )
       }
   }
-
-  implicit protected def Ask: Routing
-  implicit protected def Write: Write[A]
-  implicit protected def Context: Context[A]
 }

hail/src/main/scala/is/hail/backend/ExecuteContext.scala

@@ -1,6 +1,6 @@
 package is.hail.backend
 
-import is.hail.{HailContext, HailFeatureFlags}
+import is.hail.HailFeatureFlags
 import is.hail.annotations.{Region, RegionPool}
 import is.hail.asm4s.HailClassLoader
 import is.hail.backend.local.LocalTaskContext
@@ -55,11 +55,6 @@ object NonOwningTempFileManager {
 }
 
 object ExecuteContext {
-  def scoped[T](f: ExecuteContext => T)(implicit E: Enclosing): T =
-    HailContext.sparkBackend.withExecuteContext(
-      selfContainedExecution = false
-    )(f)
-
   def scoped[T](
     tmpdir: String,
     localTmpdir: String,
@@ -69,7 +64,6 @@ object ExecuteContext {
     tempFileManager: TempFileManager,
     theHailClassLoader: HailClassLoader,
     flags: HailFeatureFlags,
-    backendContext: BackendContext,
     irMetadata: IrMetadata,
     references: mutable.Map[String, ReferenceGenome],
     blockMatrixCache: mutable.Map[String, BlockMatrix],
@@ -91,7 +85,6 @@ object ExecuteContext {
           tempFileManager,
           theHailClassLoader,
           flags,
-          backendContext,
           irMetadata,
           references,
           blockMatrixCache,
@@ -124,7 +117,6 @@ class ExecuteContext(
   _tempFileManager: TempFileManager,
   val theHailClassLoader: HailClassLoader,
   val flags: HailFeatureFlags,
-  val backendContext: BackendContext,
   var irMetadata: IrMetadata,
   val References: mutable.Map[String, ReferenceGenome],
   val BlockMatrixCache: mutable.Map[String, BlockMatrix],
@@ -197,7 +189,6 @@ class ExecuteContext(
     tempFileManager: TempFileManager = NonOwningTempFileManager(this.tempFileManager),
     theHailClassLoader: HailClassLoader = this.theHailClassLoader,
     flags: HailFeatureFlags = this.flags,
-    backendContext: BackendContext = this.backendContext,
     references: mutable.Map[String, ReferenceGenome] = this.References,
     irMetadata: IrMetadata = this.irMetadata,
     blockMatrixCache: mutable.Map[String, BlockMatrix] = this.BlockMatrixCache,
@@ -217,7 +208,6 @@ class ExecuteContext(
       tempFileManager,
       theHailClassLoader,
       flags,
-      backendContext,
       irMetadata,
       references,
       blockMatrixCache,