Skip to content

Commit c86ec9a

Browse files
armanbilgearmanbilge
authored
Merge pull request #236 from http4s/pr/i184
Support streaming requests
2 parents 11ccbdd + 2dee843 commit c86ec9a

File tree

8 files changed

+312
-88
lines changed

8 files changed

+312
-88
lines changed

‎build.sbt‎

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -120,7 +120,20 @@ lazy val dom = project
120120
"co.fs2" %%% "fs2-core" % fs2Version,
121121
"org.http4s" %%% "http4s-client" % http4sVersion,
122122
"org.scala-js" %%% "scalajs-dom" % scalaJSDomVersion
123-
)
123+
),
124+
mimaBinaryIssueFilters ++= {
125+
import com.typesafe.tools.mima.core._
126+
if (tlIsScala3.value)
127+
Seq(
128+
ProblemFilters.exclude[DirectMissingMethodProblem](
129+
"org.http4s.dom.package.closeReadableStream"),
130+
ProblemFilters.exclude[DirectMissingMethodProblem](
131+
"org.http4s.dom.package.fromReadableStream"),
132+
ProblemFilters.exclude[DirectMissingMethodProblem](
133+
"org.http4s.dom.package.toDomHeaders")
134+
)
135+
else Seq()
136+
}
124137
)
125138
.enablePlugins(ScalaJSPlugin)
126139

@@ -146,7 +159,8 @@ def configureTest(project: Project): Project =
146159
libraryDependencies ++= Seq(
147160
"org.http4s" %%% "http4s-client-testkit" % http4sVersion,
148161
"org.scalameta" %%% "munit" % munitVersion % Test,
149-
"org.typelevel" %%% "munit-cats-effect" % munitCEVersion % Test
162+
"org.typelevel" %%% "munit-cats-effect" % munitCEVersion % Test,
163+
"org.typelevel" %%% "scalacheck-effect-munit" % "2.0.0-M2" % Test
150164
),
151165
Compile / unmanagedSourceDirectories +=
152166
(LocalRootProject / baseDirectory).value / "tests" / "src" / "main" / "scala",

‎dom/src/main/scala/org/http4s/dom/FetchClient.scala‎

Lines changed: 73 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -26,9 +26,11 @@ import cats.syntax.all._
2626
import org.http4s.client.Client
2727
import org.http4s.headers.Referer
2828
import org.scalajs.dom.AbortController
29+
import org.scalajs.dom.BodyInit
2930
import org.scalajs.dom.Fetch
3031
import org.scalajs.dom.Headers
3132
import org.scalajs.dom.HttpMethod
33+
import org.scalajs.dom.RequestDuplex
3234
import org.scalajs.dom.RequestInit
3335
import org.scalajs.dom.{Response => FetchResponse}
3436

@@ -41,64 +43,84 @@ private[dom] object FetchClient {
4143
requestTimeout: Duration,
4244
options: FetchOptions
4345
)(implicit F: Async[F]): Client[F] = Client[F] { (req: Request[F]) =>
44-
Resource
45-
.eval(
46-
(if (req.isChunked) req.toStrict(None) else req.pure)
47-
.mproduct(_.body.chunkAll.filter(_.nonEmpty).compile.last)
48-
)
49-
.flatMap {
50-
case (req, body) =>
51-
Resource
52-
.makeCaseFull { (poll: Poll[F]) =>
53-
F.delay(new AbortController()).flatMap { abortController =>
54-
val requestOptions = req.attributes.lookup(FetchOptions.Key)
55-
val mergedOptions = requestOptions.fold(options)(options.merge)
46+
Resource.eval(F.fromPromise(F.delay(supportsRequestStreams))).flatMap {
47+
supportsRequestStreams =>
48+
val reqBody =
49+
if (req.body eq EmptyBody)
50+
Resource.pure[F, (Request[F], Option[BodyInit])]((req, None))
51+
else if (supportsRequestStreams)
52+
toReadableStream(req.body).map(Some[BodyInit](_)).tupleLeft(req)
53+
else
54+
Resource.eval {
55+
(if (req.isChunked) req.toStrict(None) else req.pure).mproduct { req =>
56+
req
57+
.body
58+
.chunkAll
59+
.filter(_.nonEmpty)
60+
.map(c => c.toUint8Array: BodyInit)
61+
.compile
62+
.last
63+
}
64+
}
5665

57-
val init = new RequestInit {}
66+
reqBody.flatMap {
67+
case (req, body) =>
68+
Resource
69+
.makeCaseFull { (poll: Poll[F]) =>
70+
F.delay(new AbortController()).flatMap { abortController =>
71+
val requestOptions = req.attributes.lookup(FetchOptions.Key)
72+
val mergedOptions = requestOptions.fold(options)(options.merge)
5873

59-
init.method = req.method.name.asInstanceOf[HttpMethod]
60-
init.headers = new Headers(toDomHeaders(req.headers))
61-
body.foreach { body => init.body = body.toJSArrayBuffer }
62-
init.signal = abortController.signal
63-
mergedOptions.cache.foreach(init.cache = _)
64-
mergedOptions.credentials.foreach(init.credentials = _)
65-
mergedOptions.integrity.foreach(init.integrity = _)
66-
mergedOptions.keepAlive.foreach(init.keepalive = _)
67-
mergedOptions.mode.foreach(init.mode = _)
68-
mergedOptions.redirect.foreach(init.redirect = _)
69-
// Referer headers are forbidden in Fetch, but we make a best effort to preserve behavior across clients.
70-
// See https://developer.mozilla.org/en-US/docs/Glossary/Forbidden_header_name
71-
// If there's a Referer header, it will have more priority than the client's `referrer` (if present)
72-
// but less priority than the request's `referrer` (if present).
73-
requestOptions
74-
.flatMap(_.referrer)
75-
.orElse(req.headers.get[Referer].map(_.uri))
76-
.orElse(options.referrer)
77-
.foreach(referrer => init.referrer = referrer.renderString)
78-
mergedOptions.referrerPolicy.foreach(init.referrerPolicy = _)
74+
val init = new RequestInit {}
7975

80-
val fetch =
81-
poll(F.fromPromise(F.delay(Fetch.fetch(req.uri.renderString, init))))
82-
.onCancel(F.delay(abortController.abort()))
76+
init.method = req.method.name.asInstanceOf[HttpMethod]
77+
init.headers = new Headers(toDomHeaders(req.headers, request = true))
78+
body.foreach { body =>
79+
init.body = body
80+
if (supportsRequestStreams)
81+
init.duplex = RequestDuplex.half
82+
}
83+
init.signal = abortController.signal
84+
mergedOptions.cache.foreach(init.cache = _)
85+
mergedOptions.credentials.foreach(init.credentials = _)
86+
mergedOptions.integrity.foreach(init.integrity = _)
87+
mergedOptions.keepAlive.foreach(init.keepalive = _)
88+
mergedOptions.mode.foreach(init.mode = _)
89+
mergedOptions.redirect.foreach(init.redirect = _)
90+
// Referer headers are forbidden in Fetch, but we make a best effort to preserve behavior across clients.
91+
// See https://developer.mozilla.org/en-US/docs/Glossary/Forbidden_header_name
92+
// If there's a Referer header, it will have more priority than the client's `referrer` (if present)
93+
// but less priority than the request's `referrer` (if present).
94+
requestOptions
95+
.flatMap(_.referrer)
96+
.orElse(req.headers.get[Referer].map(_.uri))
97+
.orElse(options.referrer)
98+
.foreach(referrer => init.referrer = referrer.renderString)
99+
mergedOptions.referrerPolicy.foreach(init.referrerPolicy = _)
83100

84-
requestTimeout match {
85-
case d: FiniteDuration =>
86-
fetch.timeoutTo(
87-
d,
88-
F.raiseError[FetchResponse](new TimeoutException(
89-
s"Request to ${req.uri.renderString} timed out after ${d.toMillis} ms"))
90-
)
91-
case _ =>
92-
fetch
101+
val fetch =
102+
poll(F.fromPromise(F.delay(Fetch.fetch(req.uri.renderString, init))))
103+
.onCancel(F.delay(abortController.abort()))
104+
105+
requestTimeout match {
106+
case d: FiniteDuration =>
107+
fetch.timeoutTo(
108+
d,
109+
F.raiseError[FetchResponse](new TimeoutException(
110+
s"Request to ${req.uri.renderString} timed out after ${d.toMillis} ms"))
111+
)
112+
case _ =>
113+
fetch
114+
}
93115
}
116+
} {
117+
case (r, exitCase) =>
118+
OptionT.fromOption(Option(r.body)).foreachF(cancelReadableStream(_, exitCase))
94119
}
95-
} {
96-
case (r, exitCase) =>
97-
OptionT.fromOption(Option(r.body)).foreachF(closeReadableStream(_, exitCase))
98-
}
99-
.evalMap(fromDomResponse[F])
120+
.evalMap(fromDomResponse[F])
100121

101-
}
122+
}
123+
}
102124
}
103125

104126
}

‎dom/src/main/scala/org/http4s/dom/ServiceWorker.scala‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -92,7 +92,7 @@ object ServiceWorker {
9292
new ResponseInit {
9393
this.status = response.status.code
9494
this.statusText = response.status.reason
95-
this.headers = toDomHeaders(response.headers)
95+
this.headers = toDomHeaders(response.headers, request = false)
9696
}
9797
)
9898
}

‎dom/src/main/scala/org/http4s/dom/package.scala‎

Lines changed: 96 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -18,16 +18,27 @@ package org.http4s
1818

1919
import cats.effect.kernel.Async
2020
import cats.effect.kernel.Resource
21+
import cats.effect.std.Dispatcher
22+
import cats.effect.std.Queue
23+
import cats.effect.syntax.all._
2124
import cats.syntax.all._
25+
import fs2.Chunk
2226
import fs2.Stream
27+
import org.http4s.headers.`Transfer-Encoding`
2328
import org.scalajs.dom.Blob
29+
import org.scalajs.dom.Fetch
2430
import org.scalajs.dom.File
31+
import org.scalajs.dom.HttpMethod
2532
import org.scalajs.dom.ReadableStream
33+
import org.scalajs.dom.ReadableStreamType
34+
import org.scalajs.dom.ReadableStreamUnderlyingSource
35+
import org.scalajs.dom.RequestDuplex
36+
import org.scalajs.dom.RequestInit
2637
import org.scalajs.dom.{Headers => DomHeaders}
38+
import org.scalajs.dom.{Request => DomRequest}
2739
import org.scalajs.dom.{Response => DomResponse}
2840

2941
import scala.scalajs.js
30-
import scala.scalajs.js.JSConverters._
3142
import scala.scalajs.js.typedarray.Uint8Array
3243

3344
package object dom {
@@ -37,73 +48,124 @@ package object dom {
3748

3849
implicit def blobEncoder[F[_]](implicit F: Async[F]): EntityEncoder[F, Blob] =
3950
EntityEncoder.entityBodyEncoder.contramap { blob =>
40-
Stream
41-
.bracketCase {
42-
F.delay(blob.stream())
43-
} { case (rs, exitCase) => closeReadableStream(rs, exitCase) }
44-
.flatMap(fromReadableStream[F])
51+
readReadableStream[F](F.delay(blob.stream()))
4552
}
4653

4754
implicit def readableStreamEncoder[F[_]: Async]
4855
: EntityEncoder[F, ReadableStream[Uint8Array]] =
49-
EntityEncoder.entityBodyEncoder.contramap { rs => fromReadableStream(rs) }
56+
EntityEncoder.entityBodyEncoder.contramap { rs => readReadableStream(rs.pure) }
5057

5158
private[dom] def fromDomResponse[F[_]](response: DomResponse)(
5259
implicit F: Async[F]): F[Response[F]] =
5360
F.fromEither(Status.fromInt(response.status)).map { status =>
5461
Response[F](
5562
status = status,
5663
headers = fromDomHeaders(response.headers),
57-
body = Stream.fromOption(Option(response.body)).flatMap(fromReadableStream[F])
64+
body = Stream.fromOption(Option(response.body)).flatMap { rs =>
65+
readReadableStream[F](rs.pure)
66+
}
5867
)
5968
}
6069

61-
private[dom] def toDomHeaders(headers: Headers): DomHeaders =
62-
new DomHeaders(
63-
headers
64-
.headers
65-
.view
66-
.map {
67-
case Header.Raw(name, value) =>
68-
name.toString -> value
69-
}
70-
.toMap
71-
.toJSDictionary)
70+
private[dom] def toDomHeaders(headers: Headers, request: Boolean): DomHeaders = {
71+
val domHeaders = new DomHeaders()
72+
headers.foreach {
73+
case Header.Raw(name, value) =>
74+
val skip = request && name == `Transfer-Encoding`.name
75+
if (!skip) domHeaders.append(name.toString, value)
76+
}
77+
domHeaders
78+
}
7279

7380
private[dom] def fromDomHeaders(headers: DomHeaders): Headers =
7481
Headers(
7582
headers.map { header => header(0) -> header(1) }.toList
7683
)
7784

78-
private[dom] def fromReadableStream[F[_]](rs: ReadableStream[Uint8Array])(
79-
implicit F: Async[F]): Stream[F, Byte] =
80-
Stream.bracket(F.delay(rs.getReader()))(r => F.delay(r.releaseLock())).flatMap { reader =>
81-
Stream.unfoldChunkEval(reader) { reader =>
82-
F.fromPromise(F.delay(reader.read())).map { chunk =>
83-
if (chunk.done)
84-
None
85-
else
86-
Some((fs2.Chunk.uint8Array(chunk.value), reader))
85+
private[dom] def readReadableStream[F[_]](
86+
readableStream: F[ReadableStream[Uint8Array]]
87+
)(implicit F: Async[F]): Stream[F, Byte] = {
88+
def read(readableStream: ReadableStream[Uint8Array]) =
89+
Stream
90+
.bracket(F.delay(readableStream.getReader()))(r => F.delay(r.releaseLock()))
91+
.flatMap { reader =>
92+
Stream.unfoldChunkEval(reader) { reader =>
93+
F.fromPromise(F.delay(reader.read())).map { chunk =>
94+
if (chunk.done)
95+
None
96+
else
97+
Some((fs2.Chunk.uint8Array(chunk.value), reader))
98+
}
99+
}
87100
}
88-
}
89-
}
90101

91-
private[dom] def closeReadableStream[F[_], A](
102+
Stream.bracketCase(readableStream)(cancelReadableStream(_, _)).flatMap(read(_))
103+
}
104+
105+
private[dom] def cancelReadableStream[F[_], A](
92106
rs: ReadableStream[A],
93-
exitCase: Resource.ExitCase)(implicit F: Async[F]): F[Unit] = F.fromPromise {
107+
exitCase: Resource.ExitCase
108+
)(implicit F: Async[F]): F[Unit] = F.fromPromise {
94109
F.delay {
95110
// Best guess: Firefox internally locks a ReadableStream after it is "drained"
96111
// This checks if the stream is locked before canceling it to avoid an error
97112
if (!rs.locked) exitCase match {
98113
case Resource.ExitCase.Succeeded =>
99114
rs.cancel(js.undefined)
100115
case Resource.ExitCase.Errored(ex) =>
101-
rs.cancel(ex.getLocalizedMessage())
116+
rs.cancel(ex.toString())
102117
case Resource.ExitCase.Canceled =>
103118
rs.cancel(js.undefined)
104119
}
105120
else js.Promise.resolve[Unit](())
106121
}
107-
}.void
122+
}
123+
124+
private[dom] def toReadableStream[F[_]](in: Stream[F, Byte])(
125+
implicit F: Async[F]): Resource[F, ReadableStream[Uint8Array]] =
126+
Dispatcher.sequential.flatMap { dispatcher =>
127+
Resource.eval(Queue.synchronous[F, Option[Chunk[Byte]]]).flatMap { chunks =>
128+
in.enqueueNoneTerminatedChunks(chunks).compile.drain.background.evalMap { _ =>
129+
F.delay {
130+
val source = new ReadableStreamUnderlyingSource[Uint8Array] {
131+
`type` = ReadableStreamType.bytes
132+
pull = js.defined { controller =>
133+
dispatcher.unsafeToPromise {
134+
chunks.take.flatMap {
135+
case Some(chunk) =>
136+
F.delay(controller.enqueue(chunk.toUint8Array))
137+
case None => F.delay(controller.close())
138+
}
139+
}
140+
}
141+
}
142+
ReadableStream[Uint8Array](source)
143+
}
144+
}
145+
}
146+
}
147+
148+
private[dom] lazy val supportsRequestStreams = {
149+
val request = new DomRequest(
150+
"data:a/a;charset=utf-8,",
151+
new RequestInit {
152+
body = ReadableStream()
153+
method = HttpMethod.POST
154+
duplex = RequestDuplex.half
155+
}
156+
)
157+
158+
val supportsStreamsInRequestObjects = !request.headers.has("Content-Type")
159+
160+
if (!supportsStreamsInRequestObjects)
161+
js.Promise.resolve[Boolean](false)
162+
else
163+
Fetch
164+
.fetch(request)
165+
.`then`[Boolean](
166+
_ => true,
167+
(_ => false): js.Function1[Any, Boolean]
168+
)
169+
}
108170

109171
}

0 commit comments

Comments
 (0)