httpserver.r2py

"""
<Program Name>
  httpserver.r2py

<Started>
  July 29, 2009

<Author>
  Conrad Meyer

<Purpose>
  This is a library that abstracts away the details of the HTTP protocol,
  instead calling a user-supplied function on each request. The return
  value of the user-supplied function determines the response that is sent
  to the HTTP client.

"""



dy_import_module_symbols("librepy.r2py")
dy_import_module_symbols("urllib.r2py")
dy_import_module_symbols("urlparse.r2py")
dy_import_module_symbols("uniqueid.r2py")
dy_import_module_symbols("sockettimeout.r2py")
dy_import_module_symbols("httpretrieve.r2py")




class _httpserver_ClientClosedSockEarly(Exception):
  # Raised internally when the client unexpectedly closes the socket. The
  # correct behavior in this instance is to clean up that handler and
  # continue.
  pass




class _httpserver_BadRequest(Exception):
  # Raised internally when the client's request is malformed.
  pass




class _httpserver_ServerError(Exception):
  # Raised internally when the callback function unexpectedly raises an
  # exception.
  pass




class _httpserver_BadTransferCoding(Exception):
  # Raised internally when the request's encoding is something we can't
  # handle (most everything at the time of writing).
  pass




# This global dictionary is used to keep track of open HTTP callbacks.
# The 'lock' entry is used to serialize changes to the other entries.
# The 'handles' dictionary maps numeric ids we hand out in
# httpserver_registercallback() and take in httpserver_stopcallback()
# to ids returned and used by waitforconn(). The 'cbfuncs' entry
# maps httpserver numeric ids to the callback function associated
# with them.
_httpserver_context = {
    'handles': {},
    'cbfuncs': {},
    'lock': createlock()}



def httpserver_registercallback(addresstuple, cbfunc):
  """
  <Purpose>
    Registers a callback function on the (host, port).

  <Arguments>
    addresstuple:
      An address 2-tuple to bind to: ('host', port).

    cbfunc:
      The callback function to process requests. It takes one argument,
      which is a dictionary describing the HTTP request. It looks like
      this (just an example):
        {
          'verb': 'HEAD',
          'path': '/',
          'querystr': 'foo=bar&baz',
          'querydict': { 'foo': 'bar', 'baz': None }
          'version': '0.9',
          'datastream': object with a file-like read() method,
          'headers': { 'Content-Type': 'application/x-xmlrpc-data'},
          'httpdid': 17,
          'remoteipstr': '10.0.0.4',
          'remoteportnum': 54001
        }
      ('datastream' is a stream of any HTTP message body data sent by the
      client.)

      It is expected that this callback function returns a dictionary of:
        {
          'version': '0.9' or '1.0' or '1.1',
          'statuscode': any integer from 100 to 599,
          'statusmsg' (optional): an arbitrary string without newlines,
          'headers': { 'X-Header-Foo': 'Bar' },
          'message': arbitrary string
        }

  <Exceptions>
    TypeError, ValueError, KeyError, IndexError if arguments to this
    function are malformed.

    Raises any exception waitforconn() will raise if the (hostname, port)
    tuple is restricted, already taken, etc.

  <Side Effects>
    Starts a listener on the given host and port.

  <Returns>
    A handle for the listener (an httpdid). This can be used to stop the
    server.

  """

  _httpserver_context['lock'].acquire(True)

  try:
    newhttpdid = uniqueid_getid()

    # Keep track of this server's id in a closure:
    def _httpserver_cbclosure(remoteip, remoteport, sock, ch, listench):
      # Do the actual processing on the request.
      _httpserver_socketcb(remoteip, remoteport, sock, ch, listench, \
          newhttpdid)

      # Close the socket afterwards.
      try:
        sock.close()
      except Exception, e:
        if "socket" not in str(e).lower():
          raise
        pass    # Best effort.
	

    _httpserver_context['handles'][newhttpdid] = \
        waitforconn(addresstuple[0], addresstuple[1], _httpserver_cbclosure)
    _httpserver_context['cbfuncs'][newhttpdid] = cbfunc

    return newhttpdid
  finally:
    _httpserver_context['lock'].release()




def _httpserver_socketcb(remoteip, remoteport, sock, ch, listench, httpdid):
  # This function gets invoked each time a client connects to our socket.

  # It proceeds in a loop -- reading in requests, handing them off to the
  # callback function, and sending the result to the client. If errors are
  # encountered, it sends an error message (we choose HTTP/1.0 for
  # compatibility and because we don't always know what version the client
  # wants) and closes the connection. Additionally, if the response
  # generated by the callback requests protocol version 0.9 or 1.0, or is
  # 1.1 but includes the Connection: close header, the connection is closed
  # and the loop broken.

  _httpserver_context['lock'].acquire(True)
  try:
    cbfunc = _httpserver_context['cbfuncs'][httpdid]
  finally:
    _httpserver_context['lock'].release()

  extradata = ""

  # HTTP/1.0 and HTTP/1.1 Connection: close requests break out of this
  # loop immediately; HTTP/1.1 clients can keep sending requests and
  # receiving responses in this loop indefinitely.
  while True:
    try:
      # Reads request headers, parses them, lets callback handle headers
      # and possible request body, sends the response that the callback
      # function tells it to send. On error, may raise one of many
      # exceptions, which we deal with here:
      closeconn, extradata = \
          _httpserver_process_single_request(sock, cbfunc, extradata, \
          httpdid, remoteip, remoteport)

      if closeconn:
        break

    except _httpserver_BadRequest, br:
      # There was some sort of flaw in the client's request.
      response = "HTTP/1.0 400 Bad Request\r\n" + \
          "Content-Type: text/plain\r\n\r\n" + str(br) + "\r\n"
      _httpserver_sendAll(sock, response, besteffort=True)
      break
    
    except _httpserver_ServerError, se:
      # The callback function raised an exception or returned some object
      # we didn't expect.
      response = "HTTP/1.0 500 Internal Server Error\r\n" + \
          "Content-Type: text/plain\r\n\r\n" + str(se) + "\r\n"
      _httpserver_sendAll(sock, response, besteffort=True)
      break
    
    except _httpserver_BadTransferCoding, bte:
      # The HTTP/1.1 client sent us something with a Transport-Encoding we
      # can't handle.
      response = "HTTP/1.1 501 Not Implemented\r\n" + \
          ("Content-Length: %d\r\n" % (len(str(bte)) + 2)) + \
          "Connection: close\r\n" + \
          "Content-Type: text/plain\r\n\r\n" + str(bte) + "\r\n"
      _httpserver_sendAll(sock, response, besteffort=True)
      break

    except _httpserver_ClientClosedSockEarly:
      # Not much else we can do.
      break
    
    except Exception, e:
      if "Socket closed" in str(e):
        break

      # We shouldn't encounter these, other than 'Socket closed' ones. They
      # represent a bug in our code somewhere. However, not raising the
      # exception makes HTTP server software incredibly unintuitive to
      # debug.
      raise




def _httpserver_readHTTPheader(sock, data):
  # Reads data from the socket in 4k chunks, replacing \r\n newlines with
  # \n newlines. When it encounters a (decoded) \n\n sequence, it returns
  # (data_before, data_after).

  headers = []
  command = True
  while True:
    line, data = _httpserver_getline(sock, data)
    if len(line) == 0:
      raise _httpserver_ClientClosedSockEarly()

    # Be a well-behaved server, and handle normal newlines and telnet-style
    # newlines in the same fashion.
    line = line.rstrip("\r")
    if command:
      splitln = line.split(" ")
      if len(splitln) != 3:
        raise _httpserver_BadRequest("HTTP/0.9 or malformed request.")
      if not splitln[2].lower().startswith("http/1."):
        raise _httpserver_BadRequest("Malformed request.")
      command = False

    if len(line) == 0:
      break

    headers.append(line)

  return (headers, data)




def _httpserver_parseHTTPheader(headerdatalist):
  lineslist = headerdatalist
  commandstr = lineslist[0]
  otherheaderslist = lineslist[1:]

  infodict = {}

  verbstr, rawpathstr, versionstr = commandstr.split(" ", 2)
  
  infodict['verb'] = verbstr

  if len(versionstr) != len("HTTP/1.1"):
    raise _httpserver_BadRequest("Bad HTTP command")
  versionstr = versionstr.upper()
  if versionstr == "HTTP/1.0":
    infodict['version'] = '1.0'
  elif versionstr == "HTTP/1.1":
    infodict['version'] = '1.1'
  else:
    raise _httpserver_BadRequest("Unrecognized HTTP version")

  if rawpathstr.find("?") != -1:
    infodict['path'], infodict['querystr'] = rawpathstr.split("?", 1)
  else:
    infodict['path'] = rawpathstr
    infodict['querystr'] = None

  try:
    infodict['headers'] = _httpretrieve_parse_responseheaders(otherheaderslist)
  except HttpBrokenServerError:
    raise _httpserver_BadRequest("Request headers are misformed.")

  try:
    infodict['querydict'] = urllib_unquote_parameters(infodict['querystr'])
  except (ValueError, AttributeError, TypeError):
    infodict['querydict'] = None

  return infodict
    




def _httpserver_sendAll(sock, datastr, besteffort=False):
  # Sends all the data in datastr to sock. If besteffort is True,
  # we don't care if it fails or not.
  try:
    while len(datastr) > 0:
      datastr = datastr[sock.send(datastr):]
  except Exception, e:
    if "socket" not in str(e).lower():
      raise

    # If the caller didn't want this function to raise an exception for
    # any reason, we don't, and instead return silently. If they are ok
    # with exceptions, we re-raise.
    if not besteffort:
      raise




def _httpserver_getline(sock, datastr):
  # Reads a line out of datastr (if possible), or failing that, gets more from
  # the socket. Returns (line, extra).

  try:
    newdatastr = ""
    while True:
      endloc = datastr.find("\n", -len(newdatastr))
      if endloc != -1:
        return (datastr[:endloc], datastr[endloc+1:])
      newdatastr = sock.recv(4096)
      datastr += newdatastr
  except Exception, e:
    if "Socket closed" in str(e) and len(datastr) != 0:
      return (datastr, "")
    raise




def _httpserver_getblock(blocksize, sock, datastr):
  # Reads a block of size blocksize out of datastr (if possible), or failing
  # that, gets more from the socket. Returns (block, extra).

  try:
    while len(datastr) < blocksize:
      datastr += sock.recv(4096)

    return (datastr[:blocksize], datastr[blocksize:])
  except Exception, e:
    if "Socket closed" in str(e) and len(datastr) != 0:
      return (datastr, "")
    raise




def httpserver_stopcallback(callbackid):
  """
  <Purpose>
    Removes an existing callback function.

  <Arguments>
    callbackid:
      The id returned by httpserver_registercallback().

  <Exceptions>
    IndexError, KeyError if the id is invalid or has already been deleted.

  <Side Effects>
    Removes this listener from the registry, deletes the listening socket.

  <Returns>
    Nothing.

  """
  _httpserver_context['lock'].acquire(True)
  try:
    stopcomm(_httpserver_context['handles'][callbackid])
    del _httpserver_context['handles'][callbackid]
    del _httpserver_context['cbfuncs'][callbackid]
  finally:
    _httpserver_context['lock'].release()




class _httpserver_bodystream:
  """
  _httpserver_bodystream is a helper class passed as the 'datastream' entry
  in the dictionary sent to user callback functions. It has a read() method
  that behaves very similarly to file.read(). Other than that, this object
  is nothing like Python's file.

  """

  # Reads the rest of the HTTP request from the socket, erroring
  # appropriately. Understands transfer-coding. Returns chunks
  # at a time.

  def __init__(self, sock, data, verb, headers):
    # The socket object for communicating with the client:
    self._sock = sock
    # And any data we have already read off the socket, but has not been
    # consumed (we read data in 4 kilobyte chunks and keep the extra
    # around for later use).
    self._rawdata = data

    # The HTTP request verb (GET, POST, etc); a string.
    self._verb = verb

    # A dictionary of the client's request headers, as parsed by
    # _httpretrieve_parse_responseheaders() (this is a function which
    # should be moved elsewhere; for example, an http_common.r2py
    # library would work).
    self._headers = headers

    # The number of bytes left in the current chunk, if the client's
    # request body is transfer-encoded (integer, or None), or the
    # number of bytes in the entire request body if it is not transfer-
    # encoded.
    self._leftinchunk = None

    # Whether or not the client's request body was transfer-encoded.
    self._chunked = False

    # A flag set when we know we have finished reading the current
    # request body, so we can return the empty string.
    self._done = False

    # A lock used to serialize reads from this file-like object.
    self._lock = createlock()

    # A flag set when we are finished reading HTTP "trailers". Only applies
    # to client requests sent with chunked transfer-encoding.
    self._trailersread = False

    # For chunked transfers only: Keep a queue of unconsumed but decoded
    # data (string).
    self._data = ""


    # Deal with methods that cannot send a message body:
    if verb in ("GET", "HEAD", "TRACE", "DELETE"):
      if "Content-Length" in headers or \
          "Transfer-Encoding" in headers:
        raise _httpserver_BadRequest("Method '" + verb + \
            "' cannot send an entity-body and therefore a " + \
            "Content-Length or Transfer-Encoding header is invalid.")
      else:
        self._done = True
        return

    # Deal with methods that send a message body.
    if "Content-Length" not in headers and "Transfer-Encoding" not in headers:
      raise _httpserver_BadRequest("Method '" + str(verb) + \
          "' can send an entity-body and requires a Content-Length " + \
          "or Transfer-Encoding header.")

    if "Transfer-Encoding" in headers:
      # Decode Transfer-coded messages
      if len(headers["Transfer-Encoding"]) != 1:
        raise _httpserver_BadRequest("Multiple Transfer-Encoding headers " + \
            "is unacceptable.")

      # "chunked" must be in the codings list, and it must be last.
      codings = headers["Transfer-Encoding"][0].split(",")
      codings.reverse()

      realcodings = []

      # Strip 'identity' codings.
      for coding in codings:
        coding = coding.strip().lower()
        token = coding.split(None, 1)
        if token != "identity":
          realcodings.append(coding)
      
      if len(realcodings) > 1 or realcodings[0] != "chunked":
        raise _httpserver_BadTransferCoding("Cannot handle any transfer-" + \
            "codings other than chunked.")

      self._chunked = True

    else:
      # If we get here, that means we have a Content-Length and no Transfer-
      # Encoding, so we can read the message body directly.
      msglen = headers["Content-Length"]

      if len(msglen) != 1:
        raise _httpserver_BadRequest("Multiple Content-Length headers is " + \
            "unacceptable.")
      
      self._leftinchunk = int(msglen[0])



  def read(self, size=None):
    """
    <Purpose>
      Read a sequence of bytes from an HTTP request body.

    <Arguments>
      size (optional):
        An upper limit on the number of bytes to return.

    <Exceptions>
      Any raised by socket.read().

    <Side Effects>
      Possibly reads more from the socket that the HTTP request is passed
      on.

    <Returns>
      A string of bytes comprising part of the HTTP message body. Does not
      include any chunked encoding trailers.

    """

    self._lock.acquire(True)  # Serialize file-like object reads.

    # Keep the same API as file(), but use a more understandable variable
    # name through this method.
    requestsize = size

    # The following 'Try' block is used to always unlock self._lock, no
    # matter how we return up the stack:
    try:
      # If we already know we are finished, simply return the empty string.
      if self._done:
        return ""

      if requestsize == 0:
        return ""

      # If the client's request was not chunked, but instead they specified
      # the Content-length header:
      if not self._chunked:
        # toyieldstr is the string we will return to the user this time,
        # as if this were a python generator (which is a nice way to think
        # about it). We determine the amount to return, toyieldlen:
        toyieldlen = self._leftinchunk
        if requestsize is not None:
          toyieldlen = min(requestsize, toyieldlen)

        # And strive to return as much of it as possible:
        toyieldstr, self._rawdata = _httpserver_getblock(toyieldlen, \
            self._sock, self._rawdata)
        self._leftinchunk -= len(toyieldstr)

        # If there is nothing left in the request, we're done; keep a note
        # for future read() calls.
        if self._leftinchunk == 0:
          self._done = True

        return toyieldstr

      # If the client's request was chunked:
      else:
        # Read until there isn't anymore, OR until we have enough to satisfy
        # the user's request.
        while requestsize is None or len(self._data) < requestsize:
          # If we have more raw bytes to read from the socket before
          # reaching the end of this chunk:
          if self._leftinchunk > 0:
            # Determine how many bytes to (attempt to) read from the client:
            nextblocksize = self._leftinchunk
            if requestsize is not None:
              nextblocksize = min(self._leftinchunk, requestsize)

            # Try and request the rest of the chunk, or as much as is
            # needed to fulfill the caller's request.
            chunkstr, self._rawdata = _httpserver_getblock(nextblocksize, self._sock, \
                self._rawdata)
            self._leftinchunk -= len(chunkstr)
            self._data += chunkstr

            # We stop trying if we can't get as much data as we wanted to,
            # because this means that the client has closed the socket.
            if len(chunkstr) < nextblocksize:
              break

          # We are finished with this chunk; now we must determine if there
          # is another chunk, and what its length is.
          else:
            # HTTP chunks have "\r\n" appended to the end of them; if we
            # just finished reading a chunk, read off the trailing "\r\n"
            # and discard it.
            if self._leftinchunk is not None:
              _, self._rawdata = _httpserver_getblock(2, self._sock, \
                  self._rawdata)
            
            # Read the next chunk's size information:
            line, self._rawdata = _httpserver_getline(self._sock, \
                self._rawdata)

            # Remove optional chunk extensions (";" -> newline) that we don't
            # understand (this is advised by the HTTP 1.1 RFC), and read the
            # hex chunk size:
            self._leftinchunk = int(line.split(";", 1)[0].strip(), 16)

            # A chunk size of '0' indicates the end of a chunked message:
            if self._leftinchunk == 0:
              self._done = True
              break

        # Determine how much data we should return, and how much we should
        # keep around for the next read.
        retlen = len(self._data)
        if requestsize is not None:
          retlen = min(requestsize, retlen)

        retval, self._data = self._data[:retlen], self._data[retlen:]
        return retval

    finally:
      self._lock.release()



  def get_trailers(self):
    """
    <Purpose>
      Read any 'trailers' sent by the client. The caller does not need to
      ensure that the entire message body has been read first, though they
      should be aware that calling this method consumes any remaining
      message body and immediately forgets it.

      Note: not multithread safe, not multi-call safe (calls after the
      first call will simply return the empty dictionary).

    <Arguments>
      None.

    <Exceptions>
      Any raised by socket.read().

    <Side Effects>
      Possibly reads more from the socket that the HTTP request is passed
      on.

    <Returns>
      A dictionary of headers in the same style as
      _httpretrieve_parse_responseheaders().

    """
    # Reads the rest of the message, and then reads and returns any trailer
    # headers (only sent in chunked messages).

    # Discard extra message without filling RAM. No-op if the user function
    # has already read the entire stream.
    while True:
      unuseddatastr = self.read(4096)
      if len(unuseddatastr) == 0:
        break

    if not self._chunked:
      return {}

    if self._trailersread:
      return {}

    trailers = {}

    # Read 'trailer' headers.
    while True:
      line, self._rawdata = _httpserver_getline(self._sock, self._rawdata)

      # Empty line? Then the trailers are done.
      if len(line.strip("\r")) == 0:
        break

      # Check for a multi-line header by peeking ahead:
      while True:
        nextchar, self._rawdata = _httpserver_getblock(1, self._sock, \
            self._rawdata)
        self._rawdata = nextchar + self._rawdata
        if nextchar in (" ", "\t"):
          line2, self._rawdata = _httpserver_getline(self._sock, self._rawdata)
          line += " " + line2.lstrip()
        else:
          break

      # Insert header into existing headers
      hdrname, hdrval = line.split(":", 1)
      if hdrname not in trailers:
        trailers[hdrname] = []
      trailers[hdrname].append(hdrval.strip())

    self._trailersread = True
    return trailers




  def _get_extra(self):
    # Private function of httpserver, *should not be used by callback
    # functions*! NOT parallel-safe, NOT meant to be called more than
    # once.

    # Read the socket up to (at least) the end of the current message;
    # if we read beyond the end of our message, return any extra.
    
    # Discard extra message without filling RAM
    while True:
      if len(self.read(4096)) == 0:
        break

    # Discard trailers, if any are left:
    if self._chunked:
      self.get_trailers()

    rawdata = self._rawdata
    self._rawdata = ""

    return rawdata




class _httpserver_StringIO:
  # Implements a read-only file-like object encapsulating a string.
  def __init__(self, string):
    self._data = string
    self._closed = False



  def read(self, limit=None):
    if limit is None:
      limit = 4096

    if self._closed:
      raise ValueError("Trying to read from a closed StringIO object.")

    res, self._data = self._data[:limit], self._data[limit:]
    return res



  def close(self):
    if self._closed:
      raise ValueError("Trying to close a closed StringIO object.")
    self._closed = True




def _httpserver_sendfile(sock, filelikeobj):
  # Attempts to forward all of the data from filelikeobj to sock.
  while True:
    chunk = filelikeobj.read(4096)
    if len(chunk) == 0:
      break
    _httpserver_sendAll(sock, chunk, besteffort=True)




def _httpserver_sendfile_chunked(sock, filelikeobj):
  # Attempts to forward all of the data from filelikeobj to sock, using chunked
  # encoding.
  totallen = 0
  while True:
    chunk = filelikeobj.read(4096)
    if len(chunk) == 0:
      break
    # encode as HTTP/1.1 chunks:
    totallen += len(chunk)
    chunk = "%X\r\n%s\r\n" % (len(chunk), chunk)
    _httpserver_sendAll(sock, chunk)

  lastchunk = "0\r\n"
  lastchunk += ("Content-Length: %d\r\n" % totallen)
  lastchunk += "\r\n"
  _httpserver_sendAll(sock, lastchunk)




def _httpserver_process_single_request(sock, cbfunc, extradata, httpdid, \
    remoteip, remoteport):
  # This function processes a single request in from sock, and either
  # puts a response to the socket and returns, or raises an exception.

  # Read HTTP request off the socket
  headerdata, extradata = _httpserver_readHTTPheader(sock, extradata)

  # Interpret the header.
  reqinfo = _httpserver_parseHTTPheader(headerdata)

  # Wrap the (possibly) remaining data into a file-like object.
  messagebodystream = _httpserver_bodystream(sock, \
      extradata, reqinfo['verb'], reqinfo['headers'])
  reqinfo['datastream'] = messagebodystream
  reqinfo['httpdid'] = httpdid
  reqinfo['remoteipstr'] = remoteip
  reqinfo['remoteportnum'] = remoteport

  # By default, we don't want to close the connection. (Though, nearly
  # every case will change this to True -- only HTTP/1.1 sockets
  # that don't set Connection: close will keep this False.)
  closeconn = False

  # Send request information to callback.
  try:
    result = cbfunc(reqinfo)

  except Exception, e:
    raise _httpserver_ServerError("httpserver: Callback function " + \
        "raised an exception: " + str(e))

  # Get any extra data consumed from sock by the message body stream object,
  # but that was not actually part of the message body.
  extradata = messagebodystream._get_extra()

  # Interpret result of callback function
  try:
    version = result['version']
    statuscode = result['statuscode']
    statusmsg = result['statusmsg']
    headers = result['headers']
    messagestream = result['message']
  except (KeyError, TypeError):
    raise _httpserver_ServerError("httpserver: Callback function " + \
        "returned malformed dictionary")

  # Do some basic sanity checks:
  if None in (version, statuscode, statusmsg, headers, messagestream):
    raise _httpserver_ServerError("httpserver: Callback function " + \
        "returned dictionary with None for some values.")

  if (type(version), type(statuscode), type(statusmsg), type(headers)) != \
      (str, int, str, dict):
    raise _httpserver_ServerError("httpserver: Callback function " + \
        "returned dictionary with invalid values (wrong types).")

  # If the message object is a string, wrap it in a file-like object.
  if type(messagestream) is str:
    messagestream = _httpserver_StringIO(messagestream)

  # Don't let the callback function serve HTTP/1.1 responses to an HTTP/1.0
  # request.
  if reqinfo['version'] == "1.0" and version == "1.1":
    version = "1.0"

  # Send response as instructed by callback
  if version == "0.9":
    # HTTP/0.9 doesn't have response headers.
    _httpserver_sendfile(sock, messagestream)
    closeconn = True

  elif version == "1.0":
    # Send the response headers:
    response = "HTTP/1.0 " + str(statuscode) + " " + statusmsg + "\r\n"
    for key, val in headers.items():
      response += key + ": " + val + "\r\n"
    response += "\r\n"
    _httpserver_sendAll(sock, response, besteffort=True)

    # Send the response body:
    _httpserver_sendfile(sock, messagestream)
    closeconn = True

  elif version == "1.1":
    response = "HTTP/1.1 " + str(statuscode) + " " + statusmsg + "\r\n"
    for key, val in headers.items():
      response += key + ": " + val + "\r\n"
    response += "Transfer-Encoding: chunked\r\n"
    response += "\r\n"

    # Close client socket if they or the callback function asked us
    # to close.
    if ("Connection" in headers and "close" == headers["Connection"]) \
        or ("Connection" in reqinfo['headers'] and "close" in \
        reqinfo['headers']["Connection"]):
      closeconn = True

    try:
      # Send response headers.
      _httpserver_sendAll(sock, response)

      # Read chunks from the callback and efficiently send them to
      # the client using HTTP/1.1 chunked encoding.
      _httpserver_sendfile_chunked(sock, messagestream)

    except Exception, e:
      if "socket" not in str(e).lower():
        raise

      # The exception we're trying to catch here is anything sock.send()
      # raises. However, it just raises plain exceptions.

      # The reason we care about the data actually going through for HTTP/1.1
      # is that we keep connections open. If there is an error, we shouldn't
      # keep going, so we indicate that the socket should be closed.
      closeconn = True

  else:
    # If the cbfunc's response dictionary didn't specify 0.9, 1.0, or 1.1,
    # it's an error.
    raise _httpserver_ServerError("httpserver: Callback function gave " + \
        "invalid HTTP version")

  # Clean up open file handles:
  messagestream.close()

  return (closeconn, extradata)