docs: def v1_highlights_post(request, response, kvlclient, tfidf, 
                         min_delay=3, task_master=None):
      '''Obtain highlights for a document POSTed inside a JSON object.
  
      Get our Diffeo Highlighter browser extension here:
      https://chrome.google.com/webstore/detail/jgfcplgdmjkdepnmbdkmgohaldaiplpo
  
      While you're at it, pre-register for a beta account on
      http://diffeo.com.
  
      `min_delay` and `task_master` are used by tests.
  
      The route for this endpoint is:
      ``POST /dossier/v1/highlights``.
  
      The expected input structure is a JSON encoded string of an
      object with these keys:
  
      .. code-block:: javascript
        {
          // only text/html is supported at this time; hopefully PDF.js
          // enables this to support PDF rendering too.
          "content-type": "text/html",
  
          // URL of the page (after resolving all redirects)
          "content-location": "http://...",
  
          // If provided by the original host, this will be populated,
          // otherwise it is empty.
          "last-modified": "datetime string or empty string",
  
          // Boolean indicating whether the content may be stored by the
          // server.  If set to `false`, then server must respond
          // synchronously with a newly computed response payload, and
          // must purge any stored copies of this `content-location`.
          // If `true`, server may respond with `state` of `pending`.
          "store": false,
  
          // full page contents obtained by Javascript in the browser
          // extension accessing `document.documentElement.innerHTML`.
          // This must be UTF-8 encoded.
          // N.B. This needs experimentation to figure out whether the
          // browser will always encode this as Unicode.
          "body": "... the body content ...",
        }
  
      The output structure is a JSON UTF-8 encoded string of an
      object with these keys:
  
      .. code-block:: javascript
  
        {
          "highlights": [Highlight, Highlight, ...],
          "state":  State,
          "id": StoreID,
          "delay": 10.0,
          "error": Error
        }
  
      where a `State` is one of these strings: `completed`, `stored`,
      `pending`, or `error`.  The `StoreID` is an opaque string computed
      by the backend that the client can use to poll this end point with
      `GET` requests for a `pending` request.  The `delay` value is a
      number of seconds that the client should wait before beginning
      polling, e.g. ten seconds.
  
      An `Error` object has this structure:
  
      .. code-block:: javascript
        {
  
          // Error codes are (0, wrong content type), (1, empty body),
          // (2, JSON decode error), (3, payload structure incorrect),
          // (4, payload missing required keys), (5, invalid
          // content-location), (6, too small body content), (7,
          // internal error), (8, internal time out), (9, file_id does
          // not exist)
          "code": 0,
  
          "message": "wrong content_type"
        }
  
      A `Highlight` object has this structure:
  
      .. code-block:: javascript
  
        {
          // float in the range [0, 1]
          "score": 0.7
  
          // a string presented with a check box inside the options
          // bubble when the user clicks the extension icon to choose
          // which categories of highlights should be displayed.
          "category": "Organization",
  
          // `queries` are strings that are to be presented as
          // suggestions to the user, and the extension enables the user
          // to click any of the configured search engines to see
          // results for a selected query string.
          "queries": [],
  
          // zero or more strings to match in the document and highlight
          // with a single color.
          "strings": [],
  
          // zero or more xpath highlight objects to lookup in the document
          // and highlight with a single color.
          "xranges": [],
  
          // zero or more Regex objects to compile and
          // execute to find spans to highlight with a single color.
          "regexes": []
        }
  
      where a Regex object is:
  
      .. code-block:: javascript
  
        {
          "regex": "...", // e.g., "[0-9]"
          "flags": "..."  // e.g., "i" for case insensitive
        }
  
      where an xpath highlight object is:
  
      .. code-block:: javascript
  
        {
          "range": XPathRange
        }
  
      where an XpathRange object is:
  
      .. code-block:: javascript
  
        {
          "start": XPathOffset,
          "end": XPathOffset
        }
  
      where an XpathOffset object is:
  
      .. code-block:: javascript
  
        {
          "node": "/html[1]/body[1]/p[1]/text()[2]",
          "idx": 4,
        }
  
      All of the `strings`, `ranges`, and `regexes` in a `Highlight`
      object should be given the same highlight color.  A `Highlight`
      object can provide values in any of the three `strings`, `ranges`,
      or `regexes` lists, and all should be highlighted.
      '''
      tfidf = tfidf or None
      content_type = request.headers.get('content-type', '')
      if not content_type.startswith('application/json'):
          logger.critical('content-type=%r', content_type)
          response.status = 415
          return {
  	    'state': ERROR,
              'error': {
                  'code': 0,
                  'message': 'content_type=%r and should be '
                             'application/json' % content_type,
              },
          }
  
      body = request.body.read()
      if len(body) == 0:
          response.status = 400
          return {
              'state': ERROR,
              'error': {'code': 1, 'message': 'empty body'}
          }
      try:
          data = json.loads(body.decode('utf-8'))
      except Exception, exc:
          response.status = 400
          return {
  	    'state': ERROR,
              'error': {
                  'code': 2,
                  'message':
                  'failed to read JSON body: %s' % exc,
              },
          }
  
      if not isinstance(data, dict):
          response.status = 400
          return {
  	    'state': ERROR,
              'error': {
                  'code': 3,
                  'message': 'JSON request payload deserialized to'
                        ' other than an object: %r' % type(data),
              },
          }
  
      expected_keys = set([
          'content-type', 'content-location', 'last-modified', 'body',
  	'store',
      ])
      if set(data.keys()) != expected_keys:
          response.status = 400
          return {
  	    'state': ERROR,
              'error': {
                  'code': 4,
                  'message': 'other than expected keys in JSON object. '
                             'Expected %r and received %r'
                             % (sorted(expected_keys), sorted(data.keys())),
              },
          }
  
      if len(data['content-location']) < 3:
          response.status = 400
          return {
  	    'state': ERROR,
              'error': {
                  'code': 5,
                  'message': 'received invalid content-location=%r'
                             % data['content-location'],
              },
          }
  
      if len(data['body']) < 3:
          response.status = 400
          return {
  	    'state': ERROR,
              'error': {
                  'code': 6,
                  'message': 'received too little body=%r' % data['body'],
              },
          }
  
      if data['last-modified']:
          try:
              last_modified = int(datetime.datetime(*eut.parsedate(data['last-modified'])[:6]).strftime('%s'))
          except Exception, exc:
              logger.info('failed to parse last-modified=%r', data['last-modified'])
              last_modified = 0
      else:
          last_modified = 0
      doc_id = md5(data['content-location']).hexdigest()
      content_hash = Nilsimsa(data['body']).hexdigest()
      file_id = (doc_id, last_modified, content_hash)
      file_id_str = '%s-%d-%s' % file_id
  
      kvlclient.setup_namespace(highlights_kvlayer_tables)
      if data['store'] is False:
          kvlclient.delete('files', (file_id[0],))
          kvlclient.delete('highlights', (file_id[0],))
          logger.info('cleared all store records related to doc_id=%r', file_id[0])
      else: # storing is allowed
          payload_strs = list(kvlclient.get('highlights', file_id))
          if payload_strs and payload_strs[0][1]:
              payload_str = payload_strs[0][1]
              try:
                  payload = json.loads(payload_str)
              except Exception, exc:
                  logger.critical('failed to decode out of %r', 
                                  payload_str, exc_info=True)
              if payload['state'] != ERROR:
                  logger.info('returning stored payload for %r', file_id)
                  return payload
              else:
                  logger.info('previously stored data was an error so trying again')
  
          delay = len(data['body']) / 5000 # one second per 5KB
          if delay > min_delay:
              # store the data in `files` table
              kvlclient.put('files', (file_id, json.dumps(data)))
              payload = {
                  'state': HIGHLIGHTS_PENDING,
                  'id': file_id_str,
                  'delay': delay,
                  'start': time.time()
              }
              # store the payload, so that it gets returned during
              # polling until replaced by the work unit.
              payload_str = json.dumps(payload)
              kvlclient.put('highlights', (file_id, payload_str))
  
              logger.info('launching highlights async work unit')
              if task_master is None:
                  conf = yakonfig.get_global_config('coordinate')
                  task_master = coordinate.TaskMaster(conf)
              task_master.add_work_units('highlights', [(file_id_str, {})])
  
              return payload
  
      return maybe_store_highlights(file_id, data, tfidf, kvlclient)