parameters: (url_file_stream_or_string, etag=None, modified=None, agent=None, referrer=None, handlers=[])
return_statement: return result
docstring: Parse a feed from a URL, file, stream, or string
docstring_summary: Parse a feed from a URL, file, stream, or string
docstring_tokens: Parse a feed from a URL file stream or string
function: def parse(url_file_stream_or_string, etag=None, modified=None, agent=None, referrer=None, handlers=[]):
      '''Parse a feed from a URL, file, stream, or string'''
      result = FeedParserDict()
      result['feed'] = FeedParserDict()
      result['entries'] = []
      if _XML_AVAILABLE:
          result['bozo'] = 0
      if type(handlers) == types.InstanceType:
          handlers = [handlers]
      try:
          f = _open_resource(url_file_stream_or_string, etag, modified, agent, referrer, handlers)
          data = f.read()
      except Exception, e:
          result['bozo'] = 1
          result['bozo_exception'] = e
          data = ''
          f = None
  
      # if feed is gzip-compressed, decompress it
      if f and data and hasattr(f, 'headers'):
          if gzip and f.headers.get('content-encoding', '') == 'gzip':
              try:
                  data = gzip.GzipFile(fileobj=_StringIO(data)).read()
              except Exception, e:
                  # Some feeds claim to be gzipped but they're not, so
                  # we get garbage.  Ideally, we should re-request the
                  # feed without the 'Accept-encoding: gzip' header,
                  # but we don't.
                  result['bozo'] = 1
                  result['bozo_exception'] = e
                  data = ''
          elif zlib and f.headers.get('content-encoding', '') == 'deflate':
              try:
                  data = zlib.decompress(data, -zlib.MAX_WBITS)
              except Exception, e:
                  result['bozo'] = 1
                  result['bozo_exception'] = e
                  data = ''
  
      # save HTTP headers
      if hasattr(f, 'info'):
          info = f.info()
          result['etag'] = info.getheader('ETag')
          last_modified = info.getheader('Last-Modified')
          if last_modified:
              result['modified'] = _parse_date(last_modified)
      if hasattr(f, 'url'):
          result['href'] = f.url
          result['status'] = 200
      if hasattr(f, 'status'):
          result['status'] = f.status
      if hasattr(f, 'headers'):
          result['headers'] = f.headers.dict
      if hasattr(f, 'close'):
          f.close()
  
      # there are four encodings to keep track of:
      # - http_encoding is the encoding declared in the Content-Type HTTP header
      # - xml_encoding is the encoding declared in the <?xml declaration
      # - sniffed_encoding is the encoding sniffed from the first 4 bytes of the XML data
      # - result['encoding'] is the actual encoding, as per RFC 3023 and a variety of other conflicting specifications
      http_headers = result.get('headers', {})
      result['encoding'], http_encoding, xml_encoding, sniffed_xml_encoding, acceptable_content_type = \
          _getCharacterEncoding(http_headers, data)
      if http_headers and (not acceptable_content_type):
          if http_headers.has_key('content-type'):
              bozo_message = '%s is not an XML media type' % http_headers['content-type']
          else:
              bozo_message = 'no Content-type specified'
          result['bozo'] = 1
          result['bozo_exception'] = NonXMLContentType(bozo_message)
          
      result['version'], data = _stripDoctype(data)
  
      baseuri = http_headers.get('content-location', result.get('href'))
      baselang = http_headers.get('content-language', None)
  
      # if server sent 304, we're done
      if result.get('status', 0) == 304:
          result['version'] = ''
          result['debug_message'] = 'The feed has not changed since you last checked, ' + \
              'so the server sent no data.  This is a feature, not a bug!'
          return result
  
      # if there was a problem downloading, we're done
      if not data:
          return result
  
      # determine character encoding
      use_strict_parser = 0
      known_encoding = 0
      tried_encodings = []
      # try: HTTP encoding, declared XML encoding, encoding sniffed from BOM
      for proposed_encoding in (result['encoding'], xml_encoding, sniffed_xml_encoding):
          if not proposed_encoding: continue
          if proposed_encoding in tried_encodings: continue
          tried_encodings.append(proposed_encoding)
          try:
              data = _toUTF8(data, proposed_encoding)
              known_encoding = use_strict_parser = 1
              break
          except:
              pass
      # if no luck and we have auto-detection library, try that
      if (not known_encoding) and chardet:
          try:
              proposed_encoding = chardet.detect(data)['encoding']
              if proposed_encoding and (proposed_encoding not in tried_encodings):
                  tried_encodings.append(proposed_encoding)
                  data = _toUTF8(data, proposed_encoding)
                  known_encoding = use_strict_parser = 1
          except:
              pass
      # if still no luck and we haven't tried utf-8 yet, try that
      if (not known_encoding) and ('utf-8' not in tried_encodings):
          try:
              proposed_encoding = 'utf-8'
              tried_encodings.append(proposed_encoding)
              data = _toUTF8(data, proposed_encoding)
              known_encoding = use_strict_parser = 1
          except:
              pass
      # if still no luck and we haven't tried windows-1252 yet, try that
      if (not known_encoding) and ('windows-1252' not in tried_encodings):
          try:
              proposed_encoding = 'windows-1252'
              tried_encodings.append(proposed_encoding)
              data = _toUTF8(data, proposed_encoding)
              known_encoding = use_strict_parser = 1
          except:
              pass
      # if still no luck, give up
      if not known_encoding:
          result['bozo'] = 1
          result['bozo_exception'] = CharacterEncodingUnknown( \
              'document encoding unknown, I tried ' + \
              '%s, %s, utf-8, and windows-1252 but nothing worked' % \
              (result['encoding'], xml_encoding))
          result['encoding'] = ''
      elif proposed_encoding != result['encoding']:
          result['bozo'] = 1
          result['bozo_exception'] = CharacterEncodingOverride( \
              'documented declared as %s, but parsed as %s' % \
              (result['encoding'], proposed_encoding))
          result['encoding'] = proposed_encoding
  
      if not _XML_AVAILABLE:
          use_strict_parser = 0
      if use_strict_parser:
          # initialize the SAX parser
          feedparser = _StrictFeedParser(baseuri, baselang, 'utf-8')
          saxparser = xml.sax.make_parser(PREFERRED_XML_PARSERS)
          saxparser.setFeature(xml.sax.handler.feature_namespaces, 1)
          saxparser.setContentHandler(feedparser)
          saxparser.setErrorHandler(feedparser)
          source = xml.sax.xmlreader.InputSource()
          source.setByteStream(_StringIO(data))
          if hasattr(saxparser, '_ns_stack'):
              # work around bug in built-in SAX parser (doesn't recognize xml: namespace)
              # PyXML doesn't have this problem, and it doesn't have _ns_stack either
              saxparser._ns_stack.append({'http://www.w3.org/XML/1998/namespace':'xml'})
          try:
              saxparser.parse(source)
          except Exception, e:
              if _debug:
                  import traceback
                  traceback.print_stack()
                  traceback.print_exc()
                  sys.stderr.write('xml parsing failed\n')
              result['bozo'] = 1
              result['bozo_exception'] = feedparser.exc or e
              use_strict_parser = 0
      if not use_strict_parser:
          feedparser = _LooseFeedParser(baseuri, baselang, known_encoding and 'utf-8' or '')
          feedparser.feed(data)
      result['feed'] = feedparser.feeddata
      result['entries'] = feedparser.entries
      result['version'] = result['version'] or feedparser.version
      result['namespaces'] = feedparser.namespacesInUse
      return result
function_tokens: def parse ( url_file_stream_or_string , etag = None , modified = None , agent = None , referrer = None , handlers = [ ] ) : result = FeedParserDict ( ) result [ 'feed' ] = FeedParserDict ( ) result [ 'entries' ] = [ ] if _XML_AVAILABLE : result [ 'bozo' ] = 0 if type ( handlers ) == types . InstanceType : handlers = [ handlers ] try : f = _open_resource ( url_file_stream_or_string , etag , modified , agent , referrer , handlers ) data = f . read ( ) except Exception , e : result [ 'bozo' ] = 1 result [ 'bozo_exception' ] = e data = '' f = None # if feed is gzip-compressed, decompress it if f and data and hasattr ( f , 'headers' ) : if gzip and f . headers . get ( 'content-encoding' , '' ) == 'gzip' : try : data = gzip . GzipFile ( fileobj = _StringIO ( data ) ) . read ( ) except Exception , e : # Some feeds claim to be gzipped but they're not, so # we get garbage.  Ideally, we should re-request the # feed without the 'Accept-encoding: gzip' header, # but we don't. result [ 'bozo' ] = 1 result [ 'bozo_exception' ] = e data = '' elif zlib and f . headers . get ( 'content-encoding' , '' ) == 'deflate' : try : data = zlib . decompress ( data , - zlib . MAX_WBITS ) except Exception , e : result [ 'bozo' ] = 1 result [ 'bozo_exception' ] = e data = '' # save HTTP headers if hasattr ( f , 'info' ) : info = f . info ( ) result [ 'etag' ] = info . getheader ( 'ETag' ) last_modified = info . getheader ( 'Last-Modified' ) if last_modified : result [ 'modified' ] = _parse_date ( last_modified ) if hasattr ( f , 'url' ) : result [ 'href' ] = f . url result [ 'status' ] = 200 if hasattr ( f , 'status' ) : result [ 'status' ] = f . status if hasattr ( f , 'headers' ) : result [ 'headers' ] = f . headers . dict if hasattr ( f , 'close' ) : f . close ( ) # there are four encodings to keep track of: # - http_encoding is the encoding declared in the Content-Type HTTP header # - xml_encoding is the encoding declared in the <?xml declaration # - sniffed_encoding is the encoding sniffed from the first 4 bytes of the XML data # - result['encoding'] is the actual encoding, as per RFC 3023 and a variety of other conflicting specifications http_headers = result . get ( 'headers' , { } ) result [ 'encoding' ] , http_encoding , xml_encoding , sniffed_xml_encoding , acceptable_content_type = _getCharacterEncoding ( http_headers , data ) if http_headers and ( not acceptable_content_type ) : if http_headers . has_key ( 'content-type' ) : bozo_message = '%s is not an XML media type' % http_headers [ 'content-type' ] else : bozo_message = 'no Content-type specified' result [ 'bozo' ] = 1 result [ 'bozo_exception' ] = NonXMLContentType ( bozo_message ) result [ 'version' ] , data = _stripDoctype ( data ) baseuri = http_headers . get ( 'content-location' , result . get ( 'href' ) ) baselang = http_headers . get ( 'content-language' , None ) # if server sent 304, we're done if result . get ( 'status' , 0 ) == 304 : result [ 'version' ] = '' result [ 'debug_message' ] = 'The feed has not changed since you last checked, ' + 'so the server sent no data.  This is a feature, not a bug!' return result # if there was a problem downloading, we're done if not data : return result # determine character encoding use_strict_parser = 0 known_encoding = 0 tried_encodings = [ ] # try: HTTP encoding, declared XML encoding, encoding sniffed from BOM for proposed_encoding in ( result [ 'encoding' ] , xml_encoding , sniffed_xml_encoding ) : if not proposed_encoding : continue if proposed_encoding in tried_encodings : continue tried_encodings . append ( proposed_encoding ) try : data = _toUTF8 ( data , proposed_encoding ) known_encoding = use_strict_parser = 1 break except : pass # if no luck and we have auto-detection library, try that if ( not known_encoding ) and chardet : try : proposed_encoding = chardet . detect ( data ) [ 'encoding' ] if proposed_encoding and ( proposed_encoding not in tried_encodings ) : tried_encodings . append ( proposed_encoding ) data = _toUTF8 ( data , proposed_encoding ) known_encoding = use_strict_parser = 1 except : pass # if still no luck and we haven't tried utf-8 yet, try that if ( not known_encoding ) and ( 'utf-8' not in tried_encodings ) : try : proposed_encoding = 'utf-8' tried_encodings . append ( proposed_encoding ) data = _toUTF8 ( data , proposed_encoding ) known_encoding = use_strict_parser = 1 except : pass # if still no luck and we haven't tried windows-1252 yet, try that if ( not known_encoding ) and ( 'windows-1252' not in tried_encodings ) : try : proposed_encoding = 'windows-1252' tried_encodings . append ( proposed_encoding ) data = _toUTF8 ( data , proposed_encoding ) known_encoding = use_strict_parser = 1 except : pass # if still no luck, give up if not known_encoding : result [ 'bozo' ] = 1 result [ 'bozo_exception' ] = CharacterEncodingUnknown ( 'document encoding unknown, I tried ' + '%s, %s, utf-8, and windows-1252 but nothing worked' % ( result [ 'encoding' ] , xml_encoding ) ) result [ 'encoding' ] = '' elif proposed_encoding != result [ 'encoding' ] : result [ 'bozo' ] = 1 result [ 'bozo_exception' ] = CharacterEncodingOverride ( 'documented declared as %s, but parsed as %s' % ( result [ 'encoding' ] , proposed_encoding ) ) result [ 'encoding' ] = proposed_encoding if not _XML_AVAILABLE : use_strict_parser = 0 if use_strict_parser : # initialize the SAX parser feedparser = _StrictFeedParser ( baseuri , baselang , 'utf-8' ) saxparser = xml . sax . make_parser ( PREFERRED_XML_PARSERS ) saxparser . setFeature ( xml . sax . handler . feature_namespaces , 1 ) saxparser . setContentHandler ( feedparser ) saxparser . setErrorHandler ( feedparser ) source = xml . sax . xmlreader . InputSource ( ) source . setByteStream ( _StringIO ( data ) ) if hasattr ( saxparser , '_ns_stack' ) : # work around bug in built-in SAX parser (doesn't recognize xml: namespace) # PyXML doesn't have this problem, and it doesn't have _ns_stack either saxparser . _ns_stack . append ( { 'http://www.w3.org/XML/1998/namespace' : 'xml' } ) try : saxparser . parse ( source ) except Exception , e : if _debug : import traceback traceback . print_stack ( ) traceback . print_exc ( ) sys . stderr . write ( 'xml parsing failed\n' ) result [ 'bozo' ] = 1 result [ 'bozo_exception' ] = feedparser . exc or e use_strict_parser = 0 if not use_strict_parser : feedparser = _LooseFeedParser ( baseuri , baselang , known_encoding and 'utf-8' or '' ) feedparser . feed ( data ) result [ 'feed' ] = feedparser . feeddata result [ 'entries' ] = feedparser . entries result [ 'version' ] = result [ 'version' ] or feedparser . version result [ 'namespaces' ] = feedparser . namespacesInUse return result