parameters: (self, wget_log_file, discarded_url_list=_MARKER,
                       prohibited_file_name_list=None,
                       prohibited_folder_name_list=None)
return_statement: return x_varnish_reference_list, discarded_url_list[:]
docstring: read wget logs and test Caching configuration
docstring_summary: read wget logs and test Caching configuration
docstring_tokens: read wget logs and test Caching configuration
function: def _parseWgetLogs(self, wget_log_file, discarded_url_list=_MARKER,
                       prohibited_file_name_list=None,
                       prohibited_folder_name_list=None):
      """read wget logs and test Caching configuration
      """
      if discarded_url_list is _MARKER:
        first_pass = True
      else:
        first_pass = False
      x_varnish_reference_list = []
  
      for line in  wget_log_file.splitlines():
        logging.debug('wget output:%r' % line)
        code, value = self._parseWgetLine(line)
        if code == self.URL_CODE:
          # This is the first Line by URL checked in wget stdout
          url = value
          logging.debug('url:%r' % url)
          discarded = False
          if not first_pass and url in discarded_url_list:
            # URL already checked during first pass
            logging.debug('%r Discarded' % url)
            discarded = True
          elif self._isSameUrl(url):
            discarded = True
        if discarded:
          # keep reading wget process without doing anything
          continue
        if code == self.STATUS_CODE:
          if value not in self.ACCEPTABLE_STATUS_LIST:
            message = 'Page in error status:%r' % (value)
            self.report_dict.setdefault(url, []).append(message)
        if code == self.FILE_PATH_CODE:
          # Here we check if Response was cached
          file_path = os.path.sep.join((self.working_directory, value))
          folder_path , filename = os.path.split(file_path)
          if prohibited_file_name_list:
            if '?' in filename:
              filename = filename.rpartition('?')[0]
            if filename in prohibited_file_name_list:
              # Error
              message = '%r is prohibited as filename' % filename
              self.report_dict.setdefault(url, []).append(message)
          if prohibited_folder_name_list:
            for prohibited_folder_name in prohibited_folder_name_list:
              if prohibited_folder_name in folder_path.split(os.path.sep):
                message = '%r is prohibited as folder name' % prohibited_folder_name
                self.report_dict.setdefault(url, []).append(message)
                break
          file_object = None
          try:
            file_object = open(file_path, 'r')
          except IOError:
            # This is probably a folder try with /index.html
            index_file_path = file_path + os.path.sep + 'index.html'
            try:
              file_object = open(index_file_path, 'r')
            except IOError:
             # sometimes this is appended with .1
             number_file_path = file_path + '.1'
             try:
               file_object = open(number_file_path, 'r')
             except IOError:
               logging.info('File not found for url:%r %r' %\
                          (url, (file_path, index_file_path, number_file_path),))
               continue
          fetched_data = file_object.read()
          file_object.close()
          x_cache_header_match_object =\
                            self.x_cache_header_search_regex.search(fetched_data)
          if x_cache_header_match_object is None:
            # This RESPONSE is not cached by Varnish
            message = 'X-Cache header not found'
            self.report_dict.setdefault(url, []).append(message)
          else:
            # means X-Cache header is present in reponse
            # Read the X-Varnish header to know if backend has been touched
            x_varnish_match_object =\
                          self.x_varnish_header_search_regex.search(fetched_data)
            x_varnish_reference = x_varnish_match_object.group(1)
            logging.debug('x_varnish_reference:%r for url:%r' %\
                                                      (x_varnish_reference, url))
            hits = x_cache_header_match_object.group(1)
            if hits.isdigit():
              # This is a cached content with a positive hit value
              # Check if request didn't goes to the backend
              x_varnish_reference_list.append((x_varnish_reference, False, url))
              # dot not check this url in second pass
              logging.debug('will be discarded:%r' % url)
              discarded_url_list.append(url)
            else:
              x_varnish_reference_list.append((x_varnish_reference, True, url))
          # parse the web checker configuration file and run the header
          # validation method
          for header, reference_value in self._getHeaderPolicyList(
                                                url, fetched_data):
            self._validateHeader(url, header, reference_value, fetched_data)
      return x_varnish_reference_list, discarded_url_list[:]
function_tokens: def _parseWgetLogs ( self , wget_log_file , discarded_url_list = _MARKER , prohibited_file_name_list = None , prohibited_folder_name_list = None ) : if discarded_url_list is _MARKER : first_pass = True else : first_pass = False x_varnish_reference_list = [ ] for line in wget_log_file . splitlines ( ) : logging . debug ( 'wget output:%r' % line ) code , value = self . _parseWgetLine ( line ) if code == self . URL_CODE : # This is the first Line by URL checked in wget stdout url = value logging . debug ( 'url:%r' % url ) discarded = False if not first_pass and url in discarded_url_list : # URL already checked during first pass logging . debug ( '%r Discarded' % url ) discarded = True elif self . _isSameUrl ( url ) : discarded = True if discarded : # keep reading wget process without doing anything continue if code == self . STATUS_CODE : if value not in self . ACCEPTABLE_STATUS_LIST : message = 'Page in error status:%r' % ( value ) self . report_dict . setdefault ( url , [ ] ) . append ( message ) if code == self . FILE_PATH_CODE : # Here we check if Response was cached file_path = os . path . sep . join ( ( self . working_directory , value ) ) folder_path , filename = os . path . split ( file_path ) if prohibited_file_name_list : if '?' in filename : filename = filename . rpartition ( '?' ) [ 0 ] if filename in prohibited_file_name_list : # Error message = '%r is prohibited as filename' % filename self . report_dict . setdefault ( url , [ ] ) . append ( message ) if prohibited_folder_name_list : for prohibited_folder_name in prohibited_folder_name_list : if prohibited_folder_name in folder_path . split ( os . path . sep ) : message = '%r is prohibited as folder name' % prohibited_folder_name self . report_dict . setdefault ( url , [ ] ) . append ( message ) break file_object = None try : file_object = open ( file_path , 'r' ) except IOError : # This is probably a folder try with /index.html index_file_path = file_path + os . path . sep + 'index.html' try : file_object = open ( index_file_path , 'r' ) except IOError : # sometimes this is appended with .1 number_file_path = file_path + '.1' try : file_object = open ( number_file_path , 'r' ) except IOError : logging . info ( 'File not found for url:%r %r' % ( url , ( file_path , index_file_path , number_file_path ) , ) ) continue fetched_data = file_object . read ( ) file_object . close ( ) x_cache_header_match_object = self . x_cache_header_search_regex . search ( fetched_data ) if x_cache_header_match_object is None : # This RESPONSE is not cached by Varnish message = 'X-Cache header not found' self . report_dict . setdefault ( url , [ ] ) . append ( message ) else : # means X-Cache header is present in reponse # Read the X-Varnish header to know if backend has been touched x_varnish_match_object = self . x_varnish_header_search_regex . search ( fetched_data ) x_varnish_reference = x_varnish_match_object . group ( 1 ) logging . debug ( 'x_varnish_reference:%r for url:%r' % ( x_varnish_reference , url ) ) hits = x_cache_header_match_object . group ( 1 ) if hits . isdigit ( ) : # This is a cached content with a positive hit value # Check if request didn't goes to the backend x_varnish_reference_list . append ( ( x_varnish_reference , False , url ) ) # dot not check this url in second pass logging . debug ( 'will be discarded:%r' % url ) discarded_url_list . append ( url ) else : x_varnish_reference_list . append ( ( x_varnish_reference , True , url ) ) # parse the web checker configuration file and run the header # validation method for header , reference_value in self . _getHeaderPolicyList ( url , fetched_data ) : self . _validateHeader ( url , header , reference_value , fetched_data ) return x_varnish_reference_list , discarded_url_list [ : ]