code: private void verifyUrls(Set<CrawlerURL> allUrls, Set<HTMLPageResponse> verifiedUrls,
        Set<HTMLPageResponse> nonWorkingUrls, Map<String, String> requestHeaders) {
  
      Set<CrawlerURL> urlsThatNeedsVerification = new LinkedHashSet<CrawlerURL>(allUrls);
  
      urlsThatNeedsVerification.removeAll(verifiedUrls);
  
      final Set<Callable<HTMLPageResponse>> tasks =
          new HashSet<Callable<HTMLPageResponse>>(urlsThatNeedsVerification.size());
  
      for (CrawlerURL testURL : urlsThatNeedsVerification) {
        tasks.add(new HTMLPageResponseCallable(testURL, responseFetcher, true, requestHeaders, false));
      }
  
      try {
        // wait for all urls to verify
        List<Future<HTMLPageResponse>> responses = service.invokeAll(tasks);
  
        for (Future<HTMLPageResponse> future : responses) {
          if (!future.isCancelled()) {
            HTMLPageResponse response = future.get();
            if (response.getResponseCode() == HttpStatus.SC_OK
                && response.getResponseType().indexOf("html") > 0) {
              // remove, way of catching interrupted / execution e
              urlsThatNeedsVerification.remove(response.getPageUrl());
              verifiedUrls.add(response);
            } else if (response.getResponseCode() == HttpStatus.SC_OK) {
              // it is not HTML
              urlsThatNeedsVerification.remove(response.getPageUrl());
            } else {
              nonWorkingUrls.add(response);
            }
          }
        }
  
      } catch (InterruptedException e1) {
        // TODO add some logging
        e1.printStackTrace();
      } catch (ExecutionException e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
      }
  
      // TODO: We can have a delta here if the exception occur
  
    }
docs: Verify that all urls in allUrls returns 200. If not, they will be removed from that set and
  instead added to the nonworking list.
  
  @param allUrls all the links that has been fetched
  @param nonWorkingUrls links that are not working