code: void reduceVocab()
      {
          table = new int[vocabSize];
          int j = 0;
          for (int i = 0; i < vocabSize; i++)
          {
              if (vocab[i].cn > minReduce)
              {
                  vocab[j].cn = vocab[i].cn;
                  vocab[j].word = vocab[i].word;
                  table[vocabIndexMap.get(vocab[j].word)] = j;
                  j++;
              }
              else
              {
                  table[vocabIndexMap.get(vocab[j].word)] = -4;
              }
          }
          // adjust the index in the cache
          try
          {
              cache.close();
              File fixingFile = new File(cacheFile.getAbsolutePath() + ".fixing");
              cache = new DataOutputStream(new FileOutputStream(fixingFile));
              DataInputStream oldCache = new DataInputStream(new FileInputStream(cacheFile));
              while (oldCache.available() >= 4)
              {
                  int oldId = oldCache.readInt();
                  if (oldId < 0)
                  {
                      cache.writeInt(oldId);
                      continue;
                  }
                  int id = table[oldId];
                  if (id == -4) continue;
                  cache.writeInt(id);
              }
              oldCache.close();
              cache.close();
              if (!fixingFile.renameTo(cacheFile))
              {
                  throw new RuntimeException(String.format("moving %s to %s failed", fixingFile.getAbsolutePath(), cacheFile.getName()));
              }
              cache = new DataOutputStream(new FileOutputStream(cacheFile));
          }
          catch (IOException e)
          {
              throw new RuntimeException(String.format("failed to adjust cache file", e));
          }
          table = null;
          vocabSize = j;
          vocabIndexMap.clear();
          for (int i = 0; i < vocabSize; i++)
          {
              vocabIndexMap.put(vocab[i].word, i);
          }
          minReduce++;
      }
docs: Reduces the vocabulary by removing infrequent tokens