feat(post-processing): allow `write_tables` to compact output LevelDB
@@ -23,6 +23,7 @@ import (
"errors"
"flag"
"log"
+ "time"
"kythe.io/kythe/go/platform/vfs"
"kythe.io/kythe/go/services/graphstore"
@@ -61,6 +62,7 @@ var (
experimentalBeamPipeline = flag.Bool("experimental_beam_pipeline", false, "Whether to use the Beam experimental pipeline implementation")
experimentalColumnarData = flag.Bool("experimental_beam_columnar_data", false, "Whether to emit columnar data from the Beam pipeline implementation")
+ compactTable = flag.Bool("compact_table", false, "Whether to compact the output LevelDB after its creation")
)
func init() {
@@ -78,6 +80,11 @@ func main() {
if err := runExperimentalBeamPipeline(ctx); err != nil {
log.Fatalf("Pipeline error: %v", err)
}
+ if *compactTable {
+ if err := compactLevelDB(*tablePath); err != nil {
+ log.Fatalf("Error compacting LevelDB: %v", err)
+ }
+ }
return
}
@@ -123,6 +130,17 @@ func main() {
}); err != nil {
log.Fatal("FATAL ERROR: ", err)
}
+
+ if *compactTable {
+ if err := compactLevelDB(*tablePath); err != nil {
+ log.Fatalf("Error compacting LevelDB: %v", err)
+ }
+ }
+}
+
+func compactLevelDB(path string) error {
+ defer func(start time.Time) { log.Printf("Compaction completed in %s", time.Since(start)) }(time.Now())
+ return leveldb.CompactRange(*tablePath, nil)
}
func runExperimentalBeamPipeline(ctx context.Context) error {
