from __future__ import print_function
import shutil
import logging
logger = logging.getLogger(__name__)
from atlasseq.storage.base import BerkeleyDBStorage


def rowjoin(partitioned_data, out_db, N=25000000):
    db_out = BerkeleyDBStorage(config={'filename': out_db})
    for x in ["colour_to_sample_lookup", "sample_to_colour_lookup", "metadata"]:
        shutil.copy("".join([partitioned_data, "_0", x]), "".join([out_db, x]))
    batch = 0
    db = BerkeleyDBStorage(
        config={'filename': "".join([partitioned_data, "_", str(batch)])})
    for i in range(N):
        if i % 10000 == 0 and not i == 0:
            logger.info("%i of %i %f%% " % (i, N, 100*i/N))
            db.storage.close()
            batch += 1
            db = BerkeleyDBStorage(
                config={'filename': "".join([partitioned_data, "_", str(batch)])})
        db_out[i] = db[i]
    return {'graph': out_db}

from __future__ import print_function
import shutil
import logging
logger = logging.getLogger(__name__)
from atlasseq.storage.base import BerkeleyDBStorage


def rowjoin(partitioned_data, out_db, N=25000000):
    N = int(N)
    db_out = BerkeleyDBStorage(config={'filename': out_db})
    for x in ["colour_to_sample_lookup", "sample_to_colour_lookup", "metadata"]:
        shutil.copy("".join([partitioned_data, "_0", x]), "".join([out_db, x]))
    batch = 0
    db = BerkeleyDBStorage(
        config={'filename': "".join([partitioned_data, "_", str(batch)])})
    for i in range(N):
        if i % 10000 == 0 and not i == 0:
            logger.info("%i of %i %f%% " % (i, N, 100*i/N))
            db.storage.close()
            batch += 1
            db = BerkeleyDBStorage(
                config={'filename': "".join([partitioned_data, "_", str(batch)])})
        db_out[i] = db[i]
    return {'graph': out_db}

Add row join command for merging berkeley DBs
Add row join command for merging berkeley DBs

  from __future__ import print_function
  import shutil
  import logging
  logger = logging.getLogger(__name__)
  from atlasseq.storage.base import BerkeleyDBStorage
  
  
  def rowjoin(partitioned_data, out_db, N=25000000):
+     N = int(N)
      db_out = BerkeleyDBStorage(config={'filename': out_db})
      for x in ["colour_to_sample_lookup", "sample_to_colour_lookup", "metadata"]:
          shutil.copy("".join([partitioned_data, "_0", x]), "".join([out_db, x]))
      batch = 0
      db = BerkeleyDBStorage(
          config={'filename': "".join([partitioned_data, "_", str(batch)])})
      for i in range(N):
          if i % 10000 == 0 and not i == 0:
              logger.info("%i of %i %f%% " % (i, N, 100*i/N))
              db.storage.close()
              batch += 1
              db = BerkeleyDBStorage(
                  config={'filename': "".join([partitioned_data, "_", str(batch)])})
          db_out[i] = db[i]
      return {'graph': out_db}
  
Add row join command for merging berkeley DBs
## Code Before:
from __future__ import print_function
import shutil
import logging
logger = logging.getLogger(__name__)
from atlasseq.storage.base import BerkeleyDBStorage


def rowjoin(partitioned_data, out_db, N=25000000):
    db_out = BerkeleyDBStorage(config={'filename': out_db})
    for x in ["colour_to_sample_lookup", "sample_to_colour_lookup", "metadata"]:
        shutil.copy("".join([partitioned_data, "_0", x]), "".join([out_db, x]))
    batch = 0
    db = BerkeleyDBStorage(
        config={'filename': "".join([partitioned_data, "_", str(batch)])})
    for i in range(N):
        if i % 10000 == 0 and not i == 0:
            logger.info("%i of %i %f%% " % (i, N, 100*i/N))
            db.storage.close()
            batch += 1
            db = BerkeleyDBStorage(
                config={'filename': "".join([partitioned_data, "_", str(batch)])})
        db_out[i] = db[i]
    return {'graph': out_db}

## Instruction:
Add row join command for merging berkeley DBs
## Code After:
from __future__ import print_function
import shutil
import logging
logger = logging.getLogger(__name__)
from atlasseq.storage.base import BerkeleyDBStorage


def rowjoin(partitioned_data, out_db, N=25000000):
    N = int(N)
    db_out = BerkeleyDBStorage(config={'filename': out_db})
    for x in ["colour_to_sample_lookup", "sample_to_colour_lookup", "metadata"]:
        shutil.copy("".join([partitioned_data, "_0", x]), "".join([out_db, x]))
    batch = 0
    db = BerkeleyDBStorage(
        config={'filename': "".join([partitioned_data, "_", str(batch)])})
    for i in range(N):
        if i % 10000 == 0 and not i == 0:
            logger.info("%i of %i %f%% " % (i, N, 100*i/N))
            db.storage.close()
            batch += 1
            db = BerkeleyDBStorage(
                config={'filename': "".join([partitioned_data, "_", str(batch)])})
        db_out[i] = db[i]
    return {'graph': out_db}
