text: from __future__ import absolute_import, division, print_function
  
  import atexit
  import glob
  import hashlib
  import math
  import os
  import re
  import shutil
  import sys
  import time
  import traceback
  import warnings
  
  from six.moves import cStringIO as StringIO
  
  from libtbx.queuing_system_utils import pbs_utils, sge_utils
  from libtbx.math_utils import round2
  from libtbx.str_utils import show_string
  from six.moves import range
  from six.moves import zip
  from six.moves import input
  
  try: import gzip
  except ImportError: gzip = None
  try: import bz2
  except ImportError: bz2 = None
  
  hashlib_md5 = hashlib.md5
  
  op = os.path
  
  windows_device_names = """\
  CON PRN AUX NUL COM1 COM2 COM3 COM4 COM5 COM6 COM7 COM8 COM9
  LPT1 LPT2 LPT3 LPT4 LPT5 LPT6 LPT7 LPT8 LPT9""".split()
  
  def xfrange(start, stop=None, step=None, tolerance=None):
    """
    A float range generator.
  
    Parameters
    ----------
    start : float
    stop : float, optional
        If empty, start at 0 and stop at the start parameter.
    step : float, optional
    tolerance : float, optional
  
    Returns
    -------
    generator of float
    """
  
    if stop is None:
      stop = start + 0.0
      start = 0.0
    else:
      start += 0.0 # force it to be a float
    if step is None:
      step = 1.0
    else:
      assert step != 0.0
    count = int(math.ceil((stop - start) / step))
    if (    tolerance is not None
        and abs(start + count * step - stop) < abs(step * tolerance)):
      count += 1
    for i in range(count):
      yield start + i * step
  
  def safe_div(a,b):
    if abs(b) < 1e-8:
      return 0
    else:
      return a/b
  
  def frange(start, stop=None, step=None):
    """
    Non-generator version of xfrange.
  
    Parameters
    ----------
    start : float
    stop : float, optional
        If empty, start at 0 and stop at the start parameter.
    step : float, optional
  
    Returns
    -------
    list of float
  
    See Also
    --------
    libtbx.utils.xfrange
    """
    return list(xfrange(start, stop=stop, step=step))
  
  def xsamples(start, stop=None, step=None, tolerance=1e-6):
    """
    Wraps xfrange, acts identically.
  
    Parameters
    ----------
    start : float
    stop : float, optional
        If empty, start at 0 and stop at the start parameter.
    step : float, optional
    tolerance : float, optional
  
    Returns
    -------
    generator of float
  
    See Also
    --------
    libtbx.utils.xfrange
    """
    return xfrange(start, stop, step, tolerance)
  
  def samples(start, stop=None, step=None, tolerance=1e-6):
    """
    Non-generator version of xsamples.
  
    Parameters
    ----------
    start : float
    stop : float, optional
        If empty, start at 0 and stop at the start parameter.
    step : float, optional
    tolerance : float, optional
  
    Returns
    -------
    list of float
  
    See Also
    --------
    libtbx.utils.xfrange, libtbx.utils.xsamples
    """
    return list(xsamples(start, stop, step, tolerance))
  
  def escape_sh_double_quoted(s):
    """
    The result is supposed to be double-quoted when passed to sh.
    """
    if (s is None): return None
    return s.replace('\\','\\\\').replace('"','\\"')
  
  def xlen(seq):
    """
    Returns the length of a sequence or None.
  
    Parameters
    ----------
    seq : iterable or None
  
    Returns
    -------
    int or None
    """
    if (seq is None): return seq
    return len(seq)
  
  def product(seq):
    """
    Calculates the result of multiplying all elements of a sequence together.
  
    Parameters
    ----------
    seq : iterable
    """
    result = None
    for val in seq:
      if (result is None):
        result = val
      else:
        result *= val
    return result
  
  def sequence_index_dict(seq, must_be_unique=True):
    """
    Builds a dictionary for each element in seq mapped to its index in the sequence.
  
    Parameters
    ----------
    seq : iterable of object
    must_be_unique : bool, optional
  
    Returns
    -------
    dict of object, int
  
    Examples
    --------
    >>> libtbx.utils.sequence_index_dict(['a', 'b'])
    {'a': 0, 'b': 1}
    """
    result = {}
    for i, elem in enumerate(seq):
      if must_be_unique:
        assert elem not in result
      result[elem] = i
    return result
  
  def number_from_string(string):
    """
    Tries to covert a string into an integer, using builtin int() as well as eval().
  
    Parameters
    ----------
    string : str
  
    Returns
    -------
    int
  
    Raises
    ------
    ValueError
        If string cannot be converted into an integer.
    """
    # similar to libtbx.phil.number_from_value_string
    # (please review if making changes here)
    if (string.lower() in ["true", "false"]):
      raise ValueError(
        'Error interpreting "%s" as a numeric expression.' % string)
    try: return int(string)
    except ValueError: pass
    try: return eval(string, math.__dict__, {})
    except KeyboardInterrupt: raise
    except Exception:
      raise ValueError(
        'Error interpreting "%s" as a numeric expression: %s' % (
          string, format_exception()))
  
  def gzip_open(file_name, mode):
    """
    Wraps gzip.open to open a .gz file.
  
    Parameters
    ----------
    file_name : str
    mode : str
  
    Returns
    -------
    file
  
    Raises
    ------
    RuntimeError
        If gzip is not available.
    """
    assert mode in ["r", "rb", "w", "wb", "a", "ab"]
    if (gzip is None):
      un = ""
      if (mode[0] == "r"): un = "un"
      raise RuntimeError(
        "gzip module not available: cannot %scompress file %s"
          % (un, show_string(file_name)))
    return gzip.open(file_name, mode)
  
  def bz2_open(file_name, mode):
    """
    Wraps bz2.open to open a .bz2 file.
  
    Parameters
    ----------
    file_name : str
    mode : str
  
    Returns
    -------
    file
  
    Raises
    ------
    RuntimeError
        If bz2 is not available.
    """
    assert mode in ('r', 'w')
    if bz2 is None:
      raise RuntimeError('bz2 module not available: cannot %compress file %s'
                         % ({'r':'un', 'w':''}[mode], file_name))
    return bz2.BZ2File(file_name, mode)
  
  def warn_if_unexpected_md5_hexdigest(
        path,
        expected_md5_hexdigests,
        hints=[],
        out=None):
    """
    Checks the md5 hash of a file to see if it matches the expected hash.
  
    Parameters
    ----------
    path : str
    expected_md5_hexdigests : list of str
    hints : list of str, optional
    out : file, optional
  
    Returns
    -------
    bool
        False if md5 hash of file does not appear in expected_md5_hexdigests.
    """
    m = hashlib.md5()
    m.update("\n".join(open(path).read().splitlines()).encode('utf-8'))
    current_md5_hexdigest = m.hexdigest()
    if (m.hexdigest() in expected_md5_hexdigests): return False
    warning = "Warning: unexpected md5 hexdigest:"
    file_name = "  File: %s" % show_string(path)
    new_hexdigest = "  New md5 hexdigest: %s" % m.hexdigest()
    width = max([len(s) for s in [warning, file_name, new_hexdigest]])
    if (out is None): out = sys.stdout
    print("*"*width, file=out)
    print(warning, file=out)
    print(file_name, file=out)
    print(new_hexdigest, file=out)
    for hint in hints:
      print(hint, file=out)
    print("*"*width, file=out)
    return True
  
  def md5_hexdigest(filename=None, blocksize=256):
    """ Compute the MD5 hexdigest of the content of the given file,
        efficiently even for files much larger than the available RAM.
  
        The file is read by chunks of `blocksize` MB.
    """
    blocksize *= 1024**2
    m = hashlib.md5()
    with open(filename, 'rb') as f:
      buf = f.read(blocksize)
      while buf:
        m.update(buf)
        buf = f.read(blocksize)
    return m.hexdigest()
  
  def get_memory_from_string(mem_str):
    """
    Converts a string of a memory or file size (i.e. "10G") into a number.
  
    Parameters
    ----------
    mem_str : int or float
  
    Returns
    -------
    int or float
  
    Examples
    --------
    >>> libtbx.utils.get_memory_from_string("10G")
    10737418240.0
    >>> libtbx.utils.get_memory_from_string("10M")
    10485760.0
  
    Raises
    ------
    RuntimeError
    """
    if type(mem_str) in [type(1), type(1.)]: return mem_str
    mem_str = mem_str.replace(" ","").strip().upper()
    if mem_str == "": return 0
    factor=1024
    for i, greek in enumerate(["K","M","G","T","E","Z","Y"]):
      num_str=None
      if mem_str[-1]==greek:
        num_str = mem_str[:-1]
      if mem_str.find("%sB" % greek)==len(mem_str)-2:
        num_str = mem_str[:-2]
      if num_str is not None:
        try:
          num = float(num_str)
        except ValueError:
          raise RuntimeError("""
     The numerical portion of %s is not a valid float
  """ % mem_str)
        break
      factor*=1024
    else:
      try:
        num = int(mem_str)
      except ValueError:
        raise RuntimeError("""
     There is no memory unit or valid float in %s
  """ % mem_str)
      factor=1
    return num*factor
  
  def getenv_bool(variable_name, default=False):
    """
    Checks the environment variables for variable, returning it as a boolean.
  
    Parameters
    ----------
    variable_name : str
    default : bool, optional
        Returned if variable_name is not found.
  
    Returns
    -------
    bool
    """
    value = os.environ.get(variable_name, None)
    if (value is None): return default
    value_lower = value.lower()
    if (value_lower not in ["false", "true", "0", "1"]):
      raise Sorry(
        'Environment variable %s must be "True", "False", "0", or "1"'
        ' (current value: "%s").' % (variable_name, value))
    return (value_lower in ["true", "1"])
  
  def file_size(file_name):
    """
    Wraps os.stat to calculate a file's size.
  
    Parameters
    ----------
    file_name : str
  
    Returns
    -------
    int : size of file, in bytes
    """
    return os.stat(file_name).st_size
  
  def copy_file(source, target, compress=None):
    """
    Copies a file from source to target, optionally compressing it before writing
    it out.
  
    Parameters
    ----------
    source : str
    target : str
    compress : str, optional
        The compression algorithm to use. Currently only ".gz" is supported. If
        set, target becomes target + compress.
    """
    assert op.isfile(source)
    if (op.isdir(target)):
      target = op.join(target, op.basename(source))
    if (compress is None):
      t = open(target, "wb")
    else:
      assert compress == ".gz"
      t = gzip_open(file_name=target+compress, mode="wb")
    with open(source, "rb") as f:
      t.write(f.read())
    t.close()
  
  def remove_files(pattern=None, paths=None, ensure_success=True):
    """
    Removes a file from disk.
  
    Parameters
    ----------
    pattern : str, optional
    paths : iterable of str, optional
    ensure_success : bool, optional
    """
    assert [pattern, paths].count(None) == 1
    if (paths is None):
      paths = glob.glob(pattern)
    for path in paths:
      if (ensure_success):
        if (op.exists(path)):
          os.remove(path)
          if (op.exists(path)):
            raise RuntimeError("Cannot remove file: %s" % show_string(path))
      else:
        if (op.isfile(path)):
          os.remove(path)
  
  def find_files(dir_name, pattern="*", files_only=True):
    """
    Find files matching a pattern in a directory.
  
    Parameters
    ----------
    dir_name : str
    pattern: str, optional
    files_only : bool, optional
  
    Returns
    -------
    list of str
    """
    assert os.path.isdir(dir_name) and (pattern is not None)
    regex = re.compile(pattern)
    files = os.listdir(dir_name)
    matching_files = []
    for file_name in files :
      full_path = os.path.join(dir_name, file_name)
      if (files_only) and (not os.path.isfile(full_path)):
        continue
      if (regex.search(file_name) is not None):
        matching_files.append(full_path)
    return matching_files
  
  def sort_files_by_mtime(file_names=None, dir_name=None, reverse=False):
    """
    Sorts a list of file names by when they were last modified, ascending.
  
    Parameters
    ----------
    file_names : iterable of str, optional
    dir_name : str, optional
    reverse : bool, optional
  
    Returns
    -------
    list of str
    """
    assert ([file_names, dir_name].count(None) == 1)
    if (dir_name is not None):
      assert os.path.isdir(dir_name)
      file_names = [ os.path.join(dir_name, fn) for fn in os.listdir(dir_name) ]
    files_and_mtimes = []
    for file_name in file_names :
      files_and_mtimes.append((file_name, os.path.getmtime(file_name)))
    files_and_mtimes.sort(key=lambda x: x[1])
    if (reverse):
      files_and_mtimes.reverse()
    return [ file_name for file_name, mtime in files_and_mtimes ]
  
  def tupleize(x):
    """
    Coverts x into a tuple, either as a direct cast or by making it the sole
    element of a tuple.
  
    Parameters
    ----------
    x : object
  
    Returns
    -------
    tuple
    """
    try:
      return tuple(x)
    except TypeError:
      return (x,)
  
  def plural_s(n, suffix="s"):
    """
    Returns a suffix if n != 1.
  
    Parameters
    ----------
    n : int
    suffix : str, optional
  
    Returns
    -------
    int
    str
    """
    if (n == 1): return n, ""
    return n, suffix
  
  def n_dim_index_from_one_dim(i1d, sizes):
    assert len(sizes) > 0
    result = []
    for sz in reversed(sizes):
      assert sz > 0
      result.append(i1d % sz)
      i1d //= sz
    result.reverse()
    return result
  
  def flat_list(nested_list):
    result = []
    if (hasattr(nested_list, "__len__")):
      for sub_list in nested_list:
        result.extend(flat_list(sub_list))
    else:
      result.append(nested_list)
    return result
  
  def select_matching(key, choices, default=None):
    """
    Selects a value from choices where its key pattern matches key.
  
    Parameters
    ----------
    key : str
    choices : iterable of str, object
    default : object, optional
        Returned if no pattern matches key.
  
    Returns
    -------
    object
    """
    for key_pattern, value in choices:
      m = re.search(key_pattern, key)
      if m is not None: return value
    return default
  
  class KeepType(object):
    def __str__(self):
      return "Keep"
    def __repr__(self):
      return "Keep"
    def __eq__(self, other):
      return isinstance(other, self.__class__)
    def __ne__(self, other):
      return not self.__eq__(other)
    def __hash__(self):
      return hash(KeepType) # return the same hash value for all instances
  
  Keep = KeepType()
  
  class Sorry(Exception):
    """
    Basic exception type for user errors; the traceback will be suppressed.
    """
    __orig_module__ = __module__
    # trick to get just "Sorry" instead of "libtbx.utils.Sorry"
    __module__ = Exception.__module__
  
    def reset_module(self):
      """
      Reset the class module on an instance to libtbx.utils.
      """
      self.__class__.__module__ = self.__class__.__orig_module__
  
  disable_tracebacklimit = "LIBTBX_DISABLE_TRACEBACKLIMIT" in os.environ
  
  __prev_excepthook = sys.excepthook
  
  def sorry_excepthook(type, value, traceback):
    """
    Intercepts exception tracebacks, removing tracebacks for Sorry exceptions.
  
    Parameters
    ----------
    type : type
    value : Exception
    traceback : traceback
  
    Returns
    -------
    str
    """
    tb_off = (not disable_tracebacklimit and isinstance(value, Sorry))
    if (tb_off):
      class __not_set(object): pass
      prev_tracebacklimit = getattr(sys, "tracebacklimit", __not_set)
      sys.tracebacklimit = 0
    result = __prev_excepthook(type, value, traceback)
    if (tb_off):
      if (prev_tracebacklimit is __not_set):
        del sys.tracebacklimit
      else:
        sys.tracebacklimit = prev_tracebacklimit
    return result
  
  sys.excepthook = sorry_excepthook
  
  class Usage(Sorry):
    """
    Subclass of Sorry, for printing out usage instructions upon program
    invocation without arguments (or --help, etc.).
    """
    __module__ = Exception.__module__
  
  class Abort(Sorry):
    """
    Subclass of Sorry, primarily used in the Phenix GUI in response to user
    input.
    """
    __module__ = Exception.__module__
  
  class Failure(Sorry):
    """
    Subclass of Sorry.
    """
    __module__ = Exception.__module__
  
  def kludge_show_to_str(obj):
    """
    Take an object which has a show method which we shall assume will by default
    write it's output to stdout - capture this with cStringIO and return the
    string. Allows objects which have show() but not __repr__ or __str__ methods
    to add without much code changes.
  
    Returns
    -------
    str - output
  
    Raises
    ------
    AttrbuteError if object does not have callable show() method
    """
  
    out = StringIO()
  
    stdout = sys.stdout
    sys.stdout = out
  
    try:
      obj.show()
    finally:
      sys.stdout = stdout
  
    return out.getvalue().rstrip()
  
  def detect_multiprocessing_problem():
    """
    Checks python and library versions and availability to diagnose why
    multiprocessing fails to work.
  
    Returns
    -------
    str or None
        String indicating why multiprocessing is not working.
    """
    vers_info = sys.version_info[:2]
    if (vers_info < (2,6)):
      return "multiprocessing module not available:" \
        " Python 2.6 or higher is required" \
        " (version currently in use: %d.%d)" % vers_info
    import libtbx.load_env
    if (libtbx.env.has_module("omptbx")):
      import omptbx
      if (omptbx.omp_version is not None):
        return "multiprocessing is not compatible with OpenMP"
    sem_open_msg = "This platform lacks a functioning sem_open implementation"
    pool = None
    try:
      try:
        import multiprocessing
        pool = multiprocessing.Pool(processes=2)
        pool.map(func=abs, iterable=range(2), chunksize=1)
      except ImportError as e:
        if (not str(e).startswith(sem_open_msg)):
          raise
        return "multiprocessing import error: " + sem_open_msg
    finally:
      if (pool is not None):
        pool.close()
        pool.join()
    return None
  
  def if_none(value, default):
    """
    Returns value or default if value is None.
    """
    if (value is None): return default
    return value
  
  def format_exception():
    """
    Formats an Exception object...
    """
    ei = sys.exc_info()
    type_ = ei[0].__name__
    value = str(ei[1])
    if (value != ""):
      value = value.replace(" (<string>, line ", " (line ")
    else:
      file_name, line = traceback.extract_tb(sys.exc_info()[2], 1)[0][:2]
      if (file_name is not None):
        value = file_name+" "
      if (line is not None):
        value += "line %d" % line
    return ("%s: %s" % (type_, value)).rstrip()
  
  def show_exception_info_if_full_testing(prefix="EXCEPTION_INFO: "):
    """
    Shows information about an exception.
  
    Parameters
    ----------
    prefix : str, optional
  
    Returns
    -------
    str or None
    """
    import libtbx.load_env
    if (    not libtbx.env.full_testing
        and not disable_tracebacklimit):
      return
    from libtbx import introspection
    from six.moves import cStringIO as StringIO
    sio = StringIO()
    introspection.show_stack(out=sio)
    traceback.print_exc(file=sio)
    msg = "\n".join([prefix+line for line in sio.getvalue().splitlines()]) + "\n"
    del sio
    done = []
    for out in [sys.stdout, sys.stderr, sys.__stdout__, sys.__stderr__]:
      def is_done():
        for o in done:
          if (o is out): return True
        return False
      if (is_done()): continue
      out.write(msg)
      flush = getattr(out, "flush", None)
      if (flush is not None): flush()
      done.append(out)
    return msg
  
  def base36_encode(integer, width=None):
    """
    Encodes integer as a string in base 36, prepending 0's until string is of
    length equal to width.
  
    Parameters
    ----------
    integer : int
    width : int, optional
  
    Returns
    -------
    str
    """
    digit_set = "0123456789abcdefghijklmnopqrstuvwxyz"
    digits = []
    while (integer != 0):
      integer, i = divmod(integer, 36)
      digits.append(digit_set[i])
    if (width is not None):
      while (len(digits) < width):
        digits.append("0")
    digits.reverse()
    return "".join(digits)
  
  def base36_timestamp(seconds_since_epoch=None, multiplier=1000, width=10):
    """
    Encodes the number of seconds since the epoch in base 36.
  
    Parameters
    ----------
    seconds_since_epoch : time, optional
    multiplier : int, optional
    width : int, optional
    """
    s = seconds_since_epoch
    if (s is None):
      s = time.time()
    return base36_encode(integer=int(s * multiplier + 0.5), width=width)
  
  def date_and_time():
    """
    Converts the current time into a string.
  
    Returns
    -------
    str
    """
    seconds_since_epoch = time.time()
    localtime = time.localtime(seconds_since_epoch)
    if (time.daylight and localtime[8] != 0):
      tzname = time.tzname[1]
      offs = -time.altzone
    else:
      tzname = time.tzname[0]
      offs = -time.timezone
    return time.strftime("Date %Y-%m-%d Time %H:%M:%S", localtime) \
         + " %s %+03d%02d (%.2f s)" % (
             tzname, offs//3600, offs//60%60, seconds_since_epoch)
  
  class timer_base(object):
    """
    Base timer class used to calculate the time elapsed by various operations.
    """
  
    def __init__(self):
      self.t = self.get()
  
    def elapsed(self):
      """
      Returns the time elapsed since object was initialized.
  
      Returns
      -------
      time
      """
      t = self.get()
      d = t - self.t
      return d
  
    def delta(self):
      """
      Returns time since last call of delta().
  
      Returns
      -------
      time
      """
      t = self.get()
      d = t - self.t
      self.t = t
      return d
  
    def show_elapsed(self, prefix="", out=None):
      """
      Prints the time since object was initialized.
  
      Parameters
      ----------
      prefix : str, optional
      out : file, optional
      """
      if (out == None): out = sys.stdout
      print(prefix+"%.2f s" % self.elapsed(), file=out)
  
    def show_delta(self, prefix="", out=None):
      """
      Prints the time since last call of delta() or show_delta().
  
      Parameters
      ----------
      prefix : str, optional
      out : file, optional
      """
      if (out == None): out = sys.stdout
      print(prefix+"%.2f s" % self.delta(), file=out)
  
  
  class user_plus_sys_time(timer_base):
    """
    Timer class using os.times() to calculate time. Subclasses timer_base.
  
    Methods
    -------
    get
    """
  
    def get(self):
      """
      Uses os.times() to calculate the time.
      """
      t = os.times()
      return t[0] + t[1]
  
  class wall_clock_time(timer_base):
    """
    Timer class using time.time() to calculate time. Subclasses timer_base.
  
    Methods
    -------
    get
  
    Notes
    -----
    When running multithreaded code, user_plus_sys_time would report the cumulated
    times for all threads: not very useful to analyse the scaling with the number
    of threads! Wall clock time, although it is less reliable is the only solution
    in that case.
    """
  
    def get(self):
      """
      Uses time.time() to calculate the time.
      """
      return time.time()
  
  class time_log(object):
    """
    Class used to log the time that tasks take.
    """
  
    def __init__(self, label, use_wall_clock=False):
      self.label = label
      self.use_wall_clock = use_wall_clock
      self.accumulation = 0
      self.n = 0
      self.delta = 0
      self.timer = None
  
    def start(self):
      """
      Starts the timer.
      """
      if (self.use_wall_clock):
        self.timer = wall_clock_time()
      else:
        self.timer = user_plus_sys_time()
      return self
  
    def stop(self):
      """
      Stops the timer.
      """
      self.delta = self.timer.delta()
      self.timer = None
      self.accumulation += self.delta
      self.n += 1
  
    def average(self):
      """
      Calculates the average length for runs of the timer.
  
      Returns
      -------
      float
      """
      return self.accumulation / max(1, self.n)
  
    def log(self):
      """
      Stops the timer and runs its report method.
      """
      self.stop()
      return self.report()
  
    def log_elapsed(self, local_label):
      """
      Returns a string displaying the time elapsed since the timer was started.
  
      Parameters
      ----------
      local_label : str
          String appended to the end of the output
  
      Returns
      -------
      str
      """
      return "time_log: %s: %.2f elapsed %s" % (
        self.label, self.timer.elapsed(), local_label)
  
    legend = "time_log: label: n accumulation delta average"
  
    def report(self):
      """
      Returns a string including the label, number of stops, accumulated time,
      delta time, and average time for runs of timer.
  
      Returns
      -------
      str
      """
      assert self.timer is None
      return "time_log: %s: %d %.2f %.3g %.3g" % (
        self.label, self.n, self.accumulation,
        self.delta, self.average())
  
  def human_readable_time(time_in_seconds):
    """
    Rounds a time in seconds to the nearest days / hours / minutes, depending on
    what unit is appropriate.
  
    Parameters
    ----------
    time_in_seconds : int
        Time, in seconds.
  
    Returns
    -------
    int : The rounded time in some unit.
    str : The accompanying units for the time.
    """
    time_units = time_in_seconds
    time_unit = "seconds"
    if (time_units > 120):
      time_units /= 60
      time_unit = "minutes"
      if (time_units > 120):
        time_units /= 60
        time_unit = "hours"
        if (time_units > 48):
          time_units /= 24
          time_unit = "days"
    return time_units, time_unit
  
  def human_readable_time_as_seconds(time_units, time_unit):
    """
    Converts time_units and time_unit back into a time with units of seconds.
  
    Parameters
    ----------
    time_units : int
    time_unit : str
  
    Returns
    -------
    int
    """
    if (isinstance(time_units, str)): time_units = float(time_units)
    if (time_unit == "seconds"): return time_units
    if (time_unit == "minutes"): return time_units*60
    if (time_unit == "hours"): return time_units*60*60
    if (time_unit == "days"): return time_units*60*60*24
    raise RuntimeError("Unknown time_unit: %s" % time_unit)
  
  def format_timestamp_12_hour(unix_time, short=False, replace_with="unknown"):
    """
    Formats a unix_time in a 12-hour format.
  
    Parameters
    ----------
    unix_time : time
    short : bool, optional
    replace_with : str, optional
        Returned when unix_time is None.
  
    Returns
    -------
    str
    """
    if unix_time is None :
      return replace_with
    elif short :
      return time.strftime("%d-%m-%y %I:%M %p", time.localtime(float(unix_time)))
    else :
      return time.strftime("%b %d %Y %I:%M %p", time.localtime(float(unix_time)))
  
  def format_timestamp_24_hour(unix_time, short=False, replace_with="unknown"):
    """
    Formats a unix_time in a 24-hour format.
  
    Parameters
    ----------
    unix_time : time
    short : bool, optional
    replace_with : str, optional
        Returned when unix_time is None.
  
    Returns
    -------
    str
    """
    if unix_time is None :
      return "unknown"
    elif short :
      return time.strftime("%d-%m-%y %H:%M", time.localtime(float(unix_time)))
    else :
      return time.strftime("%b %d %Y %H:%M", time.localtime(float(unix_time)))
  
  format_timestamp = format_timestamp_12_hour
  
  def format_cpu_times(show_micro_seconds_per_tick=True):
    t = os.times()
    result = "u+s,u,s: %.2f %.2f %.2f" % (t[0] + t[1], t[0], t[1])
    if (show_micro_seconds_per_tick):
      try: python_ticker = sys.gettickeraccumulation()
      except AttributeError: pass
      else:
        result += " micro-seconds/tick: %.3f" % ((t[0]+t[1])/python_ticker*1.e6)
    return result
  
  def show_total_time(
        out=None,
        show_micro_seconds_per_bytecode_instruction=True):
    """
    Prints the total CPU time and average time for each Python bytecode
    instruction since the process was started.
  
    Parameters
    ----------
    out : file, optional
    show_micro_seconds_per_bytecode_instruction : bool, optional
    """
    if (out == None): out = sys.stdout
    total_time = user_plus_sys_time().get()
    try: python_ticker = sys.gettickeraccumulation()
    except AttributeError: pass
    else:
      print("Time per interpreted Python bytecode instruction:", end=' ', file=out)
      print("%.3f micro seconds" % (total_time / python_ticker * 1.e6), file=out)
    print("Total CPU time: %.2f %s" % human_readable_time(total_time), file=out)
  
  def show_wall_clock_time(seconds, out=None):
    """
    Prints seconds in a human-readable format.
  
    Parameters
    ----------
    seconds : float
    out : file, optional
  
    Examples
    --------
    >>> show_wall_clock_time(750)
    wall clock time: 12 minutes 30.00 seconds (750.00 seconds total)
    >>> show_wall_clock_time(20)
    wall clock time: 20.00 seconds
    """
    if (out is None): out = sys.stdout
    print("wall clock time:", end=' ', file=out)
    if (seconds < 120):
      print("%.2f seconds" % seconds, file=out)
    else:
      m = int(seconds / 60 + 1.e-6)
      s = seconds - m * 60
      print("%d minutes %.2f seconds (%.2f seconds total)" % (
        m, s, seconds), file=out)
    out_flush = getattr(out, "flush", None)
    if (out_flush is not None):
      out_flush()
  
  class show_times:
    """
    Class to track the time past an instance's initialization.
    """
  
    def __init__(self, time_start=None, out=None):
      """
      Parameters
      ----------
      time_start : time or str, optional
      out : file, optional
      """
      if (time_start is None):
        t = os.times()
        self.time_start = time.time() - (t[0] + t[1])
      elif (time_start == "now"):
        self.time_start = time.time()
      else:
        self.time_start = -(0-time_start) # be sure time_start is a number
      self.out = out
  
    def __call__(self):
      out = self.out
      if (out is None): out = sys.stdout
      t = os.times()
      usr_plus_sys = t[0] + t[1]
      try: ticks = sys.gettickeraccumulation()
      except AttributeError: ticks = None
      s = "usr+sys time: %.2f seconds" % usr_plus_sys
      if (ticks is not None):
        s += ", ticks: %d" % ticks
        if (ticks != 0):
          s += ", micro-seconds/tick: %.3f" % (usr_plus_sys*1.e6/ticks)
      print(s, file=out)
      show_wall_clock_time(seconds=time.time()-self.time_start, out=out)
  
  def show_times_at_exit(time_start=None, out=None):
    """
    Shows the time since time_start at exit.
  
    Parameters
    ----------
    time_start : time, optional
    out : file, optional
  
    See Also
    --------
    libtbx.utils.show_times
    """
    atexit.register(show_times(time_start=time_start, out=out))
  
  class host_and_user:
  
    def __init__(self):
      self.host = os.environ.get("HOST")
      self.hostname = os.environ.get("HOSTNAME")
      self.computername = os.environ.get("COMPUTERNAME")
      self.hosttype = os.environ.get("HOSTTYPE")
      self.processor_architecture = os.environ.get("PROCESSOR_ARCHITECTURE")
      self.machtype = os.environ.get("MACHTYPE")
      self.ostype = os.environ.get("OSTYPE")
      self.vendor = os.environ.get("VENDOR")
      self.user = os.environ.get("USER")
      self.username = os.environ.get("USERNAME")
      self.homedir = None
      if (os.name == "nt"):
        homedrive = os.environ.get("HOMEDRIVE")
        homepath = os.environ.get("HOMEPATH")
        if (not None in [homedrive, homepath]):
          self.homedir = os.path.join(homedrive, homepath)
      else :
        self.homedir = os.environ.get("HOME")
      getpid = getattr(os, "getpid", None)
      if (getpid is None):
        self.pid = None
      else:
        self.pid = getpid()
      self.sge_info = sge_utils.info()
      self.pbs_info = pbs_utils.chunk_info()
  
    def get_user_name(self):
      if (self.user is not None):
        return self.user
      else :
        return self.username
  
    def get_host_name(self):
      if (self.host is not None):
        return self.host
      elif (self.hostname is not None):
        return self.hostname
      elif (self.computername is not None):
        return self.computername
      return None
  
    def show(self, out=None, prefix=""):
      if (out is None): out = sys.stdout
      if (self.host is not None):
        print(prefix + "HOST =", self.host, file=out)
      if (    self.hostname is not None
          and self.hostname != self.host):
        print(prefix + "HOSTNAME =", self.hostname, file=out)
      if (    self.computername is not None
          and self.computername != self.host):
        print(prefix + "COMPUTERNAME =", self.computername, file=out)
      if (self.hosttype is not None):
        print(prefix + "HOSTTYPE =", self.hosttype, file=out)
      if (self.processor_architecture is not None):
        print(prefix + "PROCESSOR_ARCHITECTURE =", \
          self.processor_architecture, file=out)
      if (   self.hosttype is None
          or self.machtype is None
          or self.ostype is None
          or "-".join([self.machtype, self.ostype]) != self.hosttype):
        if (self.machtype is not None):
          print(prefix + "MACHTYPE =", \
            self.machtype, file=out)
        if (self.ostype is not None):
          print(prefix + "OSTYPE =", \
            self.ostype, file=out)
      if (self.vendor is not None and self.vendor != "unknown"):
        print(prefix + "VENDOR =", \
          self.vendor, file=out)
      if (self.user is not None):
        print(prefix + "USER =", self.user, file=out)
      if (    self.username is not None
          and self.username != self.user):
        print(prefix + "USERNAME =", self.username, file=out)
      if (self.pid is not None):
        print(prefix + "PID =", self.pid, file=out)
      self.sge_info.show(out=out, prefix=prefix)
      self.pbs_info.show(out=out, prefix=prefix)
  
  def allow_delete_directory(target_dir):
    """
    Check for specified reserved directories which are standard on many systems;
    these should never be deleted as part of any program.
  
    Parameters
    ----------
    target_dir : str
  
    Returns
    -------
    bool
    """
    homedir = host_and_user().homedir
    safe_dirs = [
      homedir,
      os.path.join(homedir, "Documents"),
      os.path.join(homedir, "Desktop"),
      os.path.join(homedir, "Downloads"),
      os.path.join(homedir, "Library"),
      os.path.join(homedir, "Movies"),
      os.path.join(homedir, "data"),
      "/",
      "/home",
      "/Users",
    ]
    target_dir = os.path.abspath(target_dir)
    for safe_dir in safe_dirs :
      if (target_dir == safe_dir):
        return False
    return True
  
  def _indentor_write_loop(write_method, indent, incomplete_line, lines):
    for line in lines:
      if (len(line) == 0):
        incomplete_line = False
      elif (incomplete_line):
        write_method(line)
        incomplete_line = False
      else:
        write_method(indent)
        write_method(line)
      write_method("\n")
  
  class indentor(object):
  
    def __init__(self, file_object=None, indent="", parent=None):
      if (file_object is None):
        if (parent is None):
          file_object = sys.stdout
        else:
          file_object = parent.file_object
      self.file_object = file_object
      if (hasattr(self.file_object, "flush")):
        self.flush = self._flush
      self.indent = indent
      self.parent = parent
      self.incomplete_line = False
  
    def write(self, block):
      if (len(block) == 0): return
      if (block.endswith("\n")):
        _indentor_write_loop(
          write_method=self.file_object.write,
          indent=self.indent,
          incomplete_line=self.incomplete_line,
          lines=block.splitlines())
        self.incomplete_line = False
      else:
        lines = block.splitlines()
        if (len(lines) == 1):
          if (self.incomplete_line):
            self.file_object.write(' ')
            self.file_object.write(lines[-1])
          else:
            self.file_object.write(self.indent + lines[-1])
        else:
          _indentor_write_loop(
            write_method=self.file_object.write,
            indent=self.indent,
            incomplete_line=self.incomplete_line,
            lines=lines[:-1])
          self.file_object.write(self.indent + lines[-1])
        self.incomplete_line = True
  
    def _flush(self):
      self.file_object.flush()
  
    def shift_right(self, indent="  "):
      return self.__class__(indent=self.indent+indent, parent=self)
  
  class buffered_indentor(indentor):
  
    def __init__(self, file_object=None, indent="", parent=None):
      indentor.__init__(self, file_object, indent, parent)
      self.buffer = []
  
    def write(self, block):
      self.buffer.append(block)
  
    def write_buffer(self):
      if (self.parent is not None):
        self.parent.write_buffer()
      for block in self.buffer:
        indentor.write(self, block)
      self.buffer = []
  
  class null_out(object):
    """
    Pseudo-filehandle for suppressing printed output.
    """
  
    def isatty(self): return False
    def close(self): pass
    def flush(self): pass
    def write(self, str): pass
    def writelines(self, sequence): pass
  
  class raise_if_output(object):
    """
    Raises an exception when written to.
  
    Examples
    --------
    >>> import sys
    >>> sys.stdout = libtbx.utils.raise_if_output()
    >>> print
    RuntimeError
    """
  
    def isatty(self):
      return False
  
    def close(self):
      pass
  
    def flush(self):
      pass
  
    def write(self, str):
      """
      Raises
      ------
      RuntimeError
      """
      raise RuntimeError
    def writelines(self, sequence):
      """
      Raises
      ------
      RuntimeError
      """
      raise RuntimeError
  
  class multi_out(object):
    """
    Multiplexing output stream, e.g. for simultaneously printing to stdout
    and a logfile.
    """
  
    def __init__(self):
      self.labels = []
      self.file_objects = []
      self.atexit_send_to = []
      self.closed = False
      self.softspace = 0
      atexit.register(self._atexit)
  
    def _atexit(self):
      if (not self.closed):
        for f,a in zip(self.file_objects, self.atexit_send_to):
          if (a is not None): a.write(f.getvalue())
  
    def register(self, label, file_object, atexit_send_to=None):
      """
      Adds an output stream to the list.
  
      Parameters
      ----------
      label : str
          Label for replacing the stream later
      file_object : file
      atexit_send_to : file, optional
      """
      assert not self.closed
      self.labels.append(label)
      self.file_objects.append(file_object)
      self.atexit_send_to.append(atexit_send_to)
      return self
  
    def replace_stringio(self,
          old_label,
          new_label,
          new_file_object,
          new_atexit_send_to=None):
      """
      Replaces a registered stream with a new file. Dumps everything accumulated
      in stream to the file.
      Useful when at the time of init of this object file name is not known.
      Allows to use StringIO buffer to accumulate output and then dump it to
      file and continue with it.
  
      Parameters
      ----------
      old_label : str
      new_label : str
      new_file_object : file
      new_atexit_send_to : file, optional
      """
      i = self.labels.index(old_label)
      old_file_object = self.file_objects[i]
      new_file_object.write(old_file_object.getvalue())
      old_file_object.close()
      self.labels[i] = new_label
      self.file_objects[i] = new_file_object
      self.atexit_send_to[i] = new_atexit_send_to
  
    def isatty(self):
      return False
  
    def close(self):
      for file_object in self.file_objects:
        if (file_object is sys.__stdout__): continue
        if (file_object is sys.__stderr__): continue
        file_object.close()
      self.closed = True
  
    def flush(self):
      for file_object in self.file_objects:
        flush = getattr(file_object, "flush", None)
        if (flush is not None): flush()
  
    def write(self, str):
      for file_object in self.file_objects:
        file_object.write(str)
  
    def writelines(self, sequence):
      for file_object in self.file_objects:
        file_object.writelines(sequence)
  
  def write_this_is_auto_generated(f, file_name_generator):
    """
    Writes a C header to a file indicating it was generated automatically.
  
    Parameters
    ----------
    f : file
    file_name_generator : str
        Name of source generator.
    """
    print("""\
  /* *****************************************************
     THIS IS AN AUTOMATICALLY GENERATED FILE. DO NOT EDIT.
     *****************************************************
  
     Generated by:
       %s
   */
  """ % file_name_generator, file=f)
  
  class import_python_object:
  
    def __init__(self, import_path, error_prefix, target_must_be, where_str):
      path_elements = import_path.split(".")
      if (len(path_elements) < 2):
        raise ValueError(
          '%simport path "%s" is too short%s%s' % (
            error_prefix, import_path, target_must_be, where_str))
      module_path = ".".join(path_elements[:-1])
      try: module = __import__(module_path)
      except ImportError:
        raise ImportError("%sno module %s%s or possibly import errors in "
        "module %s" % (
          error_prefix, module_path, where_str, module_path))
      for attr in path_elements[1:-1]:
        module = getattr(module, attr)
      try: self.object = getattr(module, path_elements[-1])
      except AttributeError:
        raise AttributeError(
          '%sobject "%s" not found in module "%s"%s' % (
            error_prefix, path_elements[-1], module_path, where_str))
      self.path_elements = path_elements
      self.module_path = module_path
      self.module = module
  
  class input_with_prompt(object):
  
    def __init__(self, prompt, tracebacklimit=0):
      try: import readline
      except Exception: pass
      try: self.previous_tracebacklimit = sys.tracebacklimit
      except Exception: self.previous_tracebacklimit = None
      if (tracebacklimit is not None):
        sys.tracebacklimit = tracebacklimit
      self.input = input(prompt)
  
    def __del__(self):
      if (self.previous_tracebacklimit is None):
        del sys.tracebacklimit
      else:
        sys.tracebacklimit = self.previous_tracebacklimit
  
  def count_max(assert_less_than):
    """
    Counts the number of times its generator is called, raising an exception if
    called too many times.
  
    Parameters
    ----------
    assert_less_than : int
  
    Returns
    -------
    generator of None
  
    Raises
    ------
    AssertionError
        If .next() is called more than assert_less_than on the generator.
    """
    i = 0
    while True:
      yield None
      i += 1
      assert i < assert_less_than
  
  class detect_binary_file(object):
  
    def __init__(self, monitor_initial=None, max_fraction_non_ascii=None):
      if (monitor_initial is None):
        self.monitor_initial = 1000
      else:
        self.monitor_initial = monitor_initial
      if (max_fraction_non_ascii is None):
        self.max_fraction_non_ascii = 0.05
      else:
        self.max_fraction_non_ascii = max_fraction_non_ascii
      self.n_ascii_characters = 0
      self.n_non_ascii_characters = 0
      self.status = None
  
    def is_binary_file(self, block):
      if (self.monitor_initial > 0):
        if block and not isinstance(block[0], int):
          block = (ord(c) for c in block)
        for c in block:
          if (1 < c < 128):
            self.n_ascii_characters += 1
          else:
            self.n_non_ascii_characters += 1
          self.monitor_initial -= 1
          if (self.monitor_initial == 0):
            if (  self.n_non_ascii_characters
                > self.n_ascii_characters * self.max_fraction_non_ascii):
              self.status = True
            else:
              self.status = False
            break
      return self.status
  
    @staticmethod
    def from_initial_block(
          file_name,
          monitor_initial=None,
          max_fraction_non_ascii=None):
      detector = detect_binary_file(
        monitor_initial=monitor_initial,
        max_fraction_non_ascii=max_fraction_non_ascii)
      with open(file_name, "rb") as fh:
        block = fh.read(detector.monitor_initial)
      if not block:
        return False
      detector.monitor_initial = min(len(block), detector.monitor_initial)
      return detector.is_binary_file(block=block)
  
  def search_for(
        pattern,
        mode,
        re_flags=0,
        lines=None,
        file_name=None):
    """
    Searches for a pattern in a file's contents.
  
    Parameters
    ----------
    pattern : str
    mode : str
        One of "==", "find", "startswith", "endswith", "re.search", "re.match"
    re_flags : int, optional
    lines : iterable of str, optional
    file_name : str, optional
    """
    assert mode in ["==", "find", "startswith", "endswith", "re.search", "re.match"]
    assert [lines, file_name].count(None) == 1
    if (lines is None):
      lines = open(file_name).read().splitlines()
    result = []
    a = result.append
    if (mode == "=="):
      for l in lines:
        if (l == pattern): a(l)
    elif (mode == "startswith"):
      for l in lines:
        if (l.startswith(pattern)): a(l)
    elif (mode == "endswith"):
      for l in lines:
        if (l.endswith(pattern)): a(l)
    elif (mode == "find"):
      for l in lines:
        if (l.find(pattern) >= 0): a(l)
    elif (mode == "re.search"):
      import re
      for l in lines:
        if (re.search(pattern=pattern, string=l, flags=re_flags) is not None):
          a(l)
    else:
      import re
      for l in lines:
        if (re.match(pattern=pattern, string=l, flags=re_flags) is not None):
          a(l)
    return result
  
  class progress_displayed_as_fraction(object):
  
    def __init__(self, n):
      self.n = n
      self.i = 0
      if self.n == 1: self.advance = lambda: None
      self.advance()
  
    def advance(self):
      if self.i > 0: sys.stdout.write('\r')
      sys.stdout.write("%i / %i" % (self.i, self.n))
      sys.stdout.flush()
      self.i += 1
  
    def done(self):
      if self.n == 1: return
      sys.stdout.write("\n")
      sys.stdout.flush()
  
  
  class progress_bar(progress_displayed_as_fraction):
  
    def advance(self):
      characters = ['|']
      if self.i > 0:
        characters.extend(['=']*(self.i-1))
        characters.append('>')
      characters.extend(' '*(self.n - self.i))
      characters.append('|\r')
      sys.stdout.write(''.join(characters))
      sys.stdout.flush()
      self.i += 1
  
  def format_float_with_standard_uncertainty(value, standard_uncertainty,
                                             minimum=1e-15):
    """
    Formats a float, including the uncertainty in its value.
  
    Parameters
    ----------
    value : float
    standard_uncertainty : float
    minimum : float
  
    Returns
    -------
    str
  
    Examples
    --------
    >>> libtbx.utils.format_float_with_standard_uncertainty(5e-3, 1e-3)
    '0.0050(10)'
    >>> libtbx.utils.format_float_with_standard_uncertainty(5e-3, 1e-6)
    '0.0050000(10)'
    """
    if standard_uncertainty <= minimum: return str(value)
    precision = -int(round2(math.log10(standard_uncertainty)))
    if precision > -1:
      su = standard_uncertainty * math.pow(10, precision)
      if round2(su,1) < 2:
        su *= 10
        precision += 1
      fmt_str = "%%.%if(%%i)" %precision
      return fmt_str %(value, round2(su))
    else:
      precision += 1
      su = int(round2(standard_uncertainty, precision))
      fmt_str = "%.0f(%i)"
      return fmt_str %(round2(value, precision), su)
  
  def random_hex_code(number_of_digits):
    """
    Creates a random string of hex characters.
  
    Parameters
    ----------
    number_of_digits : int
  
    Returns
    -------
    str
    """
    import random
    digits = []
    for i_digit in range(number_of_digits):
      i = random.randrange(16)
      digits.append("0123456789abcdef"[i])
    return "".join(digits)
  
  def get_svn_revision(path=None):
    # adapted from:
    #   http://code.djangoproject.com/browser/django/trunk/django/utils/version.py
    rev = None
    if path is None:
      import libtbx.load_env
      path = op.dirname(libtbx.env.dist_path(module_name="libtbx"))
    entries_path = '%s/.svn/entries' % path
    try:
      entries = open(entries_path, 'r').read()
    except IOError:
      pass
    else:
      # Versions >= 7 of the entries file are flat text.  The first line is
      # the version number. The next set of digits after 'dir' is the revision.
      if re.match(r'(\d+)', entries):
        rev_match = re.search(r'\d+\s+dir\s+(\d+)', entries)
        if rev_match:
          rev = int(rev_match.groups()[0])
    return rev
  
  def get_build_tag(path=None):
    """
    Returns the build tag for libtbx.
  
    Parameters
    ----------
    path : str, optional
  
    Returns
    -------
    str
    """
    tag = None
    if path is None:
      import libtbx.load_env
      path = op.dirname(libtbx.env.dist_path(module_name="libtbx"))
    tag_file_path = "%s/TAG" %path
    if op.exists(tag_file_path):
      tag = open(tag_file_path).readline().strip()
    return tag
  
  def getcwd_safe():
    """
    Returns the current working directory, raising Sorry if it has been deleted or
    unmounted.
  
    Returns
    -------
    str
  
    Raises
    ------
    Sorry
        If the current working directory has been deleted or unmounted.
    """
    try :
      cwd = os.getcwd()
    except OSError as e :
      if (e.errno == 2):
        raise Sorry("Could not determine the current working directory because "+
          "it has been deleted or unmounted.")
      else :
        raise e
    return cwd
  
  def getcwd_or_default(default=None):
    """
    Returns the current working directory or default if it cannot be found.
  
    Parameters
    ----------
    default : str, optional
  
    Returns
    -------
    str
    """
    if (default is None):
      if (os.name == "nt"):
        home_drive = os.environ.get("HOMEDRIVE", "C:")
        home_dir = os.environ.get("HOMEPATH", "\\")
        default = home_drive + home_dir
      else :
        default = os.environ.get("HOME", "/")
    try :
      cwd = os.getcwd()
    except OSError as e:
      if (e.errno == 2):
        cwd = default
      else :
        raise e
    return cwd
  
  def create_run_directory(prefix, default_directory_number=None):
    """
    Create a program output directory using sequential numbering, picking the
    highest run ID.  In other words, if the prefix is 'Refine' and the current
    directory contains subdirectories named Refine_2 and Refine_9, the new
    directory will be Refine_10.
    """
    dir_number = default_directory_number
    if (dir_number is None):
      dir_ids = []
      for file_name in os.listdir(os.getcwd()):
        if (os.path.isdir(file_name)) and (file_name.startswith(prefix)):
          dir_id = file_name.split("_")[-1]
          if (dir_id.isdigit()):
            dir_ids.append(int(dir_id))
      if (len(dir_ids) > 0):
        dir_number = max(max(dir_ids) + 1, 1)
      else :
        dir_number = 1
    dir_name = prefix + "_" + str(dir_number)
    if (os.path.isdir(dir_name)):
      raise OSError("The directory %s already exists."%os.path.abspath(dir_name))
    else :
      os.makedirs(dir_name)
    return os.path.abspath(dir_name)
  
  class tmp_dir_wrapper(object):
    """
    Convenience methods for running in a (presumably empty) temporary directory
    and copying all files to another directory.  Can be used whether or not the
    temporary directory is actually defined; if None, no action will be taken.
    Otherwise, both tmp_dir and dest_dir (default is current directory) must be
    existing paths.
    """
    def __init__(self, tmp_dir, dest_dir=None, out=sys.stdout):
      if (dest_dir is None):
        dest_dir = os.getcwd()
      self.tmp_dir = tmp_dir
      self.dest_dir = dest_dir
      if (tmp_dir is None):
        pass
      elif (not os.path.isdir(tmp_dir)):
        raise Sorry("The temporary directory %s does not exist." % tmp_dir)
      else :
        if (not os.path.isdir(dest_dir)):
          raise Sorry("The destination directory %s does not exist." % dest_dir)
        print("Changing working directory to %s" % tmp_dir, file=out)
        print("Ultimate destination is %s" % dest_dir, file=out)
        os.chdir(tmp_dir)
  
    def transfer_files(self, out=sys.stdout):
      if (self.tmp_dir is None) : return False
      assert os.path.isdir(self.dest_dir)
      files = os.listdir(self.tmp_dir)
      print("Copying all output files to %s" % self.dest_dir, file=out)
      for file_name in files :
        print("  ... %s" % file_name, file=out)
        shutil.copy(os.path.join(self.tmp_dir, file_name), self.dest_dir)
      print("", file=out)
      return True
  
  def show_development_warning(out=sys.stdout):
    """
    Shows a warning when running an experimental program.
  
    Parameters
    ----------
    out : file, optional
    """
    print("""
    !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
    !!                  WARNING - EXPERIMENTAL PROGRAM                        !!
    !!                                                                        !!
    !! This program is still in development - some functionality may be       !!
    !! missing and/or untested.  Use at your own risk!  For bug reports, etc. !!
    !! email bugs@phenix-online.org.                                          !!
    !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
  """, file=out)
  
  def check_if_output_directory_exists(file_name=None, dir_name=None):
    if (file_name is not None):
      assert (dir_name is None)
      dir_name = os.path.dirname(file_name)
    if (dir_name == "") : return
    if (dir_name is None):
      raise Sorry("No output directory specified.")
    if (not op.isdir(dir_name)):
      raise Sorry(("The specified output directory (%s) does not exist or "+
        "is not a directory.") % dir_name)
    else :
      # XXX writing to Dropbox folders is generally not a good idea
      head, tail = os.path.split(dir_name)
      while tail != "" :
        if (tail == "Dropbox"):
          warnings.warn("You are directing output to a Dropbox directory.  "+
            "Please note that this is not guaranteed to work in all cases; "+
            "use at your own risk.", UserWarning)
        head, tail = os.path.split(head)
  
  def concatenate_python_script(out, file_name):
    """
    Insert a Python script into an existing file, removing any __future__
    import to prevent syntax errors.  (This could be dangerous in most contexts
    but is required for some of our Coot-related scripts to work.)
    """
    with open(file_name, "r") as f:
      data = f.read()
    print("", file=out)
    print("#--- script copied from %s" % os.path.basename(file_name), file=out)
    for line in data.splitlines():
      if line.startswith("from __future__"):
        continue
      else :
        print(line, file=out)
    print("#--- end", file=out)
    print("", file=out)
  
  def greek_time(secs):
    """
    Converts seconds into its closest form in greek units.
  
    Parameters
    ----------
    secs : float
  
    Returns
    -------
    float
    str
  
    Examples
    --------
    >>> libtbx.utils.greek_time(1e-3)
    (1, "milli")
    >>> libtbx.utils.greek_time(1e-6)
    (1, "micro")
    """
    for greek in ["","milli", "micro", "nano"]:
      if secs>1:
        break
      secs*=1000
    return secs, greek
  
  ###########################
  # URL retrieval functions #
  ###########################
  
  libtbx_urllib_proxy = None
  
  def install_urllib_http_proxy(server, port=80, user=None, password=None):
    global libtbx_urllib_proxy
    from six.moves import urllib
    if (user is None):
      proxy = urllib.request.ProxyHandler({'http': '%s:%d' % (server, port) })
      opener = urllib.request.build_opener(proxy)
    else :
      proxy = urllib.request.ProxyHandler({
        'http': 'http://%s:%s@%s:%s' % (user, password, server, port),
      })
      auth = urllib.request.HTTPBasicAuthHandler()
      opener = urllib.request.build_opener(proxy, auth, urllib.request.HTTPHandler)
    libtbx_urllib_proxy = proxy
    urllib.request.install_opener(opener)
    print("Installed urllib proxy at %s:%d" % (server, port))
    return proxy
  
  def urlopen(*args, **kwds):
    """
    Substitute for urllib.request.urlopen, with automatic HTTP proxy configuration
    if specific environment variables are defined.
    """
    if ("CCTBX_HTTP_PROXY" in os.environ) and (libtbx_urllib_proxy is None):
      server = os.environ["CCTBX_HTTP_PROXY_SERVER"]
      port = os.environ.get("CCTBX_HTTP_PROXY_PORT", 80)
      user = os.environ.get("CCTBX_HTTP_PROXY_USER", None)
      passwd = os.environ.get("CCTBX_HTTP_PROXY_PASSWORD", None)
      if (user is not None) and (password is None):
        raise Sorry("You have defined a user name for the HTTP proxy, but "+
          "no password was specified.  Please set the environment variable "+
          "CCTBX_HTTP_PROXY_PASSWORD.")
      install_urllib_http_proxy(
        server=server,
        port=port,
        user=user,
        password=password)
    from six.moves import urllib
    return urllib.request.urlopen(*args, **kwds)
  
  def retrieve_unless_exists(url, filename, digests=None):
    """ Download the file at the given url to the given local filename,
        unless that file is already here. This is asserted by computed a digest
        and comparing it to the entries in the file at the url digests. The latter
        shall have the following format:
  
        some-file.txt      xxxxxxxxxx
        another-file.txt   yyyyyyyyyy
  
        If url is ..../some-file.txt, then the expected digest is xxxxxxxxxx.
  
        If digests is None, then the url of the digest file is expected to be
        named 'digests.txt' and to be next to the downloaded file.
    """
    import urlparse
    from os import path
    from six.moves import urllib
    if digests is None:
      digests = urlparse.urljoin(url, 'digests.txt')
    digest_of = dict(tuple(li.split()) for li in urllib.request.urlopen(digests))
    src_name = os.path.basename(urlparse.urlparse(url).path)
    if (not os.path.isfile(filename)
        or md5_hexdigest(filename) != digest_of[src_name]):
      urllib.request.urlretrieve(url, filename)
      return "Downloaded"
    else:
      return "Cached"
  
  
  
  
  class download_progress(object):
    """
    Simple proxy for displaying download status - here with methods for
    writing to the console, but can be subclassed and used for graphical display.
    """
    def __init__(self, log=None, n_kb_total=None):
      if (log is None):
        log = null_out()
      self.log = log
      self.n_kb_total = n_kb_total
      self.n_kb_elapsed = 0
  
    def set_total_size(self, n_kb_total):
      """
      Updates the total number of bytes to download and resets the number of bytes
      downloaded.
  
      Parameters
      ----------
      n_kb_total : int
          Total size of download, in kilobytes.
      """
      self.n_kb_total = n_kb_total
      self.n_kb_elapsed = 0
  
    def increment(self, n_kb):
      """
      Increments the number of bytes downloaded.
  
      Parameters
      ----------
      n_kb : int
  
      Returns
      -------
      bool
      """
      assert (self.n_kb_total is not None)
      self.n_kb_elapsed += n_kb
      return self.show_progress()
  
    def show_progress(self):
      """
      Prints the number of bytes downloaded out of the total.
  
      Returns
      -------
      bool
      """
      self.log.write("\r%d/%d KB downloaded" % (self.n_kb_elapsed,
        self.n_kb_total))
      self.log.flush()
      return True
  
    def percent_finished(self):
      """
      Calculates the percent completion of download.
  
      Returns
      -------
      float
      """
      assert (self.n_kb_total is not None)
      return 100 * min(1.0, self.n_kb_elapsed / self.n_kb_total)
  
    def complete(self):
      """
      Prints a final message indicating download completion.
      """
      self.log.write("\rDownload complete")
  
    def run_continuously(self):
      """
      Placeholder for cases where the download is not being run asynchronously.
      """
      pass
  
  class download_target(object):
    """
    Flexible callable object for retrieving a file from a URL, with optional
    HTTPS authentication.  Designed to be runnable in a separate thread with
    graphical progress update.
  
    Note that in some circumstances SSL support may be missing from the socket
    module, in which case we use 'curl' to download securely.  (This will not
    work on Windows, obviously.)
    """
    def __init__(self,
        url,
        file_name,
        use_curl=None, # SSL only
        user=None, # SSL only
        password=None, # SSL only
        base_url=None) :  # SSL only
      self.url = url
      self.file_name = file_name
      self.use_curl = use_curl
      self.user = user
      self.password = password
      self.base_url = base_url
      if (not None in [self.user, self.password]):
        assert (self.base_url is not None)
        import socket
        if ((not self.use_curl) and (hasattr(socket, "ssl")) and
            (hasattr(socket.ssl, "__call__"))):
          self.use_curl = False
        else :
          self.use_curl = True
  
    def __call__(self, log=None, progress_meter=None):
      if (log is None):
        log = null_out()
      if (progress_meter is None):
        progress_meter = download_progress(log=log)
      from libtbx import easy_run
      from six.moves import urllib
      file_name = self.file_name # return value
      if (not self.use_curl):
        if (not None in [self.user, self.password]):
          passman = urllib.request.HTTPPasswordMgrWithDefaultRealm()
          passman.add_password(None, self.base_url, self.user, self.password)
          authhandler = urllib.request.HTTPBasicAuthHandler(passman)
          opener = urllib.request.build_opener(authhandler)
          urllib.request.install_opener(opener)
        req = urllib.request.urlopen(self.url)
        info = req.info()
        n_kb_total = int(info['Content-length']) / 1024
        progress_meter.set_total_size(n_kb_total)
        # TODO adjust chunk size automatically based on download speed
        n_kb_chunk = getattr(self, "n_kb_chunk", 512)
        chunksize = n_kb_chunk * 1024
        fp = open(self.file_name, 'wb')
        while True:
          chunk = req.read(chunksize)
          if not chunk: break
          if not progress_meter.increment(n_kb_chunk):
            file_name = None
            break
          fp.write(chunk)
        fp.close()
        progress_meter.complete()
      else :
        progress_meter.run_continuously()
        if (not None in [self.user, self.password]):
          curl_args = "--user %s:%s" % (self.user, self.password)
        rc = easy_run.call("curl %s \"%s\" -o %s" % (curl_args, self.url,
          self.file_name))
        progress_meter.complete()
        if (rc != 0):
          raise RuntimeError("curl exited with code %d" % rc)
      if (file_name is None):
        return None
      return op.abspath(self.file_name)
  
  def cmd_exists(cmd):
    """
    Test whether a command is available by checking the return code from
    subprocess.call
    """
    import subprocess
    return subprocess.call("type " + cmd,
                           shell=True,
                           stdout=subprocess.PIPE,
                           stderr=subprocess.PIPE) == 0
  
  def remove_path(path_name):
    """
    Bypasses trash and deletes file or directory immediately.
    """
    try:
      if op.isdir(path_name):
        shutil.rmtree(path_name)
      elif op.isfile(path_name):
        os.remove(path_name)
      else:
        raise Sorry('%s is not a file nor a directory.' % path_name)
    except OSError:
      raise Sorry('Unable to delete %s.' % path_name)
  
  def try_send_to_trash(path_name, delete_if_not_available=False,
                         delete_immediately=False):
    """
    Wrapper for deleting a path
    """
    if (delete_immediately):
      remove_path(path_name)
    else:
      try :
        import send2trash
      except ImportError :
        if delete_if_not_available :
          warnings.warn("send2trash not available; will delete path instead.",
            ImportWarning)
          remove_path(path_name)
        else :
          raise Sorry("This function not supported because the required module is "+
            "not installed.")
      else :
        send2trash.send2trash(path_name)
  
  if sys.hexversion >= 0x03000000:
    unicode = str
  
  def to_unicode(text, codec=None, errors='replace'):
    '''
    Function for handling text when it is first encountered
  
    Changes bytestring type (str or bytes in Python 2, bytes in Python 3) to
    text string type (unicode in Python 2, str in Python 3)
  
    The input is returned unmodified if it is already a text string
    Will convert other types (e.g. int, float) to text string
    None is returned as None, not as u'None'
  
    For Linux/OS X, the default filesystem encoding is utf8.
    For Windows, the default filesystem encoding is mbcs.
    This is important for handling files with basic Python functions
    With the wrong encoding, the filesystem will not recognize the file path
    import sys; sys.getfilesystemencoding()
    '''
  
    if (codec is None):
      codec = 'utf8'
      if (sys.platform == 'win32'):
        codec = 'mbcs'
  
    if (isinstance(text, unicode)):
      return text
    elif (isinstance(text, bytes)):
      new_text = text
      try:
        new_text = text.decode(codec, errors)
      except UnicodeDecodeError: # in case errors='strict'
        raise Sorry('Unable to decode text with %s' % codec)
      finally:
        return new_text
    elif (text is not None):
      return unicode(text)
    else:
      return None
  
  def to_bytes(text, codec=None, errors='replace'):
    '''
    Function for handling text when it is passed to cctbx functions that expect
    bytestrings
  
    Changes text string type (unicode in Python 2, str in Python 3) to
    bytestring type (str or bytes in Python 2, bytes in Python 3)
  
    The input is returned unmodified if it is already a bytestring
    Will convert other types (e.g. int, float) to bytestring
    None is returned as None, not as 'None'
  
    For Linux/OS X, the default filesystem encoding is utf8.
    For Windows, the default filesystem encoding is mbcs
    This is important for handling files with basic Python functions.
    With the wrong encoding, the filesystem will not recognize the file path
    import sys; sys.getfilesystemencoding()
    '''
  
    if (codec is None):
      codec = 'utf8'
      if (sys.platform == 'win32'):
        codec = 'mbcs'
  
    if (isinstance(text, bytes)):
      return text
    elif (isinstance(text, unicode)):
      new_text = text
      try:
        new_text = text.encode(codec, errors)
      except UnicodeEncodeError: # in case errors='strict'
        raise Sorry('Unable to encode text with %s' % codec)
      finally:
        return new_text
    elif (text is not None):
      return bytes(text)
    else:
      return None
  
  def to_str(text, codec=None, errors='replace'):
    '''
    Function for handling text in a way compatible with both Python 2 and 3 and
    with Boost.
  
    Boost defines boost::python::str as Unicode text in Python 3 (str type) and
    as text/byte string in Python 2 (also str type, or bytes type).
  
    This function just calls to_unicode and to_bytes to return the appropriate
    type depending on the Python version
    '''
    if sys.hexversion >= 0x03000000:
      return to_unicode(text, codec, errors)
    else:
      return to_bytes(text, codec, errors)
  
  def guess_total_memory():
    '''
    Use psutil to return the total memory on a system in bytes.
    '''
    import psutil
    return psutil.virtual_memory().total
  
  MANGLE_LEN = 256 # magic constant from compile.c
  def mangle(name, klass):
    '''
    Since the compiler module is removed in Python 3, this is a copy of the
    mangle function from compiler.misc.
  
    This function is used for name mangling in libtbx/__init__.py for the
    slots_getstate_setstate class.
    '''
    if not name.startswith('__'):
      return name
    if len(name) + 2 >= MANGLE_LEN:
      return name
    if name.endswith('__'):
      return name
    try:
      i = 0
      while klass[i] == '_':
        i = i + 1
    except IndexError:
      return name
    klass = klass[i:]
  
    tlen = len(klass) + len(name)
    if tlen > MANGLE_LEN:
      klass = klass[:MANGLE_LEN-tlen]
  
    return "_%s%s" % (klass, name)
  
  def path_is_git_lfs_pointer(path):
    '''
    Test if a file is a git lfs pointer. See
    https://github.com/git-lfs/git-lfs/blob/master/docs/spec.md
    '''
    with open(path, 'rb') as f:
      return f.read(12) == b'version http'
  
  def check_git_lfs_pointer_is_loaded(path):
    '''
    Prints a skip if the lfs pointer hasn't been loaded
    '''
    test = path_is_git_lfs_pointer(path)
    if test:
      print ("""
  Skipping. %s hasn't been loaded from git
  Run these commands to load the data:
  cd <repository name>
  git lfs install --local
  git lsf pull
  """%path)
    return not test
  