docs: def upload_magic(concat=False, dir_path='.', dmodel=None, vocab="", contribution=None,
                   input_dir_path=""):
      """
      Finds all magic files in a given directory, and compiles them into an
      upload.txt file which can be uploaded into the MagIC database.
      Parameters
      ----------
      concat : boolean where True means do concatenate to upload.txt file in dir_path,
          False means write a new file (default is False)
      dir_path : string for input/output directory (default ".")
      dmodel : pmagpy data_model.DataModel object,
          if not provided will be created (default None)
      vocab : pmagpy controlled_vocabularies3.Vocabulary object,
          if not provided will be created (default None)
      contribution : pmagpy contribution_builder.Contribution object, if not provided will be created
          in directory (default None)
      input_dir_path : str, default ""
          path for intput files if different from output dir_path (default is same)
  
      Returns
      ----------
      tuple of either: (False, error_message, errors, all_failing_items)
      if there was a problem creating/validating the upload file
      or: (filename, '', None, None) if the file creation was fully successful.
      """
      input_dir_path, dir_path = pmag.fix_directories(input_dir_path, dir_path)
      locations = []
      concat = int(concat)
      dtypes = ["locations", "samples", "specimens", "sites", "ages", "measurements",
                "criteria", "contribution", "images"]
      fnames = [os.path.join(input_dir_path, dtype + ".txt") for dtype in dtypes]
      file_names = [fname for fname in fnames if os.path.exists(fname)]
      error_fnames = [dtype + "_errors.txt" for dtype in dtypes]
      error_full_fnames = [os.path.join(
          dir_path, fname) for fname in error_fnames if os.path.exists(os.path.join(dir_path, fname))]
      print('-I- Removing old error files from {}: {}'.format(dir_path,
                                                              ", ".join(error_fnames)))
      for error in error_full_fnames:
          os.remove(error)
      if isinstance(contribution, cb.Contribution):
          # if contribution object provided, use it
          con = contribution
          for table_name in con.tables:
              con.tables[table_name].write_magic_file()
      elif file_names:
          # otherwise create a new Contribution in dir_path
          con = Contribution(input_dir_path, vocabulary=vocab)
      else:
          # if no contribution is provided and no contribution could be created,
          # you are out of luck
          print("-W- No 3.0 files found in your directory: {}, upload file not created".format(input_dir_path))
          return False, "no 3.0 files found, upload file not created", None, None
  
      # if the contribution has no tables, you can't make an upload file
      if not con.tables.keys():
          print("-W- No tables found in your contribution in directory {}, file not created".format(input_dir_path))
          return False, "-W- No tables found in your contribution, file not created", None, None
  
      con.propagate_cols(['core_depth', 'composite_depth'],
                         'sites', 'samples', down=False)
  
      # take out any extra added columns
      # con.remove_non_magic_cols()
  
      # begin the upload process
      up = os.path.join(dir_path, "upload.txt")
      if os.path.exists(up):
          os.remove(up)
      RmKeys = ('citation_label', 'compilation', 'calculation_type', 'average_n_lines', 'average_n_planes',
                'specimen_grade', 'site_vgp_lat', 'site_vgp_lon', 'direction_type', 'specimen_Z',
                'magic_instrument_codes', 'cooling_rate_corr', 'cooling_rate_mcd', 'anisotropy_atrm_alt',
                'anisotropy_apar_perc', 'anisotropy_F', 'anisotropy_F_crit', 'specimen_scat',
                'specimen_gmax', 'specimen_frac', 'site_vadm', 'site_lon', 'site_vdm', 'site_lat',
                'measurement_chi', 'specimen_k_prime', 'specimen_k_prime_sse', 'external_database_names',
                'external_database_ids', 'Further Notes', 'Typology', 'Notes (Year/Area/Locus/Level)',
                'Site', 'Object Number', 'version', 'site_definition')
      #print("-I- Removing: ", RmKeys)
      extra_RmKeys = {'measurements': ['sample', 'site', 'location'],
                      'specimens': ['site', 'location', 'age', 'age_unit', 'age_high',
                                    'age_low', 'age_sigma', 'specimen_core_depth'],
                      'samples': ['location', 'age', 'age_unit', 'age_high', 'age_low',
                                  'age_sigma', 'core_depth', 'composite_depth'],
                      'sites': ['texture', 'azimuth', 'azimuth_dec_correction', 'dip',
                                'orientation_quality', 'sample_alternatives', 'timestamp'],
                      'ages': ['level']}
  
      failing = []
      all_failing_items = {}
      if not dmodel:
          dmodel = data_model.DataModel()
      last_file_type = sorted(con.tables.keys())[-1]
      for file_type in sorted(con.tables.keys()):
          container = con.tables[file_type]
          # format all float values to have correct number of decimals
          container.all_to_str()
          # make sure all nans and Nones are changed to ''
          container.df.fillna('')
          df = container.df
          if len(df):
              print("-I- {} file successfully read in".format(file_type))
      # make some adjustments to clean up data
              # drop non MagIC keys
              DropKeys = list(RmKeys) + extra_RmKeys.get(file_type, [])
              DropKeys = set(DropKeys).intersection(df.columns)
              if DropKeys:
                  print(
                      '-I- dropping these columns: {} from the {} table'.format(', '.join(DropKeys), file_type))
                  df.drop(DropKeys, axis=1, inplace=True)
              container.df = df
              unrecognized_cols = container.get_non_magic_cols()
              if unrecognized_cols:
                  print('-W- {} table still has some unrecognized columns: {}'.format(file_type.title(),
                                                                                      ", ".join(unrecognized_cols)))
              # make sure int_b_beta is positive
              if 'int_b_beta' in df.columns:
                  # get rid of empty strings
                  df = df.replace(r'\s+( +\.)|#', np.nan,
                                  regex=True).replace('', np.nan)
                  try:
                      df['int_b_beta'] = df['int_b_beta'].astype(
                          float).apply(abs)
                  except ValueError:
                      "-W- Non numeric values found in int_b_beta column.\n   Could not apply absolute value."
              # make all declinations/azimuths/longitudes in range 0=>360.
              relevant_cols = val_up3.get_degree_cols(df)
              for col in relevant_cols:
                  df[col] = df[col].apply(pmag.adjust_val_to_360)
              # get list of location names
              if file_type == 'locations':
                  locations = sorted(df['location'].unique())
              # LJ: need to deal with this
              # use only highest priority orientation -- not sure how this works
              elif file_type == 'samples':
                  # orient,az_type=pmag.get_orient(Data,rec['sample'])
                  pass
              # include only specimen records with samples
              elif file_type == 'specimens':
                  df = df[df['sample'].notnull()]
                  if 'samples' in con.tables:
                      samp_df = con.tables['samples'].df
                      df = df[df['sample'].isin(samp_df.index.unique())]
              # include only measurements with specmiens
              elif file_type == 'measurements':
                  df = df[df['specimen'].notnull()]
                  if 'specimens' in con.tables:
                      spec_df = con.tables['specimens'].df
                      df = df[df['specimen'].isin(spec_df.index.unique())]
      # run validations
              res = val_up3.validate_table(
                  con, file_type, output_dir=dir_path)  # , verbose=True)
              if res:
                  dtype, bad_rows, bad_cols, missing_cols, missing_groups, failing_items = res
                  if dtype not in all_failing_items:
                      all_failing_items[dtype] = {}
                  all_failing_items[dtype]["rows"] = failing_items
                  all_failing_items[dtype]["missing_columns"] = missing_cols
                  all_failing_items[dtype]["missing_groups"] = missing_groups
                  failing.append(dtype)
      # write out the data
              if len(df):
                  container.write_magic_file(up, append=True, multi_type=True)
      # write out the file separator
              if last_file_type != file_type:
                  f = open(up, 'a')
                  f.write('>>>>>>>>>>\n')
                  f.close()
                  print("-I-", file_type, 'written to ', up)
              else:  # last file, no newline at end of file
                  #f = open(up, 'a')
                  # f.write('>>>>>>>>>>')
                  # f.close()
                  print("-I-", file_type, 'written to ', up)
      # if there was no understandable data
          else:
              print(file_type, 'is bad or non-existent - skipping ')
      # add to existing file
      if concat:
          f = open(up, 'a')
          f.write('>>>>>>>>>>\n')
          f.close()
  
      if not os.path.isfile(up):
          print("no data found, upload file not created")
          return False, "no data found, upload file not created", None, None
  
      # rename upload.txt according to location + timestamp
      format_string = "%d.%b.%Y"
      if locations:
          locs = set(locations)
          locs = sorted(locs)[:3]
          #location = locations[0].replace(' ', '_')
          try:
              locs = [loc.replace(' ', '-') for loc in locs]
          except AttributeError:
              locs = ["unknown_location"]
          location = "_".join(locs)
          new_up = location + '_' + time.strftime(format_string) + '.txt'
      else:
          new_up = 'unknown_location_' + time.strftime(format_string) + '.txt'
  
      new_up = os.path.join(dir_path, new_up)
      if os.path.isfile(new_up):
          fname, extension = os.path.splitext(new_up)
          for i in range(1, 100):
              if os.path.isfile(fname + "_" + str(i) + extension):
                  continue
              else:
                  new_up = fname + "_" + str(i) + extension
                  break
      if not up:
          print("-W- Could not create an upload file")
          return False, "Could not create an upload file", None, None
      os.rename(up, new_up)
      print("Finished preparing upload file: {} ".format(new_up))
      if failing:
          print("-W- These tables have errors: {}".format(", ".join(failing)))
          print("-W- validation of upload file has failed.\nYou can still upload {} to MagIC,\nbut you will need to fix the above errors before your contribution can be activated.".format(new_up))
          return False, "Validation of your upload file has failed.\nYou can still upload {} to MagIC,\nbut you will need to fix the above errors before your contribution can be activated.".format(new_up), failing, all_failing_items
      else:
          print("-I- Your file has passed validation.  You should be able to upload it to the MagIC database without trouble!")
      return new_up, '', None, None