docs: def upload_magic2(concat=0, dir_path='.', data_model=None):
      """
      Finds all magic files in a given directory, and compiles them into an
      upload.txt file which can be uploaded into the MagIC database. Returns a
      tuple of either: (False, error_message, errors) if there was a problem
      creating/validating the upload file or: (filename, '', None) if the upload
      was fully successful.
      """
      SpecDone = []
      locations = []
      concat = int(concat)
      files_list = ["er_expeditions.txt", "er_locations.txt", "er_samples.txt", "er_specimens.txt", "er_sites.txt", "er_ages.txt", "er_citations.txt", "er_mailinglist.txt", "magic_measurements.txt",
                    "rmag_hysteresis.txt", "rmag_anisotropy.txt", "rmag_remanence.txt", "rmag_results.txt", "pmag_specimens.txt", "pmag_samples.txt", "pmag_sites.txt", "pmag_results.txt", "pmag_criteria.txt", "magic_instruments.txt"]
      file_names = [os.path.join(dir_path, f) for f in files_list]
  
      # begin the upload process
      up = os.path.join(dir_path, "upload.txt")
      if os.path.exists(up):
          os.remove(up)
      RmKeys = ['citation_label', 'compilation', 'calculation_type', 'average_n_lines', 'average_n_planes',
                'specimen_grade', 'site_vgp_lat', 'site_vgp_lon', 'direction_type', 'specimen_Z',
                'magic_instrument_codes', 'cooling_rate_corr', 'cooling_rate_mcd', 'anisotropy_atrm_alt',
                'anisotropy_apar_perc', 'anisotropy_F', 'anisotropy_F_crit', 'specimen_scat',
                'specimen_gmax', 'specimen_frac', 'site_vadm', 'site_lon', 'site_vdm', 'site_lat',
                'measurement_chi', 'specimen_k_prime', 'specimen_k_prime_sse', 'external_database_names',
                'external_database_ids', 'Further Notes', 'Typology', 'Notes (Year/Area/Locus/Level)',
                'Site', 'Object Number', 'dir_n_specimens']
      print("-I- Removing: ", RmKeys)
      CheckDec = ['_dec', '_lon', '_azimuth', 'dip_direction']
      CheckSign = ['specimen_b_beta']
      last = file_names[-1]
      methods, first_file = [], 1
      for File in file_names:
          # read in the data
          Data, file_type = pmag.magic_read(File)
          if (file_type != "bad_file") and (file_type != "empty_file"):
              print("-I- file", File, " successfully read in")
              if len(RmKeys) > 0:
                  for rec in Data:
                      # remove unwanted keys
                      for key in RmKeys:
                          if key == 'specimen_Z' and key in list(rec.keys()):
                              # change  # change this to lower case
                              rec[key] = 'specimen_z'
                          if key in list(rec.keys()):
                              del rec[key]  # get rid of unwanted keys
                      # make sure b_beta is positive
                      # ignore blanks
                      if 'specimen_b_beta' in list(rec.keys()) and rec['specimen_b_beta'] != "":
                          if float(rec['specimen_b_beta']) < 0:
                              # make sure value is positive
                              rec['specimen_b_beta'] = str(
                                  -float(rec['specimen_b_beta']))
                              print('-I- adjusted to positive: ',
                                    'specimen_b_beta', rec['specimen_b_beta'])
                      # make all declinations/azimuths/longitudes in range
                      # 0=>360.
                      rec = pmag.adjust_all_to_360(rec)
              if file_type == 'er_locations':
                  for rec in Data:
                      locations.append(rec['er_location_name'])
              if file_type in ['pmag_samples', 'pmag_sites', 'pmag_specimens']:
                  # if there is NO pmag data for specimens (samples/sites),
                  # do not try to write it to file
                  # (this causes validation errors, elsewise)
                  ignore = True
                  for rec in Data:
                      if ignore == False:
                          break
                      keys = list(rec.keys())
                      exclude_keys = ['er_citation_names', 'er_site_name', 'er_sample_name',
                                      'er_location_name', 'er_specimen_names', 'er_sample_names']
                      for key in exclude_keys:
                          if key in keys:
                              keys.remove(key)
                      for key in keys:
                          if rec[key]:
                              ignore = False
                              break
                  if ignore:
                      continue
  
              if file_type == 'er_samples':  # check to only upload top priority orientation record!
                  NewSamps, Done = [], []
                  for rec in Data:
                      if rec['er_sample_name'] not in Done:
                          orient, az_type = pmag.get_orient(
                              Data, rec['er_sample_name'])
                          NewSamps.append(orient)
                          Done.append(rec['er_sample_name'])
                  Data = NewSamps
                  print(
                      'only highest priority orientation record from er_samples.txt read in ')
              if file_type == 'er_specimens':  # only specimens that have sample names
                  NewData, SpecDone = [], []
                  for rec in Data:
                      if rec['er_sample_name'] in Done:
                          NewData.append(rec)
                          SpecDone.append(rec['er_specimen_name'])
                      else:
                          print('no valid sample record found for: ')
                          print(rec)
                  Data = NewData
                  # print 'only measurements that have specimen/sample info'
              if file_type == 'magic_measurements':  # only measurements that have specimen names
                  no_specs = []
                  NewData = []
                  for rec in Data:
                      if rec['er_specimen_name'] in SpecDone:
                          NewData.append(rec)
                      else:
                          print('no valid specimen record found for: ')
                          print(rec)
                          no_specs.append(rec)
                  # print set([record['er_specimen_name'] for record in
                  # no_specs])
                  Data = NewData
      # write out the data
              if len(Data) > 0:
                  if first_file == 1:
                      keystring = pmag.first_rec(up, Data[0], file_type)
                      first_file = 0
                  else:
                      keystring = pmag.first_up(up, Data[0], file_type)
                  for rec in Data:
                      # collect the method codes
                      if "magic_method_codes" in list(rec.keys()):
                          meths = rec["magic_method_codes"].split(':')
                          for meth in meths:
                              if meth.strip() not in methods:
                                  if meth.strip() != "LP-DIR-":
                                      methods.append(meth.strip())
                      try:
                          pmag.putout(up, keystring, rec)
                      except IOError:
                          print('-W- File input error: slowing down')
                          time.sleep(1)
                          pmag.putout(up, keystring, rec)
  
      # write out the file separator
              f = open(up, 'a')
              f.write('>>>>>>>>>>\n')
              f.close()
              print(file_type, 'written to ', up)
          else:
              print('File:', File)
              print(file_type, 'is bad or non-existent - skipping ')
  
      # write out the methods table
      first_rec, MethRec = 1, {}
      for meth in methods:
          MethRec["magic_method_code"] = meth
          if first_rec == 1:
              meth_keys = pmag.first_up(up, MethRec, "magic_methods")
          first_rec = 0
          try:
              pmag.putout(up, meth_keys, MethRec)
          except IOError:
              print('-W- File input error: slowing down')
              time.sleep(1)
              pmag.putout(up, meth_keys, MethRec)
      if concat == 1:
          f = open(up, 'a')
          f.write('>>>>>>>>>>\n')
          f.close()
  
      if os.path.isfile(up):
          from . import validate_upload2 as validate_upload
          validated = False
          validated, errors = validate_upload.read_upload(up, data_model)
  
      else:
          print("no data found, upload file not created")
          return False, "no data found, upload file not created", None
  
      # rename upload.txt according to location + timestamp
      format_string = "%d.%b.%Y"
      if locations:
          location = locations[0].replace(' ', '_')
          new_up = location + '_' + time.strftime(format_string) + '.txt'
      else:
          new_up = 'unknown_location_' + time.strftime(format_string) + '.txt'
  
      new_up = os.path.join(dir_path, new_up)
      if os.path.isfile(new_up):
          fname, extension = os.path.splitext(new_up)
          for i in range(1, 100):
              if os.path.isfile(fname + "_" + str(i) + extension):
                  continue
              else:
                  new_up = fname + "_" + str(i) + extension
                  break
      os.rename(up, new_up)
      print("Finished preparing upload file: {} ".format(new_up))
      if not validated:
          print("-W- validation of upload file has failed.\nYou can still upload {} to MagIC,\nbut you will need to fix the above errors before your contribution can be activated.".format(new_up))
          return False, "Validation of your upload file has failed.\nYou can still upload {} to MagIC,\nbut you will need to fix the above errors before your contribution can be activated.".format(new_up), errors
      return new_up, '', None