parameters: (directory, portal_instance, logger, relaxed_mode, strict_maf_checks)
return_statement: return (validators_by_type, defined_case_list_fns,
              study_cancer_type, study_id, stable_ids, tags_file_path)
docstring: Parse the meta files in a directory and create data file validators.
  
      Return a tuple of:
          1. a dict listing the data file validator (or None) for each meta file
              by file type,
          2. a dict mapping any case list IDs defined *outside* of the case list
              directory to paths of the files in which they were defined
          3. the cancer type of the study, and
          4. the study id
  
      Possible file types are listed in cbioportal_common.MetaFileTypes.
docstring_summary: Parse the meta files in a directory and create data file validators.
docstring_tokens: Parse the meta files in a directory and create data file validators .
function: def process_metadata_files(directory, portal_instance, logger, relaxed_mode, strict_maf_checks):
  
      """Parse the meta files in a directory and create data file validators.
  
      Return a tuple of:
          1. a dict listing the data file validator (or None) for each meta file
              by file type,
          2. a dict mapping any case list IDs defined *outside* of the case list
              directory to paths of the files in which they were defined
          3. the cancer type of the study, and
          4. the study id
  
      Possible file types are listed in cbioportal_common.MetaFileTypes.
      """
  
      # get filenames for all meta files in the directory
      filenames = [os.path.join(directory, f) for
                   f in sorted(os.listdir(directory)) if
                   re.search(r'(\b|_)meta(\b|[_0-9])', f,
                             flags=re.IGNORECASE) and
                   not f.startswith('ONCOKB_IMPORT_BACKUP') and
                   not f.startswith('.') and
                   not f.endswith('~')]
  
      if len(filenames) == 0:
          logger.critical(
                      'No meta files found in ' + directory +'. Please make sure the directory '\
                      'is the path to the folder containing the files.')
  
      study_id = None
      study_cancer_type = None
      study_data_types = []
      validators_by_type = {}
      case_list_suffix_fns = {}
      stable_ids = []
      global meta_dictionary
      tags_file_path = None
  
      # implemented ref genomes
      reference_genome_map = {
          'hg19': ('human', 'GRCh37', 'hg19'),
          'hg38': ('human', 'GRCh38', 'hg38'),
          'mm10': ('mouse', 'GRCm38', 'mm10')
      }
  
      DISALLOWED_CHARACTERS = r'[^A-Za-z0-9_-]'
      for filename in filenames:
  
          meta_dictionary = cbioportal_common.parse_metadata_file(
              filename, logger, study_id, gene_panel_list=portal_instance.gene_panel_list)
          meta_file_type = meta_dictionary['meta_file_type']
          if meta_file_type is None:
              continue
  
          # check if geneset version is the same in database
          if portal_instance.geneset_version is not None and 'geneset_def_version' in meta_dictionary:
              geneset_def_version = meta_dictionary['geneset_def_version'].strip()
              if (geneset_def_version != portal_instance.geneset_version):
                  logger.error(
                      '`geneset_def_version` is different from the geneset_version in the database',
                      extra={'filename_': filename,
                             'cause': geneset_def_version})
  
          # validate stable_id to be unique (check can be removed once we deprecate this field):
          if 'stable_id' in meta_dictionary:
              stable_id = meta_dictionary['stable_id'].strip()
  
              # Check for non-supported characters in the stable_id
              # Note: this check is needed because when using a wildcard for STABLE_ID
              # in the allowed_file_types.txt the user can specify a custom stable_id in the meta file.
              if stable_id == '' or re.search(DISALLOWED_CHARACTERS, stable_id) != None:
                  logger.error(
                      '`stable_id` is not valid (empty string or contains one or more illegal characters)',
                      extra={'filename_': filename,
                             'cause': stable_id})
  
              if stable_id in stable_ids:
                  # stable id already used in other meta file, give error:
                  logger.error(
                      'stable_id repeated. It should be unique across all files in a study',
                      extra={'filename_': filename,
                             'cause': stable_id})
              else:
                  stable_ids.append(stable_id)
  
              # Save meta values in a study wide dictionary
              study_meta_dictionary[stable_id] = meta_dictionary
  
          # Validate data types
          if 'datatype' in meta_dictionary:
              if meta_dictionary['datatype'].lower() in study_data_types:
                  # For Seg file only one can be loaded
                  if meta_dictionary['datatype'].lower() == "seg":
                      logger.error('datatype SEG is repeated. Only one segmentation file can be loaded.')
              else:
                  study_data_types.append(meta_dictionary['datatype'].lower())
  
          if study_id is None and 'cancer_study_identifier' in meta_dictionary:
              study_id = meta_dictionary['cancer_study_identifier']
          if meta_file_type == cbioportal_common.MetaFileTypes.STUDY:
              if study_cancer_type is not None:
                  logger.error(
                      'Encountered a second meta_study file',
                      extra={'filename_': filename})
              else:
                  study_cancer_type = meta_dictionary['type_of_cancer']
                  if ('add_global_case_list' in meta_dictionary and
                          meta_dictionary['add_global_case_list'].lower() == 'true'):
                      case_list_suffix_fns['all'] = filename
  
              # if reference_genome is specified in the meta file, override the defaults in portal properties
              if 'reference_genome' in meta_dictionary:
                  if meta_dictionary['reference_genome'] not in reference_genome_map:
                      logger.error('Unknown reference genome defined. Should be one of %s' %
                                   list(reference_genome_map.keys()),
                                   extra={
                                       'filename_': filename,
                                       'cause': meta_dictionary['reference_genome'].strip()
                                   })
                  else:
                      genome_info = reference_genome_map[meta_dictionary['reference_genome']]
                      logger.info('Setting reference genome to %s (%s, %s)' % genome_info,
                                  extra={'filename_': filename})
                      portal_instance.species, portal_instance.ncbi_build, portal_instance.reference_genome = genome_info
              else:
                  logger.info('No reference genome specified -- using default (hg19)',
                              extra={'filename_': filename})
  
              # Validate PMID field in meta_study
              if 'pmid' in meta_dictionary:
                  if 'citation' not in meta_dictionary:
                      logger.warning('Citation is required when providing a PubMed ID (pmid)')
                  stripped_pmid_value_len = len(meta_dictionary['pmid'].strip())
                  no_whitespace_pmid_value_len = len(''.join(meta_dictionary['pmid'].split()))
                  if (no_whitespace_pmid_value_len != stripped_pmid_value_len):
                      logger.error('The PMID field in meta_study should not contain any embedded whitespace',
                                   extra={'cause': meta_dictionary['pmid'].strip()})
                  for pmid_entry in [x.strip() for x in meta_dictionary['pmid'].split(',')]:
                      try:
                          value = int(pmid_entry)
                      except ValueError:
                          logger.error('The PMID field in meta_study should be a comma separated list of integers',
                                       extra={'cause': pmid_entry})
  
              # Validate Study Tags file field in meta_study
              if 'tags_file' in meta_dictionary:
                  logger.debug(
                      'Study Tag file found. It will be validated.')
                  tags_file_path = os.path.join(directory, meta_dictionary['tags_file'])
  
          # create a list for the file type in the dict
          if meta_file_type not in validators_by_type:
              validators_by_type[meta_file_type] = []
          # check if data_filename is set AND if data_filename is a supported field according to META_FIELD_MAP:
          if 'data_filename' in meta_dictionary and 'data_filename' in cbioportal_common.META_FIELD_MAP[meta_file_type]:
              validator_class = globals()[VALIDATOR_IDS[meta_file_type]]
              validator = validator_class(directory, meta_dictionary,
                                          portal_instance, logger,
                                          relaxed_mode, strict_maf_checks)
              validators_by_type[meta_file_type].append(validator)
          else:
              validators_by_type[meta_file_type].append(None)
  
      if study_cancer_type is None:
          logger.error(
              'Cancer type needs to be defined for a study. Verify that you have a study file '
              'and have defined the cancer type correctly.')
  
      # prepend the cancer study id to any case list suffixes
      defined_case_list_fns = {}
      if study_id is not None:
          for suffix in case_list_suffix_fns:
              defined_case_list_fns[study_id + '_' + suffix] = \
                  case_list_suffix_fns[suffix]
  
      return (validators_by_type, defined_case_list_fns,
              study_cancer_type, study_id, stable_ids, tags_file_path)
function_tokens: def process_metadata_files ( directory , portal_instance , logger , relaxed_mode , strict_maf_checks ) : # get filenames for all meta files in the directory filenames = [ os . path . join ( directory , f ) for f in sorted ( os . listdir ( directory ) ) if re . search ( r'(\b|_)meta(\b|[_0-9])' , f , flags = re . IGNORECASE ) and not f . startswith ( 'ONCOKB_IMPORT_BACKUP' ) and not f . startswith ( '.' ) and not f . endswith ( '~' ) ] if len ( filenames ) == 0 : logger . critical ( 'No meta files found in ' + directory + '. Please make sure the directory ' 'is the path to the folder containing the files.' ) study_id = None study_cancer_type = None study_data_types = [ ] validators_by_type = { } case_list_suffix_fns = { } stable_ids = [ ] global meta_dictionary tags_file_path = None # implemented ref genomes reference_genome_map = { 'hg19' : ( 'human' , 'GRCh37' , 'hg19' ) , 'hg38' : ( 'human' , 'GRCh38' , 'hg38' ) , 'mm10' : ( 'mouse' , 'GRCm38' , 'mm10' ) } DISALLOWED_CHARACTERS = r'[^A-Za-z0-9_-]' for filename in filenames : meta_dictionary = cbioportal_common . parse_metadata_file ( filename , logger , study_id , gene_panel_list = portal_instance . gene_panel_list ) meta_file_type = meta_dictionary [ 'meta_file_type' ] if meta_file_type is None : continue # check if geneset version is the same in database if portal_instance . geneset_version is not None and 'geneset_def_version' in meta_dictionary : geneset_def_version = meta_dictionary [ 'geneset_def_version' ] . strip ( ) if ( geneset_def_version != portal_instance . geneset_version ) : logger . error ( '`geneset_def_version` is different from the geneset_version in the database' , extra = { 'filename_' : filename , 'cause' : geneset_def_version } ) # validate stable_id to be unique (check can be removed once we deprecate this field): if 'stable_id' in meta_dictionary : stable_id = meta_dictionary [ 'stable_id' ] . strip ( ) # Check for non-supported characters in the stable_id # Note: this check is needed because when using a wildcard for STABLE_ID # in the allowed_file_types.txt the user can specify a custom stable_id in the meta file. if stable_id == '' or re . search ( DISALLOWED_CHARACTERS , stable_id ) != None : logger . error ( '`stable_id` is not valid (empty string or contains one or more illegal characters)' , extra = { 'filename_' : filename , 'cause' : stable_id } ) if stable_id in stable_ids : # stable id already used in other meta file, give error: logger . error ( 'stable_id repeated. It should be unique across all files in a study' , extra = { 'filename_' : filename , 'cause' : stable_id } ) else : stable_ids . append ( stable_id ) # Save meta values in a study wide dictionary study_meta_dictionary [ stable_id ] = meta_dictionary # Validate data types if 'datatype' in meta_dictionary : if meta_dictionary [ 'datatype' ] . lower ( ) in study_data_types : # For Seg file only one can be loaded if meta_dictionary [ 'datatype' ] . lower ( ) == "seg" : logger . error ( 'datatype SEG is repeated. Only one segmentation file can be loaded.' ) else : study_data_types . append ( meta_dictionary [ 'datatype' ] . lower ( ) ) if study_id is None and 'cancer_study_identifier' in meta_dictionary : study_id = meta_dictionary [ 'cancer_study_identifier' ] if meta_file_type == cbioportal_common . MetaFileTypes . STUDY : if study_cancer_type is not None : logger . error ( 'Encountered a second meta_study file' , extra = { 'filename_' : filename } ) else : study_cancer_type = meta_dictionary [ 'type_of_cancer' ] if ( 'add_global_case_list' in meta_dictionary and meta_dictionary [ 'add_global_case_list' ] . lower ( ) == 'true' ) : case_list_suffix_fns [ 'all' ] = filename # if reference_genome is specified in the meta file, override the defaults in portal properties if 'reference_genome' in meta_dictionary : if meta_dictionary [ 'reference_genome' ] not in reference_genome_map : logger . error ( 'Unknown reference genome defined. Should be one of %s' % list ( reference_genome_map . keys ( ) ) , extra = { 'filename_' : filename , 'cause' : meta_dictionary [ 'reference_genome' ] . strip ( ) } ) else : genome_info = reference_genome_map [ meta_dictionary [ 'reference_genome' ] ] logger . info ( 'Setting reference genome to %s (%s, %s)' % genome_info , extra = { 'filename_' : filename } ) portal_instance . species , portal_instance . ncbi_build , portal_instance . reference_genome = genome_info else : logger . info ( 'No reference genome specified -- using default (hg19)' , extra = { 'filename_' : filename } ) # Validate PMID field in meta_study if 'pmid' in meta_dictionary : if 'citation' not in meta_dictionary : logger . warning ( 'Citation is required when providing a PubMed ID (pmid)' ) stripped_pmid_value_len = len ( meta_dictionary [ 'pmid' ] . strip ( ) ) no_whitespace_pmid_value_len = len ( '' . join ( meta_dictionary [ 'pmid' ] . split ( ) ) ) if ( no_whitespace_pmid_value_len != stripped_pmid_value_len ) : logger . error ( 'The PMID field in meta_study should not contain any embedded whitespace' , extra = { 'cause' : meta_dictionary [ 'pmid' ] . strip ( ) } ) for pmid_entry in [ x . strip ( ) for x in meta_dictionary [ 'pmid' ] . split ( ',' ) ] : try : value = int ( pmid_entry ) except ValueError : logger . error ( 'The PMID field in meta_study should be a comma separated list of integers' , extra = { 'cause' : pmid_entry } ) # Validate Study Tags file field in meta_study if 'tags_file' in meta_dictionary : logger . debug ( 'Study Tag file found. It will be validated.' ) tags_file_path = os . path . join ( directory , meta_dictionary [ 'tags_file' ] ) # create a list for the file type in the dict if meta_file_type not in validators_by_type : validators_by_type [ meta_file_type ] = [ ] # check if data_filename is set AND if data_filename is a supported field according to META_FIELD_MAP: if 'data_filename' in meta_dictionary and 'data_filename' in cbioportal_common . META_FIELD_MAP [ meta_file_type ] : validator_class = globals ( ) [ VALIDATOR_IDS [ meta_file_type ] ] validator = validator_class ( directory , meta_dictionary , portal_instance , logger , relaxed_mode , strict_maf_checks ) validators_by_type [ meta_file_type ] . append ( validator ) else : validators_by_type [ meta_file_type ] . append ( None ) if study_cancer_type is None : logger . error ( 'Cancer type needs to be defined for a study. Verify that you have a study file ' 'and have defined the cancer type correctly.' ) # prepend the cancer study id to any case list suffixes defined_case_list_fns = { } if study_id is not None : for suffix in case_list_suffix_fns : defined_case_list_fns [ study_id + '_' + suffix ] = case_list_suffix_fns [ suffix ] return ( validators_by_type , defined_case_list_fns , study_cancer_type , study_id , stable_ids , tags_file_path )