
inputFormatClass = 'it.polimi.genomics.spark.implementation.loaders.Loaders$CombineTextFileWithPathInputFormat'
keyFormatClass = 'org.apache.hadoop.io.LongWritable'
valueFormatClass = 'org.apache.hadoop.io.Text'
defaultCombineSize = 64

# configuration of the Hadoop loader used by spark
conf = {"textinputformat.record.delimiter": "\n",
        "mapreduce.input.fileinputformat.input.dir.recursive": "true",
        "mapred.max.split.size": str(defaultCombineSize*1024*1024)
        }


def generateHashKey(filename):
    return hash(filename)


def generateNameKey(filename):
    if filename.endswith(".meta"):
        return filename[:-5]
    else:
        return filename

import os

"""
Loader settings:
we use the GMQL scala class CombineTextFileWithPathInputFormat in order to load the region and metadata files with the
same id_sample based on the hash of the file name
"""

inputFormatClass = 'it.polimi.genomics.spark.implementation.loaders.Loaders$CombineTextFileWithPathInputFormat'
keyFormatClass = 'org.apache.hadoop.io.LongWritable'
valueFormatClass = 'org.apache.hadoop.io.Text'
defaultCombineSize = 64

# configuration of the Hadoop loader used by spark
conf = {"textinputformat.record.delimiter": "\n",
        "mapreduce.input.fileinputformat.input.dir.recursive": "true",
        "mapred.max.split.size": str(defaultCombineSize*1024*1024)
        }


"""
    Generation of the index of the pandas dataframe.
    This can be done in different ways:
    - hashing the complete file name
    - using directly the file as index (this is visually appealing :) )
"""

def generateHashKey(filename):
    return hash(filename)


def generateNameKey(filename):
    filename = os.path.basename(filename)
    if filename.endswith(".meta"):
        return filename[:-5]
    else:
        return filename

Use only the base file name for key
Use only the base file name for key


Former-commit-id: 3acbf9e93d3d501013e2a1b6aa9631bdcc663c66 [formerly eea949727d3693aa032033d84dcca3790d9072dd] [formerly ca065bc9f78b1416903179418d64b3273d437987]
Former-commit-id: 58c1d18e8960ad3236a34b8080b18ccff5684eaa
Former-commit-id: 6501068cea164df37ec055c3fb8baf8c89fc7823
+ import os
+ 
+ """
+ Loader settings:
+ we use the GMQL scala class CombineTextFileWithPathInputFormat in order to load the region and metadata files with the
+ same id_sample based on the hash of the file name
+ """
  
  inputFormatClass = 'it.polimi.genomics.spark.implementation.loaders.Loaders$CombineTextFileWithPathInputFormat'
  keyFormatClass = 'org.apache.hadoop.io.LongWritable'
  valueFormatClass = 'org.apache.hadoop.io.Text'
  defaultCombineSize = 64
  
  # configuration of the Hadoop loader used by spark
  conf = {"textinputformat.record.delimiter": "\n",
          "mapreduce.input.fileinputformat.input.dir.recursive": "true",
          "mapred.max.split.size": str(defaultCombineSize*1024*1024)
          }
  
  
+ """
+     Generation of the index of the pandas dataframe.
+     This can be done in different ways:
+     - hashing the complete file name
+     - using directly the file as index (this is visually appealing :) )
+ """
+ 
  def generateHashKey(filename):
      return hash(filename)
  
  
  def generateNameKey(filename):
+     filename = os.path.basename(filename)
      if filename.endswith(".meta"):
          return filename[:-5]
      else:
          return filename
  
Use only the base file name for key
## Code Before:

inputFormatClass = 'it.polimi.genomics.spark.implementation.loaders.Loaders$CombineTextFileWithPathInputFormat'
keyFormatClass = 'org.apache.hadoop.io.LongWritable'
valueFormatClass = 'org.apache.hadoop.io.Text'
defaultCombineSize = 64

# configuration of the Hadoop loader used by spark
conf = {"textinputformat.record.delimiter": "\n",
        "mapreduce.input.fileinputformat.input.dir.recursive": "true",
        "mapred.max.split.size": str(defaultCombineSize*1024*1024)
        }


def generateHashKey(filename):
    return hash(filename)


def generateNameKey(filename):
    if filename.endswith(".meta"):
        return filename[:-5]
    else:
        return filename

## Instruction:
Use only the base file name for key
## Code After:
import os

"""
Loader settings:
we use the GMQL scala class CombineTextFileWithPathInputFormat in order to load the region and metadata files with the
same id_sample based on the hash of the file name
"""

inputFormatClass = 'it.polimi.genomics.spark.implementation.loaders.Loaders$CombineTextFileWithPathInputFormat'
keyFormatClass = 'org.apache.hadoop.io.LongWritable'
valueFormatClass = 'org.apache.hadoop.io.Text'
defaultCombineSize = 64

# configuration of the Hadoop loader used by spark
conf = {"textinputformat.record.delimiter": "\n",
        "mapreduce.input.fileinputformat.input.dir.recursive": "true",
        "mapred.max.split.size": str(defaultCombineSize*1024*1024)
        }


"""
    Generation of the index of the pandas dataframe.
    This can be done in different ways:
    - hashing the complete file name
    - using directly the file as index (this is visually appealing :) )
"""

def generateHashKey(filename):
    return hash(filename)


def generateNameKey(filename):
    filename = os.path.basename(filename)
    if filename.endswith(".meta"):
        return filename[:-5]
    else:
        return filename
