from __future__ import print_function

import sys, os
from re import sub
import zipfile, json

# from pyspark import SparkContext
# from pyspark.sql import SQLContext
# from pyspark.sql import Row
# from pyspark.sql.types import StringType

def getSeries(fname):
    with zipfile.ZipFile(fname, 'r') as zf:
        names = zf.namelist()
        mfile = [f for f in names if f.endswith('.metadata.json')]
        series = fname
        if len(mfile) > 0:
            m = json.loads(zf.read(mfile[0]))
            series = m['identifier'][0]
            return m

if __name__ == "__main__":
    if len(sys.argv) < 2:
        print("Usage: europeana.py <input> <output>", file=sys.stderr)
        exit(-1)
    # sc = SparkContext(appName="Europeana Import")
    # sqlContext = SQLContext(sc)

    x = [os.path.join(d[0], f) for d in os.walk(sys.argv[1]) for f in d[2] if f.endswith('zip')]
    for f in x:
        print(json.dumps(getSeries(f)))
        
    # sc.parallelize(x, 200).flatMap(getSeries).toDF().write.save(sys.argv[2])

    # sc.stop()

from __future__ import print_function

import sys, os
from re import sub
import zipfile, json

# from pyspark import SparkContext
# from pyspark.sql import SQLContext
# from pyspark.sql import Row
# from pyspark.sql.types import StringType

def getSeries(fname):
    with zipfile.ZipFile(fname, 'r') as zf:
        names = zf.namelist()
        mfile = [f for f in names if f.endswith('.metadata.json')]
        series = 'europeana/' + sub('^.*newspapers-by-country/', '',
                                    sub('[\x80-\xff]', '', fname).replace('.zip', ''))
        if len(mfile) > 0:
            m = json.loads(zf.read(mfile[0]))
            return {'series': series, 'title': m['title'][0], 'lang': m['language']}

if __name__ == "__main__":
    if len(sys.argv) < 2:
        print("Usage: europeana.py <input> <output>", file=sys.stderr)
        exit(-1)
    # sc = SparkContext(appName="Europeana Import")
    # sqlContext = SQLContext(sc)

    x = [os.path.join(d[0], f) for d in os.walk(sys.argv[1]) for f in d[2] if f.endswith('zip')]
    for f in x:
        print(json.dumps(getSeries(f)))
        
    # sc.parallelize(x, 200).flatMap(getSeries).toDF().write.save(sys.argv[2])

    # sc.stop()

Use file path as Europeana series name.
Use file path as Europeana series name.

  from __future__ import print_function
  
  import sys, os
  from re import sub
  import zipfile, json
  
  # from pyspark import SparkContext
  # from pyspark.sql import SQLContext
  # from pyspark.sql import Row
  # from pyspark.sql.types import StringType
  
  def getSeries(fname):
      with zipfile.ZipFile(fname, 'r') as zf:
          names = zf.namelist()
          mfile = [f for f in names if f.endswith('.metadata.json')]
-         series = fname
+         series = 'europeana/' + sub('^.*newspapers-by-country/', '',
+                                     sub('[\x80-\xff]', '', fname).replace('.zip', ''))
          if len(mfile) > 0:
              m = json.loads(zf.read(mfile[0]))
+             return {'series': series, 'title': m['title'][0], 'lang': m['language']}
-             series = m['identifier'][0]
-             return m
  
  if __name__ == "__main__":
      if len(sys.argv) < 2:
          print("Usage: europeana.py <input> <output>", file=sys.stderr)
          exit(-1)
      # sc = SparkContext(appName="Europeana Import")
      # sqlContext = SQLContext(sc)
  
      x = [os.path.join(d[0], f) for d in os.walk(sys.argv[1]) for f in d[2] if f.endswith('zip')]
      for f in x:
          print(json.dumps(getSeries(f)))
          
      # sc.parallelize(x, 200).flatMap(getSeries).toDF().write.save(sys.argv[2])
  
      # sc.stop()
  
Use file path as Europeana series name.
## Code Before:
from __future__ import print_function

import sys, os
from re import sub
import zipfile, json

# from pyspark import SparkContext
# from pyspark.sql import SQLContext
# from pyspark.sql import Row
# from pyspark.sql.types import StringType

def getSeries(fname):
    with zipfile.ZipFile(fname, 'r') as zf:
        names = zf.namelist()
        mfile = [f for f in names if f.endswith('.metadata.json')]
        series = fname
        if len(mfile) > 0:
            m = json.loads(zf.read(mfile[0]))
            series = m['identifier'][0]
            return m

if __name__ == "__main__":
    if len(sys.argv) < 2:
        print("Usage: europeana.py <input> <output>", file=sys.stderr)
        exit(-1)
    # sc = SparkContext(appName="Europeana Import")
    # sqlContext = SQLContext(sc)

    x = [os.path.join(d[0], f) for d in os.walk(sys.argv[1]) for f in d[2] if f.endswith('zip')]
    for f in x:
        print(json.dumps(getSeries(f)))
        
    # sc.parallelize(x, 200).flatMap(getSeries).toDF().write.save(sys.argv[2])

    # sc.stop()

## Instruction:
Use file path as Europeana series name.
## Code After:
from __future__ import print_function

import sys, os
from re import sub
import zipfile, json

# from pyspark import SparkContext
# from pyspark.sql import SQLContext
# from pyspark.sql import Row
# from pyspark.sql.types import StringType

def getSeries(fname):
    with zipfile.ZipFile(fname, 'r') as zf:
        names = zf.namelist()
        mfile = [f for f in names if f.endswith('.metadata.json')]
        series = 'europeana/' + sub('^.*newspapers-by-country/', '',
                                    sub('[\x80-\xff]', '', fname).replace('.zip', ''))
        if len(mfile) > 0:
            m = json.loads(zf.read(mfile[0]))
            return {'series': series, 'title': m['title'][0], 'lang': m['language']}

if __name__ == "__main__":
    if len(sys.argv) < 2:
        print("Usage: europeana.py <input> <output>", file=sys.stderr)
        exit(-1)
    # sc = SparkContext(appName="Europeana Import")
    # sqlContext = SQLContext(sc)

    x = [os.path.join(d[0], f) for d in os.walk(sys.argv[1]) for f in d[2] if f.endswith('zip')]
    for f in x:
        print(json.dumps(getSeries(f)))
        
    # sc.parallelize(x, 200).flatMap(getSeries).toDF().write.save(sys.argv[2])

    # sc.stop()
