
import json
import os
import sys
import hashlib

def get_md5sums(directory):
    md5sums = []
    for filename in os.listdir(directory):
        md5 = hashlib.md5()
        with open(os.path.join(directory, filename), 'rb') as fp:
            for chunk in iter(lambda: fp.read(128 * md5.block_size), b''):
                md5.update(chunk)
        md5hash = md5.hexdigest()
        md5sums.append((filename, md5hash))

    return md5sums

if __name__ == '__main__':
    if len(sys.argv) < 3:
        print('Usage: ' + sys.argv[0] + ' input_directory output.json')
        sys.exit(1)
    directory = sys.argv[1]
    if not os.path.exists(directory):
        print('Directory does not exist!')
        sys.exit(1)
    output_json = sys.argv[2]

    md5sums = get_md5sums(directory)
    with open(output_json, 'w') as fp:
        json.dump(md5sums, fp, indent=0)


import json
import os
import sys
import hashlib

def get_relative_filepaths(base_directory):
    """ Return a list of file paths without the base_directory prefix"""
    file_list = []
    for root, subFolders, files in os.walk('Data'):
        relative_path="/".join(root.split('/')[1:])
        for file in files:
            file_list.append(os.path.join(relative_path,file))
    return file_list


def get_md5sums(base_directory):
    md5sums = []
    for filename in get_relative_filepaths(base_directory):
        md5 = hashlib.md5()
        full_filepath = os.path.join(base_directory, filename)
        with open(full_filepath, 'rb') as fp:
            for chunk in iter(lambda: fp.read(128 * md5.block_size), b''):
                md5.update(chunk)
        md5hash = md5.hexdigest()
        md5sums.append((filename, md5hash))
    return md5sums

if __name__ == '__main__':
    if len(sys.argv) < 3:
        print('Usage: ' + sys.argv[0] + ' input_directory output.json')
        sys.exit(1)
    directory = sys.argv[1]
    if not os.path.exists(directory):
        print('Directory does not exist!')
        sys.exit(1)
    output_json = sys.argv[2]

    md5sums = get_md5sums(directory)
    with open(output_json, 'w') as fp:
        json.dump(md5sums, fp, indent=0)

Allow directory of files to be indexed
ENH: Allow directory of files to be indexed

In the Data directory, there may be sub-directories of files that need to
be kept separate, but all of them need to be indexed.

  
  import json
  import os
  import sys
  import hashlib
  
+ def get_relative_filepaths(base_directory):
+     """ Return a list of file paths without the base_directory prefix"""
+     file_list = []
+     for root, subFolders, files in os.walk('Data'):
+         relative_path="/".join(root.split('/')[1:])
+         for file in files:
+             file_list.append(os.path.join(relative_path,file))
+     return file_list
+ 
+ 
- def get_md5sums(directory):
+ def get_md5sums(base_directory):
      md5sums = []
-     for filename in os.listdir(directory):
+     for filename in get_relative_filepaths(base_directory):
          md5 = hashlib.md5()
-         with open(os.path.join(directory, filename), 'rb') as fp:
+         full_filepath = os.path.join(base_directory, filename)
+         with open(full_filepath, 'rb') as fp:
              for chunk in iter(lambda: fp.read(128 * md5.block_size), b''):
                  md5.update(chunk)
          md5hash = md5.hexdigest()
          md5sums.append((filename, md5hash))
- 
      return md5sums
  
  if __name__ == '__main__':
      if len(sys.argv) < 3:
          print('Usage: ' + sys.argv[0] + ' input_directory output.json')
          sys.exit(1)
      directory = sys.argv[1]
      if not os.path.exists(directory):
          print('Directory does not exist!')
          sys.exit(1)
      output_json = sys.argv[2]
  
      md5sums = get_md5sums(directory)
      with open(output_json, 'w') as fp:
          json.dump(md5sums, fp, indent=0)
  
Allow directory of files to be indexed
## Code Before:

import json
import os
import sys
import hashlib

def get_md5sums(directory):
    md5sums = []
    for filename in os.listdir(directory):
        md5 = hashlib.md5()
        with open(os.path.join(directory, filename), 'rb') as fp:
            for chunk in iter(lambda: fp.read(128 * md5.block_size), b''):
                md5.update(chunk)
        md5hash = md5.hexdigest()
        md5sums.append((filename, md5hash))

    return md5sums

if __name__ == '__main__':
    if len(sys.argv) < 3:
        print('Usage: ' + sys.argv[0] + ' input_directory output.json')
        sys.exit(1)
    directory = sys.argv[1]
    if not os.path.exists(directory):
        print('Directory does not exist!')
        sys.exit(1)
    output_json = sys.argv[2]

    md5sums = get_md5sums(directory)
    with open(output_json, 'w') as fp:
        json.dump(md5sums, fp, indent=0)

## Instruction:
Allow directory of files to be indexed
## Code After:

import json
import os
import sys
import hashlib

def get_relative_filepaths(base_directory):
    """ Return a list of file paths without the base_directory prefix"""
    file_list = []
    for root, subFolders, files in os.walk('Data'):
        relative_path="/".join(root.split('/')[1:])
        for file in files:
            file_list.append(os.path.join(relative_path,file))
    return file_list


def get_md5sums(base_directory):
    md5sums = []
    for filename in get_relative_filepaths(base_directory):
        md5 = hashlib.md5()
        full_filepath = os.path.join(base_directory, filename)
        with open(full_filepath, 'rb') as fp:
            for chunk in iter(lambda: fp.read(128 * md5.block_size), b''):
                md5.update(chunk)
        md5hash = md5.hexdigest()
        md5sums.append((filename, md5hash))
    return md5sums

if __name__ == '__main__':
    if len(sys.argv) < 3:
        print('Usage: ' + sys.argv[0] + ' input_directory output.json')
        sys.exit(1)
    directory = sys.argv[1]
    if not os.path.exists(directory):
        print('Directory does not exist!')
        sys.exit(1)
    output_json = sys.argv[2]

    md5sums = get_md5sums(directory)
    with open(output_json, 'w') as fp:
        json.dump(md5sums, fp, indent=0)
