(profile, region, bucket, table, archive)
Fetch latest file named filename from S3

    Bucket must exist prior to running this function.
    filename is args.dumpPath.  File would be "args.dumpPath" with suffix .tar.bz2 or .zip
Fetch latest file named filename from S3
Fetch latest file named filename from S3
def do_get_s3_archive(profile, region, bucket, table, archive):
    """
    Fetch latest file named filename from S3

    Bucket must exist prior to running this function.
    filename is args.dumpPath.  File would be "args.dumpPath" with suffix .tar.bz2 or .zip
    """

    s3 = _get_aws_client(profile=profile, region=region, service="s3")

    if archive:
        if archive == "tar":
            archive_type = "tar.bz2"
        else:
            archive_type = "zip"

    # Make sure bucket exists before continuing
    try:
        s3.head_bucket(Bucket=bucket)
    except s3.exceptions.ClientError as e:
        logging.exception(
            "S3 bucket " + bucket + " does not exist. "
            "Can't get backup file\n\n" + str(e)
        )
        sys.exit(1)

    try:
        contents = s3.list_objects_v2(Bucket=bucket, Prefix=args.dumpPath)
    except s3.exceptions.ClientError as e:
        logging.exception(
            "Issue listing contents of bucket " + bucket + "\n\n" + str(e)
        )
        sys.exit(1)

    # Script will always overwrite older backup.  Bucket versioning stores multiple backups.
    # Therefore, just get item from bucket based on table name since that's what we name the files.
    filename = None
    for d in contents["Contents"]:
        if d["Key"] == "{}/{}.{}".format(args.dumpPath, table, archive_type):
            filename = d["Key"]

    if not filename:
        logging.exception(
            "Unable to find file to restore from.  "
            "Confirm the name of the table you're restoring."
        )
        sys.exit(1)

    output_file = "/tmp/" + os.path.basename(filename)
    logging.info("Downloading file " + filename + " to " + output_file)
    s3.download_file(bucket, filename, output_file)

    # Extract archive based on suffix
    if tarfile.is_tarfile(output_file):
        try:
            logging.info("Extracting tar file...")
            with tarfile.open(name=output_file, mode="r:bz2") as a:
                a.extractall(path=".")
        except tarfile.ReadError as e:
            logging.exception("Error reading downloaded archive\n\n" + str(e))
            sys.exit(1)
        except tarfile.ExtractError as e:
            # ExtractError is raised for non-fatal errors on extract method
            logging.error("Error during extraction: " + str(e))

    # Assuming zip file here since we're only supporting tar and zip at this time
    else:
        try:
            logging.info("Extracting zip file...")
            with zipfile.ZipFile(output_file, "r") as z:
                z.extractall(path=".")
        except zipfile.BadZipFile as e:
            logging.exception("Problem extracting zip file\n\n" + str(e))
            sys.exit(1)
def do_get_s3_archive ( profile , region , bucket , table , archive ) : s3 = _get_aws_client ( profile = profile , region = region , service = "s3" ) if archive : if archive == "tar" : archive_type = "tar.bz2" else : archive_type = "zip" # Make sure bucket exists before continuing try : s3 . head_bucket ( Bucket = bucket ) except s3 . exceptions . ClientError as e : logging . exception ( "S3 bucket " + bucket + " does not exist. " "Can't get backup file\n\n" + str ( e ) ) sys . exit ( 1 ) try : contents = s3 . list_objects_v2 ( Bucket = bucket , Prefix = args . dumpPath ) except s3 . exceptions . ClientError as e : logging . exception ( "Issue listing contents of bucket " + bucket + "\n\n" + str ( e ) ) sys . exit ( 1 ) # Script will always overwrite older backup.  Bucket versioning stores multiple backups. # Therefore, just get item from bucket based on table name since that's what we name the files. filename = None for d in contents [ "Contents" ] : if d [ "Key" ] == "{}/{}.{}" . format ( args . dumpPath , table , archive_type ) : filename = d [ "Key" ] if not filename : logging . exception ( "Unable to find file to restore from.  " "Confirm the name of the table you're restoring." ) sys . exit ( 1 ) output_file = "/tmp/" + os . path . basename ( filename ) logging . info ( "Downloading file " + filename + " to " + output_file ) s3 . download_file ( bucket , filename , output_file ) # Extract archive based on suffix if tarfile . is_tarfile ( output_file ) : try : logging . info ( "Extracting tar file..." ) with tarfile . open ( name = output_file , mode = "r:bz2" ) as a : a . extractall ( path = "." ) except tarfile . ReadError as e : logging . exception ( "Error reading downloaded archive\n\n" + str ( e ) ) sys . exit ( 1 ) except tarfile . ExtractError as e : # ExtractError is raised for non-fatal errors on extract method logging . error ( "Error during extraction: " + str ( e ) ) # Assuming zip file here since we're only supporting tar and zip at this time else : try : logging . info ( "Extracting zip file..." ) with zipfile . ZipFile ( output_file , "r" ) as z : z . extractall ( path = "." ) except zipfile . BadZipFile as e : logging . exception ( "Problem extracting zip file\n\n" + str ( e ) ) sys . exit ( 1 )