A TeraGen implementation using Lithops
import argparse
import time

from lithops import FunctionExecutor, Storage
from lithops.config import load_config

from teragen.data_generator import DataGenerator


def size_str_to_bytes(s: str) -> int:
    lower = s.lower()
    if lower.endswith("k"):
        return int(lower[:-1]) * 1000
    elif lower.endswith("m"):
        return int(lower[:-1]) * 1000 * 1000
    elif lower.endswith("g"):
        return int(lower[:-1]) * 1000 * 1000 * 1000
    elif lower.endswith("t"):
        return int(lower[:-1]) * 1000 * 1000 * 1000 * 1000
    else:
        # no suffix, so it's just a number in bytes
        return int(lower)


def size_to_size_str(size):
    kb_scale = 1000
    mb_scale = 1000 * kb_scale
    gb_scale = 1000 * mb_scale
    tb_scale = 1000 * gb_scale

    if size > tb_scale:
        return str(size // tb_scale) + "TB"
    elif size > gb_scale:
        return str(size // gb_scale) + "GB"
    elif size > mb_scale:
        return str(size // mb_scale) + "MB"
    elif size > kb_scale:
        return str(size // kb_scale) + "KB"
    else:
        return str(size) + "B"


def parse_args():
    ap = argparse.ArgumentParser()
    ap.add_argument("-s", "--size", required=True, type=str,
                    help="Size of the dataset to generate. Examples: 100k, 5m, 10g, 1t. Or just the number of bytes.")
    ap.add_argument("-b", "--bucket-name", type=str, help="Output bucket name", default="teragen-data")
    ap.add_argument("-k", "--key-name", type=str, help="Output file name", default="part-r")
    ap.add_argument("--ascii", action="store_true",
                    help="Generate ASCII data instead of binary data", default=False)
    ap.add_argument("-p", "--partitions", type=int, default=16,
                    help="Number of partitions to generate")
    ap.add_argument("-c", "--config-file", type=str)
    ap.add_argument("-l", "--localhost", action="store_true", default=False,
                    help="Run locally using processes")
    ap.add_argument("-u", "--unique-file", action="store_true", default=False,
                    help="Uses multipart upload to upload a unique file to S3. " +
                         "This needs Lithops configured with S3 as the storage backend.")

    return ap.parse_args()


def print_config(size, is_ascii, num_partitions, records_per_partition, num_records, key_name, bucket_name):
    print("===========================================================================")
    print("===========================================================================")
    print(f"Input size: {size_to_size_str(size)}")
    print(f"Total number of records: {num_records}")
    print(f"Number of output partitions: {num_partitions}")
    print(f"Number of records per partition: {records_per_partition}")
    print(f"Output format in ASCII: {is_ascii}")
    print(f"Output bucket: {bucket_name}")
    print(f"Output file name: {key_name}")
    print("===========================================================================")
    print("===========================================================================")


def write_dataset(index, records_per_partition, is_ascii, bucket_name, key_name, storage):
    dg = DataGenerator.create_data_generator(index, records_per_partition, is_ascii)
    start_time = time.time()
    storage.put_object(bucket_name, key_name, dg)
    end_time = time.time()

    return {'id': index, 'start_time': start_time, 'end_time': end_time}


def write_dataset_multipart(index, records_per_partition, is_ascii, bucket_name, key_name, upload_id, storage):
    dg = DataGenerator.create_data_generator(index, records_per_partition, is_ascii)
    start_time = time.time()
    client = storage.get_client()
    response = client.upload_part(Bucket=bucket_name, Key=key_name, PartNumber=index+1, UploadId=upload_id, Body=dg)
    end_time = time.time()
    part = {'PartNumber': index+1,
            'ETag': response['ETag']}
    return part


def main():
    args = parse_args()
    size = size_str_to_bytes(args.size)
    is_ascii = args.ascii
    key_name = args.key_name
    bucket_name = args.bucket_name
    num_partitions = args.partitions

    records_per_partition = size // num_partitions // 100
    num_records = records_per_partition * num_partitions

    print_config(size, is_ascii, num_partitions, records_per_partition, num_records, key_name, bucket_name)

    executor = FunctionExecutor(backend='localhost', storage='localhost') if args.localhost \
        else FunctionExecutor(config_file=args.config_file)

    if args.unique_file: # Create a single file
        if not (5 * 1024 * 1024 < records_per_partition * 100 < 5 * 1024 * 1024 * 1024):
            raise ValueError("Partition size must be between 5MB and 5GB for multipart upload.")

        client = Storage().get_client()
        multipart_response = client.create_multipart_upload(Bucket=bucket_name, Key=key_name)
        upload_id = multipart_response['UploadId']
        arg_iter = [(i, records_per_partition, is_ascii, bucket_name, key_name, upload_id)
                    for i in range(num_partitions)]
        executor.map(write_dataset_multipart, arg_iter)
        parts = executor.get_result()

        client.complete_multipart_upload(Bucket=bucket_name, Key=key_name, UploadId=upload_id,
                                         MultipartUpload={'Parts': parts})

    else: # Create multiple files
        arg_iter = [(i, records_per_partition, is_ascii, bucket_name, f"{key_name}-{i:05d}")
                    for i in range(num_partitions)]
        executor.map(write_dataset, arg_iter)
        executor.wait()


    print("===========================================================================")
    print("Data generation complete")
    print("===========================================================================")



if '__main__' == __name__:
    main()
teragen.py
gfinol/teragen-lithops
from teragen.data_iterator import DataIterator


class DataGenerator(object):
    """
    A file-like object which generates data for the TeraSort benchmark.

    Never actually keeps all the data in memory so can be used to generate huge data sets.
    """

    def __init__(self, data_iterator: 'DataIterator'):
        self.data_iterator = data_iterator
        self.BLOCK_SIZE_BYTES = 100

        self.bytes_total = len(data_iterator) * self.BLOCK_SIZE_BYTES
        self.pos = 0

        self.current_block_data = b""
        self.current_block_pos = 0

    @staticmethod
    def create_data_generator(index: int, records_per_partition: int, is_ascii: bool = False):
        return DataGenerator(DataIterator(index, records_per_partition, is_ascii))

    def __len__(self) -> int:
        return self.bytes_total

    @property
    def len(self) -> int:
        return self.bytes_total

    def tell(self) -> int:
        return self.pos

    def get_next_block(self) -> bytes:
        return next(self.data_iterator)

    def read(self, bytes_requested: int) -> bytes:

        remaining_bytes = self.bytes_total - self.pos
        if remaining_bytes == 0:
            return b''

        if bytes_requested < 0:
            bytes_out = remaining_bytes
        else:
            bytes_out = min(remaining_bytes, bytes_requested)

        byte_data = b''
        byte_pos = 0
        while byte_pos < bytes_out:

            bytes_remaining = bytes_out - byte_pos

            needed_data_from_current_block = min(bytes_remaining, self.BLOCK_SIZE_BYTES - self.current_block_pos)

            chunk = self.current_block_data[self.current_block_pos:self.current_block_pos + needed_data_from_current_block]
            byte_data += chunk
            byte_pos += len(chunk)
            self.current_block_pos = (self.current_block_pos + needed_data_from_current_block) % self.BLOCK_SIZE_BYTES

            if self.current_block_pos == 0 and remaining_bytes - byte_pos > 0:
                self.current_block_data = self.get_next_block()

        self.pos += bytes_out

        return byte_data

    def seek(self, byte_num: int):
        if byte_num < 0:
            raise ValueError("Seek position must be non-negative")
        if byte_num >= self.bytes_total:
            raise ValueError("Seek position must be less than total number of bytes")

        self.pos = byte_num
        self.data_iterator.seek(byte_num // self.BLOCK_SIZE_BYTES)
        self.current_block_data = self.get_next_block()
        self.current_block_pos = byte_num % self.BLOCK_SIZE_BYTES
teragen/data_generator.py
gfinol/teragen-lithops
import numpy as np

from teragen.records_generators import RECORD_LEN, generate_ascii_record, generate_record
from teragen.random16 import Random16
from teragen.unsigned16 import Unsigned16


class DataIterator:

    def __init__(self, index: int, records_per_partition: int, is_ascii: bool = False):
        self.index = index
        self.records_per_partition = records_per_partition
        self.is_ascii = is_ascii

        self.one = Unsigned16(1)
        self.first_record_number = Unsigned16(self.index * self.records_per_partition)
        self.records_to_generate = Unsigned16(self.records_per_partition)

        self.record_number = Unsigned16(self.first_record_number)
        self.last_record_number = Unsigned16(self.first_record_number)
        self.last_record_number.add(self.records_to_generate)

        self.rand = Random16.skip_ahead(self.first_record_number)

        self.rowBytes = bytearray(RECORD_LEN)

    def __iter__(self):
        self.rand = Random16.skip_ahead(self.first_record_number)
        self.record_number = Unsigned16(self.first_record_number)
        return self

    def __next__(self):
        if self.record_number.value >= self.last_record_number.value:
            raise StopIteration
        else:
            Random16.next_rand(self.rand)
            if self.is_ascii:
                generate_ascii_record(self.rowBytes, self.rand, self.record_number)
            else:
                generate_record(self.rowBytes, self.rand, self.record_number)
            self.record_number.add(self.one)
            return bytes(self.rowBytes)

    def __len__(self):
        return self.records_per_partition

    def seek(self, record_number: int):
        self.record_number = Unsigned16(record_number)
        self.record_number.add(self.first_record_number)
        self.rand = Random16.skip_ahead(self.record_number)


teragen/data_iterator.py
gfinol/teragen-lithops
from teragen.unsigned16 import Unsigned16


class Random16:
    class RandomConstant:
        def __init__(self, left: str, right: str):
            self.a = Unsigned16(left)
            self.c = Unsigned16(right)

    genArray = [
        RandomConstant("2360ed051fc65da44385df649fccf645", "4a696d47726179524950202020202001"),
        RandomConstant("17bce35bdf69743c529ed9eb20e0ae99", "95e0e48262b3edfe04479485c755b646"),
        RandomConstant("f4dd417327db7a9bd194dfbe42d45771", "882a02c315362b60765f100068b33a1c"),
        RandomConstant("6347af777a7898f6d1a2d6f33505ffe1", "5efc4abfaca23e8ca8edb1f2dfbf6478"),
        RandomConstant("b6a4239f3b315f84f6ef6d3d288c03c1", "f25bd15439d16af594c1b1bafa6239f0"),
        RandomConstant("2c82901ad1cb0cd182b631ba6b261781", "89ca67c29c9397d59c612596145db7e0"),
        RandomConstant("dab03f988288676ee49e66c4d2746f01", "8b6ae036713bd578a8093c8eae5c7fc0"),
        RandomConstant("602167331d86cf5684fe009a6d09de01", "98a2542fd23d0dbdff3b886cdb1d3f80"),
        RandomConstant("61ecb5c24d95b058f04c80a23697bc01", "954db923fdb7933e947cd1edcecb7f00"),
        RandomConstant("4a5c31e0654c28aa60474e83bf3f7801", "00be4a36657c98cd204e8c8af7dafe00"),
        RandomConstant("ae4f079d54fbece1478331d3c6bef001", "991965329dccb28d581199ab18c5fc00"),
        RandomConstant("101b8cb830c7cb927ff1ed50ae7de001", "e1a8705b63ad5b8cd6c3d268d5cbf800"),
        RandomConstant("f54a27fc056b00e7563f3505e0fbc001", "2b657bbfd6ed9d632079e70c3c97f000"),
        RandomConstant("df8a6fc1a833d201f98d719dd1f78001", "59b60ee4c52fa49e9fe90682bd2fe000"),
        RandomConstant("5480a5015f101a4ea7e3f183e3ef0001", "cc099c88030679464fe86aae8a5fc000"),
        RandomConstant("a498509e76e5d7925f539c28c7de0001", "06b9abff9f9f33dd30362c0154bf8000"),
        RandomConstant("0798a3d8b10dc72e60121cd58fbc0001", "e296707121688d5a0260b293a97f0000"),
        RandomConstant("1647d1e78ec02e665fafcbbb1f780001", "189ffc4701ff23cb8f8acf6b52fe0000"),
        RandomConstant("a7c982285e72bf8c0c8ddfb63ef00001", "5141110ab208fb9d61fb47e6a5fc0000"),
        RandomConstant("3eb78ee8fb8c56dbc5d4e06c7de00001", "3c97caa62540f2948d8d340d4bf80000"),
        RandomConstant("72d03b6f4681f2f9fe8e44d8fbc00001", "1b25cb9cfe5a0c963174f91a97f00000"),
        RandomConstant("ea85f81e4f502c9bc8ae99b1f7800001", "0c644570b4a487103c5436352fe00000"),
        RandomConstant("629c320db08b00c6bfa57363ef000001", "3d0589c28869472bde517c6a5fc00000"),
        RandomConstant("c5c4b9ce268d074a386be6c7de000001", "bc95e5ab36477e65534738d4bf800000"),
        RandomConstant("f30bbbbed1596187555bcd8fbc000001", "ddb02ff72a031c01011f71a97f000000"),
        RandomConstant("4a1000fb26c9eeda3cc79b1f78000001", "2561426086d9acdb6c82e352fe000000"),
        RandomConstant("89fb5307f6bf8ce2c1cf363ef0000001", "64a788e3c118ed1c8215c6a5fc000000"),
        RandomConstant("830b7b3358a5d67ea49e6c7de0000001", "e65ea321908627cfa86b8d4bf8000000"),
        RandomConstant("fd8a51da91a69fe1cd3cd8fbc0000001", "53d27225604d85f9e1d71a97f0000000"),
        RandomConstant("901a48b642b90b55aa79b1f780000001", "ca5ec7a3ed1fe55e07ae352fe0000000"),
        RandomConstant("118cdefdf32144f394f363ef00000001", "4daebb2e085330651f5c6a5fc0000000"),
        RandomConstant("0a88c0a91cff430829e6c7de00000001", "9d6f1a00a8f3f76e7eb8d4bf80000000"),
        RandomConstant("433bef4314f16a9453cd8fbc00000001", "158c62f2b31e496dfd71a97f00000000"),
        RandomConstant("c294b02995ae6738a79b1f7800000001", "290e84a2eb15fd1ffae352fe00000000"),
        RandomConstant("913575e0da8b16b14f363ef000000001", "e3dc1bfbe991a34ff5c6a5fc00000000"),
        RandomConstant("2f61b9f871cf4e629e6c7de000000001", "ddf540d020b9eadfeb8d4bf800000000"),
        RandomConstant("78d26ccbd68320c53cd8fbc000000001", "8ee4950177ce66bfd71a97f000000000"),
        RandomConstant("8b7ebd037898518a79b1f78000000001", "39e0f787c907117fae352fe000000000"),
        RandomConstant("0b5507b61f78e314f363ef0000000001", "659d2522f7b732ff5c6a5fc000000000"),
        RandomConstant("4f884628f812c629e6c7de0000000001", "9e8722938612a5feb8d4bf8000000000"),
        RandomConstant("be896744d4a98c53cd8fbc0000000001", "e941a65d66b64bfd71a97f0000000000"),
        RandomConstant("daf63a553b6318a79b1f780000000001", "7b50d19437b097fae352fe0000000000"),
        RandomConstant("2d7a23d8bf06314f363ef00000000001", "59d7b68e18712ff5c6a5fc0000000000"),
        RandomConstant("392b046a9f0c629e6c7de00000000001", "4087bab2d5225feb8d4bf80000000000"),
        RandomConstant("eb30fbb9c218c53cd8fbc00000000001", "b470abc03b44bfd71a97f00000000000"),
        RandomConstant("b9cdc30594318a79b1f7800000000001", "366630eaba897fae352fe00000000000"),
        RandomConstant("014ab453686314f363ef000000000001", "a2dfc77e8512ff5c6a5fc00000000000"),
        RandomConstant("395221c7d0c629e6c7de000000000001", "1e0d25a14a25feb8d4bf800000000000"),
        RandomConstant("4d972813a18c53cd8fbc000000000001", "9d50a5d3944bfd71a97f000000000000"),
        RandomConstant("06f9e2374318a79b1f78000000000001", "bf7ab5eb2897fae352fe000000000000"),
        RandomConstant("bd220cae86314f363ef0000000000001", "925b14e6512ff5c6a5fc000000000000"),
        RandomConstant("36fd3a5d0c629e6c7de0000000000001", "724cce0ca25feb8d4bf8000000000000"),
        RandomConstant("60def8ba18c53cd8fbc0000000000001", "1af42d1944bfd71a97f0000000000000"),
        RandomConstant("8d500174318a79b1f780000000000001", "0f529e32897fae352fe0000000000000"),
        RandomConstant("48e842e86314f363ef00000000000001", "844e4c6512ff5c6a5fc0000000000000"),
        RandomConstant("4af185d0c629e6c7de00000000000001", "9f40d8ca25feb8d4bf80000000000000"),
        RandomConstant("7a670ba18c53cd8fbc00000000000001", "9912b1944bfd71a97f00000000000000"),
        RandomConstant("86de174318a79b1f7800000000000001", "9c69632897fae352fe00000000000000"),
        RandomConstant("55fc2e86314f363ef000000000000001", "e1e2c6512ff5c6a5fc00000000000000"),
        RandomConstant("ccf85d0c629e6c7de000000000000001", "68058ca25feb8d4bf800000000000000"),
        RandomConstant("1df0ba18c53cd8fbc000000000000001", "610b1944bfd71a97f000000000000000"),
        RandomConstant("4be174318a79b1f78000000000000001", "061632897fae352fe000000000000000"),
        RandomConstant("d7c2e86314f363ef0000000000000001", "1c2c6512ff5c6a5fc000000000000000"),
        RandomConstant("af85d0c629e6c7de0000000000000001", "7858ca25feb8d4bf8000000000000000"),
        RandomConstant("5f0ba18c53cd8fbc0000000000000001", "f0b1944bfd71a97f0000000000000000"),
        RandomConstant("be174318a79b1f780000000000000001", "e1632897fae352fe0000000000000000"),
        RandomConstant("7c2e86314f363ef00000000000000001", "c2c6512ff5c6a5fc0000000000000000"),
        RandomConstant("f85d0c629e6c7de00000000000000001", "858ca25feb8d4bf80000000000000000"),
        RandomConstant("f0ba18c53cd8fbc00000000000000001", "0b1944bfd71a97f00000000000000000"),
        RandomConstant("e174318a79b1f7800000000000000001", "1632897fae352fe00000000000000000"),
        RandomConstant("c2e86314f363ef000000000000000001", "2c6512ff5c6a5fc00000000000000000"),
        RandomConstant("85d0c629e6c7de000000000000000001", "58ca25feb8d4bf800000000000000000"),
        RandomConstant("0ba18c53cd8fbc000000000000000001", "b1944bfd71a97f000000000000000000"),
        RandomConstant("174318a79b1f78000000000000000001", "632897fae352fe000000000000000000"),
        RandomConstant("2e86314f363ef0000000000000000001", "c6512ff5c6a5fc000000000000000000"),
        RandomConstant("5d0c629e6c7de0000000000000000001", "8ca25feb8d4bf8000000000000000000"),
        RandomConstant("ba18c53cd8fbc0000000000000000001", "1944bfd71a97f0000000000000000000"),
        RandomConstant("74318a79b1f780000000000000000001", "32897fae352fe0000000000000000000"),
        RandomConstant("e86314f363ef00000000000000000001", "6512ff5c6a5fc0000000000000000000"),
        RandomConstant("d0c629e6c7de00000000000000000001", "ca25feb8d4bf80000000000000000000"),
        RandomConstant("a18c53cd8fbc00000000000000000001", "944bfd71a97f00000000000000000000"),
        RandomConstant("4318a79b1f7800000000000000000001", "2897fae352fe00000000000000000000"),
        RandomConstant("86314f363ef000000000000000000001", "512ff5c6a5fc00000000000000000000"),
        RandomConstant("0c629e6c7de000000000000000000001", "a25feb8d4bf800000000000000000000"),
        RandomConstant("18c53cd8fbc000000000000000000001", "44bfd71a97f000000000000000000000"),
        RandomConstant("318a79b1f78000000000000000000001", "897fae352fe000000000000000000000"),
        RandomConstant("6314f363ef0000000000000000000001", "12ff5c6a5fc000000000000000000000"),
        RandomConstant("c629e6c7de0000000000000000000001", "25feb8d4bf8000000000000000000000"),
        RandomConstant("8c53cd8fbc0000000000000000000001", "4bfd71a97f0000000000000000000000"),
        RandomConstant("18a79b1f780000000000000000000001", "97fae352fe0000000000000000000000"),
        RandomConstant("314f363ef00000000000000000000001", "2ff5c6a5fc0000000000000000000000"),
        RandomConstant("629e6c7de00000000000000000000001", "5feb8d4bf80000000000000000000000"),
        RandomConstant("c53cd8fbc00000000000000000000001", "bfd71a97f00000000000000000000000"),
        RandomConstant("8a79b1f7800000000000000000000001", "7fae352fe00000000000000000000000"),
        RandomConstant("14f363ef000000000000000000000001", "ff5c6a5fc00000000000000000000000"),
        RandomConstant("29e6c7de000000000000000000000001", "feb8d4bf800000000000000000000000"),
        RandomConstant("53cd8fbc000000000000000000000001", "fd71a97f000000000000000000000000"),
        RandomConstant("a79b1f78000000000000000000000001", "fae352fe000000000000000000000000"),
        RandomConstant("4f363ef0000000000000000000000001", "f5c6a5fc000000000000000000000000"),
        RandomConstant("9e6c7de0000000000000000000000001", "eb8d4bf8000000000000000000000000"),
        RandomConstant("3cd8fbc0000000000000000000000001", "d71a97f0000000000000000000000000"),
        RandomConstant("79b1f780000000000000000000000001", "ae352fe0000000000000000000000000"),
        RandomConstant("f363ef00000000000000000000000001", "5c6a5fc0000000000000000000000000"),
        RandomConstant("e6c7de00000000000000000000000001", "b8d4bf80000000000000000000000000"),
        RandomConstant("cd8fbc00000000000000000000000001", "71a97f00000000000000000000000000"),
        RandomConstant("9b1f7800000000000000000000000001", "e352fe00000000000000000000000000"),
        RandomConstant("363ef000000000000000000000000001", "c6a5fc00000000000000000000000000"),
        RandomConstant("6c7de000000000000000000000000001", "8d4bf800000000000000000000000000"),
        RandomConstant("d8fbc000000000000000000000000001", "1a97f000000000000000000000000000"),
        RandomConstant("b1f78000000000000000000000000001", "352fe000000000000000000000000000"),
        RandomConstant("63ef0000000000000000000000000001", "6a5fc000000000000000000000000000"),
        RandomConstant("c7de0000000000000000000000000001", "d4bf8000000000000000000000000000"),
        RandomConstant("8fbc0000000000000000000000000001", "a97f0000000000000000000000000000"),
        RandomConstant("1f780000000000000000000000000001", "52fe0000000000000000000000000000"),
        RandomConstant("3ef00000000000000000000000000001", "a5fc0000000000000000000000000000"),
        RandomConstant("7de00000000000000000000000000001", "4bf80000000000000000000000000000"),
        RandomConstant("fbc00000000000000000000000000001", "97f00000000000000000000000000000"),
        RandomConstant("f7800000000000000000000000000001", "2fe00000000000000000000000000000"),
        RandomConstant("ef000000000000000000000000000001", "5fc00000000000000000000000000000"),
        RandomConstant("de000000000000000000000000000001", "bf800000000000000000000000000000"),
        RandomConstant("bc000000000000000000000000000001", "7f000000000000000000000000000000"),
        RandomConstant("78000000000000000000000000000001", "fe000000000000000000000000000000"),
        RandomConstant("f0000000000000000000000000000001", "fc000000000000000000000000000000"),
        RandomConstant("e0000000000000000000000000000001", "f8000000000000000000000000000000"),
        RandomConstant("c0000000000000000000000000000001", "f0000000000000000000000000000000"),
        RandomConstant("80000000000000000000000000000001", "e0000000000000000000000000000000"),
        RandomConstant("00000000000000000000000000000001", "c0000000000000000000000000000000"),
        RandomConstant("00000000000000000000000000000001", "80000000000000000000000000000000")
        # Add more constants for genArray here...
    ]

    # @staticmethod
    # def skip_ahead(advance: 'Unsigned16'):
    #     result = Unsigned16()
    #     bit_map = advance.getLow8()
    #     for i in range(64):
    #         if int(bit_map) & (1 << i):
    #             result.multiply(Random16.genArray[i].a)
    #             result.add(Random16.genArray[i].c)
    #             bit_map &= ~(1 << i)
    #
    #     bit_map = advance.getHigh8()
    #     for i in range(64):
    #         if int(bit_map) & (1 << i):
    #             result.multiply(Random16.genArray[i + 64].a)
    #             result.add(Random16.genArray[i + 64].c)
    #             bit_map &= ~(1 << i)
    #
    #     return result

    @staticmethod
    def skip_ahead(advance: 'Unsigned16'):
        result = Unsigned16()
        bit_map = advance.value
        for i in range(128):
            if int(bit_map) & (1 << i):
                result.multiply(Random16.genArray[i].a)
                result.add(Random16.genArray[i].c)
                bit_map &= ~(1 << i)

        return result

    @staticmethod
    def next_rand(rand):
        rand.multiply(Random16.genArray[0].a)
        rand.add(Random16.genArray[0].c)

teragen/random16.py
gfinol/teragen-lithops
from teragen.unsigned16 import Unsigned16

KEY_LEN = 10
VALUE_LEN = 90
RECORD_LEN = KEY_LEN + VALUE_LEN

ord_space = ord(' ')
ord_newline = ord('\n')
ord_cr = ord('\r')


def generate_record(rec_buf, rand: 'Unsigned16', record_number: 'Unsigned16') -> None:
    # Generate the 10-byte key using the high 10 bytes of the 128-bit random number
    for i in range(10):
        rec_buf[i] = rand.get_byte(i)

    # Add 2 bytes of "break"
    rec_buf[10] = 0x00
    rec_buf[11] = 0x11

    # Convert the 128-bit record number to 32 bits of ASCII hexadecimal
    # as the next 32 bytes of the record.
    for i in range(32):
        rec_buf[12 + i] = record_number.get_hex_digit(i)

        # Add 4 bytes of "break" data
    rec_buf[44] = 0x88
    rec_buf[45] = 0x99
    rec_buf[46] = 0xAA
    rec_buf[47] = 0xBB

    # Add 48 bytes of filler based on low 48 bits of random number
    for i in range(12):
        v = rand.get_hex_digit(20 + i)
        rec_buf[48 + i * 4] = v
        rec_buf[49 + i * 4] = v
        rec_buf[50 + i * 4] = v
        rec_buf[51 + i * 4] = v

    # Add 4 bytes of "break" data
    rec_buf[96] = 0xCC
    rec_buf[97] = 0xDD
    rec_buf[98] = 0xEE
    rec_buf[99] = 0xFF


def make_big_integer(value: int):
    return value if value >= 0 else (value + 2 ** 64)


def generate_ascii_record(rec_buf, rand: 'Unsigned16', record_number: 'Unsigned16') -> None:
    # Generate the 10-byte ascii key using mostly the high 64 bits.
    temp = make_big_integer(rand.get_high_8_bytes())
    for i in range(8):
        rec_buf[i] = temp % 95 + ord_space
        temp //= 95

    temp = make_big_integer(rand.get_low_8_bytes())
    rec_buf[8] = temp % 95 + ord_space
    temp //= 95
    rec_buf[9] = temp % 95 + ord_space

    # Add 2 bytes of "break" data
    rec_buf[10] = ord_space
    rec_buf[11] = ord_space

    # convert the 128-bit record number to 32 bits of ascii hexadecimal as the next 32 bytes of the record.
    for i in range(32):
        rec_buf[12 + i] = record_number.get_hex_digit(i)

    # Add 2 bytes of "break" data
    rec_buf[44] = ord_space
    rec_buf[45] = ord_space

    # Add 52 bytes of filler based on low 48 bits of random number
    for i in range(13):
        rec_buf[46 + i * 4] = rec_buf[47 + i * 4] = rec_buf[48 + i * 4] = rec_buf[49 + i * 4] = (
            rand.get_hex_digit(19 + i))

    # Add 2 bytes of "break" data
    rec_buf[98] = ord_cr
    rec_buf[99] = ord_newline

teragen/records_generators.py
gfinol/teragen-lithops
class Unsigned16:
    MAX_VALUE = 0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF

    def __init__(self, input_value: [int, str, 'Unsigned16'] = 0):
        if isinstance(input_value, int):
            self.value = input_value

        elif isinstance(input_value, str):
            self.set(input_value)

        elif isinstance(input_value, Unsigned16):
            self.value = input_value.value

        else:
            raise ValueError(f"Invalid value {input_value}")

    def get_value(self):
        return self._value

    def set_value(self, value):
        self._value = value & Unsigned16.MAX_VALUE

    def del_value(self):
        del self._value

    value = property(get_value, set_value, del_value, "Unsigned16 value")

    def set(self, s: str):
        self.value = int(s, 16)

    def __str__(self):
        return format(self.value, '032x')

    def set_long(self, long):
        self.value = long

    def get_byte(self, b: int) -> int:
        if 0 <= b < 16:
            return (self.value >> (120 - 8 * b)) & 0xFF
        raise ValueError(f"Invalid byte number {b}")

    # Avoid recomputing ord('0') and ord('A') every time
    ord0 = ord('0')
    ordA = ord('A')

    def get_hex_digit(self, p: int) -> int:
        digit = self.get_byte(p // 2)
        if p % 2 == 0:
            digit >>= 4
        digit &= 0xf
        if digit < 10:
            return self.ord0 + digit
        else:
            return self.ordA + digit - 10

    def multiply(self, b: 'Unsigned16') -> None:
        self.value *= b.value

    def add(self, b: 'Unsigned16') -> None:
        self.value += b.value

    def shift_left(self, bits: int) -> None:
        self.value <<= bits

    def get_high_8_bytes(self) -> int:
        return self.value >> 64

    def get_low_8_bytes(self) -> int:
        return self.value & 0xFFFFFFFFFFFFFFFF

# class Unsigned16:
#     def __init__(self, value=0):
#         if isinstance(value, int):
#             self.hi8 = np.int64(0)
#             self.lo8 = np.int64(value)
#         elif isinstance(value, str):
#             self.set(value)
#         else:
#             self.hi8 = value.hi8
#             self.lo8 = value.lo8
#
#     def set(self, s):
#         self.hi8 = np.int64(0)
#         self.lo8 = np.int64(0)
#         lastDigit = np.int64(0xF) << 60
#         for ch in s:
#             digit = self.get_hex_digit(ch)
#             if (lastDigit & self.hi8) != 0:
#                 raise ValueError(f"{s} overflowed 16 bytes")
#             self.hi8 <<= 4
#             self.hi8 |= rshift((self.lo8 & lastDigit), 60)
#             self.lo8 <<= 4
#             self.lo8 |= digit
#
#     def set_long(self, l):
#         self.lo8 = np.int64(l)
#         self.hi8 = np.int64(0)
#
#     def __str__(self):
#         if self.hi8 == 0:
#             return hex(self.lo8)
#         else:
#             result = hex(twos_complement(self.hi8)) + format(twos_complement(self.lo8), '016x')
#             return result
#
#     def __eq__(self, other):
#         if isinstance(other, Unsigned16):
#             return self.hi8 == other.hi8 and self.lo8 == other.lo8
#         else:
#             return False
#
#     def get_byte(self, b):
#         if 0 <= b < 16:
#             if b < 8:
#                 return (self.hi8 >> (56 - 8 * b)) & 0xFF
#             else:
#                 return (self.lo8 >> (120 - 8 * b)) & 0xFF
#         return 0
#
#     def get_hex_digit(self, ch):
#         if isinstance(ch, str):
#             digit = ord(ch)
#             if ord('0') <= digit <= ord('9'):
#                 return digit - ord('0')
#             if ord('a') <= digit <= ord('f'):
#                 return digit - ord('a') + 10
#             if ord('A') <= digit <= ord('F'):
#                 return digit - ord('A') + 10
#             raise ValueError(f"Invalid hex digit {ch}")
#         elif isinstance(ch, int):
#             p = ch
#             digit = self.get_byte(p // 2)
#             if p % 2 == 0:
#                 digit >>= 4
#             digit &= 0xf
#             if digit < 10:
#                 return chr(ord('0') + digit)
#             else:
#                 return chr(ord('A') + digit - 10)
#
#
#     def getHigh8(self):
#         return self.hi8
#
#     def getLow8(self):
#         return self.lo8
#
#     def multiply(self, b):
#         left = [0] * 4
#         left[0] = self.lo8 & 0xFFFFFFFF
#         left[1] = rshift(self.lo8, 32)
#         left[2] = self.hi8 & 0xFFFFFFFF
#         left[3] = rshift(self.hi8, 32)
#
#         right = [0] * 5
#         right[0] = b.lo8 & 0x7FFFFFFF
#         right[1] = rshift(b.lo8, 31) & 0x7FFFFFFF
#         right[2] = rshift(b.lo8, 62) + ((b.hi8 & 0x1FFFFFFF) << 2)
#         right[3] = rshift(b.hi8, 29) & 0x7FFFFFFF
#         right[4] = rshift(b.hi8, 60)
#
#         self.set_long(0)
#         tmp = Unsigned16()
#         for l in range(4):
#             for r in range(5):
#                 prod = left[l] * right[r]
#                 if prod != 0:
#                     off = l * 32 + r * 31
#                     tmp.set_long(prod)
#                     tmp.shift_left(off)
#                     self.add(tmp)
#
#     def add(self, b):
#         sumHi = self.hi8 + b.hi8
#         hibit0 = np.int64(self.lo8 < 0)
#         hibit1 = np.int64(b.lo8 < 0)
#         sumLo = self.lo8 + b.lo8
#         reshibit = np.int64(sumLo < 0)
#         if (hibit0 & hibit1) != 0 or ((hibit0 ^ hibit1) != 0 and reshibit == 0):
#             sumHi += 1  # add carry bit
#         self.hi8 = sumHi
#         self.lo8 = sumLo
#
#     def shift_left(self, bits):
#         if bits != 0:
#             if bits < 64:
#                 self.hi8 <<= bits
#                 self.hi8 |= (self.lo8 >> (64 - bits))
#                 self.lo8 <<= bits
#             elif bits < 128:
#                 self.hi8 = self.lo8 << (bits - 64)
#                 self.lo8 = 0
#             else:
#                 self.hi8 = 0
#                 self.lo8 = 0

teragen/unsigned16.py
gfinol/teragen-lithops