import timer
import sys
import utils

# A set of files already in the storage
seen = set()

# The total number of uploads
total_uploads = 0

# The number of files in the storage
files_in = 0

tmr = timer.Timer()
for (hsh, _) in utils.read_upload_stream():
    if hsh not in seen:
        files_in += 1
        seen.add(hsh)
    total_uploads += 1

    if total_uploads % utils.REPORT_FREQUENCY == 0:
        print("%s uploads, percentage %.4f, time %s, %s" % (
            utils.num_fmt(total_uploads),
            1 - files_in / total_uploads,
            tmr.elapsed_str,
            utils.get_mem_info()
        ), file=sys.stderr)

dedup_percentage = 1 - files_in / total_uploads
print("+++ Simulation complete. dedup_percentage=%f" % dedup_percentage,
      file=sys.stderr)

import timer
import sys
import utils


def simulate():
    # A set of files already in the storage
    seen = set()

    # The size of the all uploads combined (deduplicated or not)
    total_in = 0

    # The size of the data sent to the service
    data_in = 0

    tmr = timer.Timer()
    for (i, (hsh, size)) in enumerate(utils.read_upload_stream()):
        total_in += size
        if hsh not in seen:
            data_in += size
            seen.add(hsh)

        if (i + 1) % utils.REPORT_FREQUENCY == 0:
            print("%s uploads, percentage %.4f, time %s, mem[%s]" % (
                utils.num_fmt(i),
                1 - data_in / total_in,
                tmr.elapsed_str,
                utils.get_mem_info()
            ), file=sys.stderr)

    dedup_percentage = 1 - data_in / total_in
    print("Simulation complete. stored=%s, uploaded=%s, dedup_percentage=%f" % (
        utils.sizeof_fmt(data_in), utils.sizeof_fmt(total_in), dedup_percentage),
        file=sys.stderr)

if __name__ == "__main__":
    simulate()

Modify the perfect simulator to calculate dedup percentages based on file sizes (1 - data_in / data_total)
Modify the perfect simulator to calculate dedup percentages based on file sizes (1 - data_in / data_total)

  import timer
  import sys
  import utils
  
- # A set of files already in the storage
- seen = set()
  
- # The total number of uploads
- total_uploads = 0
+ def simulate():
+     # A set of files already in the storage
+     seen = set()
  
- # The number of files in the storage
- files_in = 0
+     # The size of the all uploads combined (deduplicated or not)
+     total_in = 0
  
+     # The size of the data sent to the service
+     data_in = 0
- tmr = timer.Timer()
- for (hsh, _) in utils.read_upload_stream():
-     if hsh not in seen:
-         files_in += 1
-         seen.add(hsh)
-     total_uploads += 1
  
+     tmr = timer.Timer()
+     for (i, (hsh, size)) in enumerate(utils.read_upload_stream()):
+         total_in += size
+         if hsh not in seen:
+             data_in += size
+             seen.add(hsh)
-     if total_uploads % utils.REPORT_FREQUENCY == 0:
-         print("%s uploads, percentage %.4f, time %s, %s" % (
-             utils.num_fmt(total_uploads),
-             1 - files_in / total_uploads,
-             tmr.elapsed_str,
-             utils.get_mem_info()
-         ), file=sys.stderr)
  
- dedup_percentage = 1 - files_in / total_uploads
- print("+++ Simulation complete. dedup_percentage=%f" % dedup_percentage,
+         if (i + 1) % utils.REPORT_FREQUENCY == 0:
+             print("%s uploads, percentage %.4f, time %s, mem[%s]" % (
+                 utils.num_fmt(i),
+                 1 - data_in / total_in,
+                 tmr.elapsed_str,
+                 utils.get_mem_info()
-       file=sys.stderr)
+             ), file=sys.stderr)
  
+     dedup_percentage = 1 - data_in / total_in
+     print("Simulation complete. stored=%s, uploaded=%s, dedup_percentage=%f" % (
+         utils.sizeof_fmt(data_in), utils.sizeof_fmt(total_in), dedup_percentage),
+         file=sys.stderr)
+ 
+ if __name__ == "__main__":
+     simulate()
+ 
Modify the perfect simulator to calculate dedup percentages based on file sizes (1 - data_in / data_total)
## Code Before:
import timer
import sys
import utils

# A set of files already in the storage
seen = set()

# The total number of uploads
total_uploads = 0

# The number of files in the storage
files_in = 0

tmr = timer.Timer()
for (hsh, _) in utils.read_upload_stream():
    if hsh not in seen:
        files_in += 1
        seen.add(hsh)
    total_uploads += 1

    if total_uploads % utils.REPORT_FREQUENCY == 0:
        print("%s uploads, percentage %.4f, time %s, %s" % (
            utils.num_fmt(total_uploads),
            1 - files_in / total_uploads,
            tmr.elapsed_str,
            utils.get_mem_info()
        ), file=sys.stderr)

dedup_percentage = 1 - files_in / total_uploads
print("+++ Simulation complete. dedup_percentage=%f" % dedup_percentage,
      file=sys.stderr)

## Instruction:
Modify the perfect simulator to calculate dedup percentages based on file sizes (1 - data_in / data_total)
## Code After:
import timer
import sys
import utils


def simulate():
    # A set of files already in the storage
    seen = set()

    # The size of the all uploads combined (deduplicated or not)
    total_in = 0

    # The size of the data sent to the service
    data_in = 0

    tmr = timer.Timer()
    for (i, (hsh, size)) in enumerate(utils.read_upload_stream()):
        total_in += size
        if hsh not in seen:
            data_in += size
            seen.add(hsh)

        if (i + 1) % utils.REPORT_FREQUENCY == 0:
            print("%s uploads, percentage %.4f, time %s, mem[%s]" % (
                utils.num_fmt(i),
                1 - data_in / total_in,
                tmr.elapsed_str,
                utils.get_mem_info()
            ), file=sys.stderr)

    dedup_percentage = 1 - data_in / total_in
    print("Simulation complete. stored=%s, uploaded=%s, dedup_percentage=%f" % (
        utils.sizeof_fmt(data_in), utils.sizeof_fmt(total_in), dedup_percentage),
        file=sys.stderr)

if __name__ == "__main__":
    simulate()
