# This script is the first script.
# It uses srcomapi (https://github.com/blha303/srcomapi) to scrape speedrun.com and identify all of the runs that must be recreated
# To install srcomapi follow the directions at the above link.

# This creates (with some user interaction..) a csv (01-runs.csv) that lists all of the runs and relevant information for the next step.

import srcomapi
import srcomapi.datatypes as dt

TIME_CUTOFF_MS = 300000
OUTPUT = "01-runs.csv"

PLATFORM_FPS = {
    "jm95z9olpr184lqn": "60.09881",
    "jm95z9olo316x197": "60.09881",
    "7m6ylw9ppr184lqn": "60.00000",
    "7g6mym6rpr184lqn": "60.09881",
    "kz9wyneppr184lqn": "60.00000",
    "nzelreqppr184lqn": "59.82400",
    "gde3rwekpr184lqn": "60.09881",
    "mr6k409zo316x197": "60.09881",
    "7m6ylw9po316x197": "60.00000"
}

api = srcomapi.SpeedrunCom()
smb = api.search(srcomapi.datatypes.Game, {"name": "super mario bros."})[0]
anyp = smb.categories[0]

lb = dt.Leaderboard(api, 
        data=api.get("leaderboards/{}/category/{}?embed=variables".format(smb.id, anyp.id)))

lines = []
seen = set()

try:
    with open(OUTPUT, "r") as f:
        f.readline()
        for line in f.readlines():
            lines.append(line)
            seen.add(line.split(",")[0])
except FileNotFoundError:
    pass



with open (OUTPUT, "w") as f:
    f.write("src_run_id,src_platform_id,src_region_id,display_name,date_played,elapsed_ms,platform_fps,video_url\n")
    for line in lines:
        f.write(line)

    for r in lb.runs:
        srcrun = r["run"]
        src_run_id = srcrun.id
        elapsed_ms = srcrun.times["primary_t"] * 1000

        if src_run_id in seen:
            continue

        if elapsed_ms > TIME_CUTOFF_MS:
            continue

        src_platform_id = srcrun.system["platform"]
        src_region_id = srcrun.system["region"]
        if src_region_id is None:
            src_region_id = "pr184lqn" # this is a hold-over from 2021 that I don't remember why.

        platform = src_platform_id + src_region_id
        platform_fps = PLATFORM_FPS[platform]

        display_name = srcrun.players[0].name
        date_played = srcrun.date

        url = None
        if srcrun.videos is not None:
            if "links" in srcrun.videos:
                urls = []
                for l in srcrun.videos["links"]:
                    urls.append(l["uri"])

                if len(urls) == 1:
                    url = urls[0]
                else:
                    print(f"CHOOSE BEST VIDEO: www.speedrun.com/smb1/run/{src_run_id}")
                    for i, turl in enumerate(urls):
                        print(i, turl)

                    l = input("index: ")
                    url = urls[int(l)]

        if url is not None:
            while url.find('&') != -1:
                print(url)
                print("WARNING URL CONTAINS AN '&', enter updated url")
                url = input("url: ")


        print(f"{src_run_id},{src_platform_id},{src_region_id},{display_name},{date_played},{elapsed_ms},{platform_fps},{url}\n")
        f.write(f"{src_run_id},{src_platform_id},{src_region_id},{display_name},{date_played},{elapsed_ms},{platform_fps},{url}\n")


src/01-scrapesrcom.py
RetroTournaments/SMB-VLB-Update
# This script is the second script.
# Because some of the runs are already done (their fm2 files are already in the repo) the timestamps and video information do not need to be defined.
# This filters 01-runs.csv and creates 02-to-crop.csv, which goes to the google sheet and discord for manual verification and time stamp definition.

INPUT = "01-runs.csv"
OUTPUT = "02-to-crop.csv"
FM2_DIRECTORY = "../fm2/"

from pathlib import Path

seen = set()

fm2_dir = Path(FM2_DIRECTORY)
for f in fm2_dir.glob("*.fm2"):
    assert(len(f.stem) == 8)
    seen.add(f.stem)

with open(OUTPUT, "w") as fout:
    with open(INPUT, "r") as fin:
        fout.write(fin.readline())

        for line in fin.readlines():
            srcid = line.split(",")[0]
            if srcid not in seen:
                fout.write(line)


src/02-filter-done.py
RetroTournaments/SMB-VLB-Update
# This script is the third script.
# Unfortunately I need to download all the raw video files, it is ~17GB! 
# Don't worry they are not tracked.

import subprocess
from pathlib import Path

INPUT = "01-runs.csv"
FM2_DIRECTORY = "../fm2/"
RAW_DIRECTORY = "../raw/"

seen = set()

fm2_dir = Path(FM2_DIRECTORY)
for f in fm2_dir.glob("*.fm2"):
    assert(len(f.stem) == 8)
    seen.add(f.stem)

raw_dir = Path(RAW_DIRECTORY)
raw_dir.mkdir(parents=True, exist_ok=True)

for f in raw_dir.glob("*"):
    src_run_id = f.stem
    assert(len(f.stem) == 8)
    seen.add(f.stem)

with open(INPUT, "r") as fin:
    fin.readline()

    for line in fin.readlines():
        vs = line.strip().split(",")
        src_run_id = vs[0]
        url = vs[-1]

        if src_run_id not in seen:
            print(src_run_id)
            subprocess.call(["yt-dlp", url, "-o", f"{RAW_DIRECTORY}/{src_run_id}.%(ext)s"])
            

src/03-download-raw.py
RetroTournaments/SMB-VLB-Update
# This script is the fourth script.
# After the raw videos are downloaded it is necessary to define where the game footage is for each video.
# For this I used 'carbon' from the graphite repository in the following fashion.
# I did them all myself because I didn't want to get other people set up doing it. Yay.

import subprocess
from pathlib import Path

CARBON_PATH = "/home/matthew/repos/graphite/build/carbon/carbon"
RAW_DIRECTORY = "../raw/"
OUTPUT = "04-crop.csv"

lines = []
seen = set()

try:
    with open(OUTPUT, "r") as f:
        f.readline()
        for line in f.readlines():
            lines.append(line)
            src_run_id, _ = line.strip().split(",")
            seen.add(src_run_id)
except FileNotFoundError:
    with open(OUTPUT, "w") as f:
        f.write("src_run_id,filter\n")
    pass

with open(OUTPUT, "a") as fout:
    raw_dir = Path(RAW_DIRECTORY)
    for f in raw_dir.glob("*"):
        if len(f.stem) != 8:
            continue

        if f.stem not in seen:
            subprocess.call([CARBON_PATH, f, "filter.txt"])
            with open("filter.txt", "r") as flt:
                fltr = flt.read().strip()

            if fltr.startswith("none"):
                break

            fout.write(f"{f.stem},{fltr}\n")

for f in ["filter.txt", "imgui.ini", "carbon.log"]:
    p = Path(f)
    p.unlink(missing_ok=True)


src/04-define-crop.py
RetroTournaments/SMB-VLB-Update
# This script is the fifth script.
# This handles all of the straightforward 'crop' videos.
# Uses ffmpeg.

import subprocess
from size_runs import *

def crop_vid(in_path, out_path, crop, timestamp_start, timestamp_end):
    vs = crop.split("_")
    cx, cy, cw, ch = [int(round(float(v))) for v in vs[3:7]]

    px = 0
    py = 0
    pw = 'iw'
    ph = 'ij'

    if cx < 0:
        pw = f'max(iw + {-cx}\\, {cw})'
        px = -cx
        cx = 0
    else:
        pw = f'max(iw\\, {cx + cw})'

    if cy < 0:
        ph = f'max(ih + {-cy}\\, {ch})'
        py = -cy
        cy = 0
    else:
        ph = f'max(ih\\, {cy + ch})'

    pad = f'pad={pw}:{ph}:{px}:{py}'
    crop = f'crop={cw}:{ch}:{cx}:{cy}'

    vf = f"{pad},{crop},scale=256:240:flags=area,setsar=1:1"

    args = ['ffmpeg', '-i', in_path, '-ss', timestamp_start, '-to', timestamp_end, "-vf", vf, "-vcodec", "libx264", "-crf", "12", out_path]

    print(" ".join(args))
    subprocess.call(args)

process_crop("crop", crop_vid)


src/05-do-crop.py
RetroTournaments/SMB-VLB-Update
# This script is the sixth script.
# This handles the perspective transform videos.
# Uses ffmpeg.

import subprocess
from size_runs import *

def warp_vid(in_path, out_path, crop, timestamp_start, timestamp_end):
    vs = crop.split("_")
    cx, cy, cw, ch = [int(round(float(v))) for v in vs[3:7]]
    nquad = []
    for i in range(7, 15, 2):
        nquad.append([float(vs[i]), float(vs[i+1])])
    pts = ""
    for x, y in nquad:
        pts += str(x) + ":" + str(y) + ":"

    persp = f'perspective={pts}'

    px = 0
    py = 0
    pw = 'iw'
    ph = 'ij'

    if cx < 0:
        pw = f'max(iw + {-cx}\\, {cw})'
        px = -cx
        cx = 0
    else:
        pw = f'max(iw\\, {cx + cw})'

    if cy < 0:
        ph = f'max(ih + {-cy}\\, {ch})'
        py = -cy
        cy = 0
    else:
        ph = f'max(ih\\, {cy + ch})'

    pad = f'pad={pw}:{ph}:{px}:{py}'
    crop = f'crop={cw}:{ch}:{cx}:{cy}'

    vf = f"{persp},{pad},{crop},scale=256:240:flags=area,setsar=1:1"

    args = ['ffmpeg', '-i', in_path, '-ss', timestamp_start, '-to', timestamp_end, "-vf", vf, "-vcodec", "libx264", "-crf", "12", out_path]

    print(" ".join(args))
    subprocess.call(args)

process_crop("perspective", warp_vid)

src/06-do-perspective.py
RetroTournaments/SMB-VLB-Update
# This script is the seventh script.
# This handles those nasty recordings that are of a curved CRT.
# Uses uncrt from the graphite directory, and ffmpeg

import subprocess
from size_runs import *

VCODEC = " -vcodec libx264 -crf 12 "
UNCRT_PATH = "/home/matthew/repos/graphite/build/rgmvideo/uncrt"

def uncrt_vid(in_path, out_path, crop, timestamp_start, timestamp_end):
    vs = crop.split("_")

    npatch = []
    for i in range(15, 15 + 16 * 2, 2):
        npatch.append([float(vs[i]), float(vs[i+1])])

    pts = ""
    for x, y in npatch:
        pts += str(x) + ":" + str(y) + ":"

    src_run_id = "xxxxxxxx"
    vp = in_path

    frames_dir = f"{src_run_id}_frames/"
    print(f'mkdir {frames_dir}')
    print(f'ffmpeg -i {vp} -ss {timestamp_start} -to {timestamp_end} {frames_dir}%06d.png')
    print(f'ffmpeg -i {vp} -ss {timestamp_start} -to {timestamp_end} -vn -acodec copy {src_run_id}_audio.mkv')
    print(f'{UNCRT_PATH} --mesh "{pts}" --inplace --directory {frames_dir}')
    print(f'FPS=$(ffprobe -v 0 -of csv=p=0 -select_streams v:0 -show_entries stream=r_frame_rate {vp})')
    print(f'echo "{src_run_id}: $FPS"')
    print(f'ffmpeg -r $FPS -f image2 -i {frames_dir}%06d.png -i {src_run_id}_audio.mkv {VCODEC} {out_path}')
    print(f'rm -rf {frames_dir}')
    print(f'rm -f {src_run_id}_audio.mkv')

process_crop("uncrt", uncrt_vid)

src/07-do-uncrt.py
RetroTournaments/SMB-VLB-Update
# 
from pathlib import Path
import re


RUNS = "01-runs.csv"
RAW_PATH = Path("../raw/")
TS = "05-timestamps.csv"

known = set()

def valid_timestamp(ts):
    return re.search("^\d\d:\d\d:\d\d$", ts) is not None


with open(TS, "r") as f:
    f.readline()
    for line in f.readlines():
        src_run_id, start, end = line.strip().split(",")
        known.add(src_run_id)

to_add_timestamps = set()

for f in RAW_PATH.glob("*"):
    src_run_id = f.stem
    assert(len(f.stem) == 8)

    if src_run_id not in known:
        to_add_timestamps.add(src_run_id)


with open(RUNS, "r") as f:
    f.readline()
    for line in f.readlines():
        toks = line.strip().split(",")
        src_run_id = toks[0]
        if src_run_id not in to_add_timestamps:
            continue
        url = toks[-1]
        print(src_run_id, ": ", url)
        start = ""
        while not valid_timestamp(start):
            start = input("start 'hh:mm:ss': ")
        end = ""
        while not valid_timestamp(end):
            end = input("end   'hh:mm:ss': ")


        with open(TS, "a") as o:
            o.write(f"{src_run_id},{start},{end}\n")









src/08-manual-timestamps.py
RetroTournaments/SMB-VLB-Update
#! python3
# process a single youtube video out of system as some people ask for.
import sys
import re
import subprocess
from pathlib import Path

CARBON_PATH = "/home/matthew/repos/graphite/build/carbon/carbon"

def valid_timestamp(ts):
    return re.search("^\d\d:\d\d:\d\d$", ts) is not None

if __name__ == '__main__':
    argv = sys.argv[1:]
    n = len(argv)
    if n < 2 or n > 4:
        print("usage: process_single.py src_run_id url [timestamp_start] [timestamp_end]")
        exit()

    src_run_id, url = argv[0:2]
    if len(src_run_id) != 8:
        print(f"error: len(src_run_id) != 8, '{src_run_id}'")
        exit()

    start = ""
    end = ""

    if n >= 3:
        start = argv[2]
    if n >= 4:
        end = argv[3]

    while not valid_timestamp(start):
        start = input("start 'hh:mm:ss': ")
    while not valid_timestamp(end):
        end = input("end   'hh:mm:ss': ")


    print(f"./process_single.py {src_run_id} '{url}' {start} {end}")

    subprocess.call(["yt-dlp", url, "-o", f"{src_run_id}_raw.%(ext)s"])
    f = list(Path(".").glob(f"{src_run_id}_raw*"))[0]
    subprocess.call([CARBON_PATH, f, "filter.txt"])
    with open("filter.txt", "r") as flt:
        crop = flt.read().strip()

    vs = crop.split("_")
    cx, cy, cw, ch = [int(round(float(v))) for v in vs[3:7]]

    px = 0
    py = 0
    pw = 'iw'
    ph = 'ij'

    if cx < 0:
        pw = f'max(iw + {-cx}\\, {cw})'
        px = -cx
        cx = 0
    else:
        pw = f'max(iw\\, {cx + cw})'

    if cy < 0:
        ph = f'max(ih + {-cy}\\, {ch})'
        py = -cy
        cy = 0
    else:
        ph = f'max(ih\\, {cy + ch})'

    pad = f'pad={pw}:{ph}:{px}:{py}'
    crop = f'crop={cw}:{ch}:{cx}:{cy}'

    vf = f"{pad},{crop},scale=256:240:flags=area,setsar=1:1"

    args = ['ffmpeg', '-i', str(f), '-ss', start, '-to', end, "-vf", vf, "-vcodec", "libx264", "-crf", "12", f"{src_run_id}.mp4"]

    print(" ".join(args))
    subprocess.call(args)
    




    



src/process_single.py
RetroTournaments/SMB-VLB-Update
# this small file provides helpers for do-crop, do-perspective, and do-uncrt

from pathlib import Path

TS = "05-timestamps.csv"
RAW_DIRECTORY = "../raw/"
VID_DIRECTORY = "../vid/"
CROP = "04-crop.csv"

def get_seen():
    seen = set()

    raw_dir = Path(RAW_DIRECTORY)

    vid_dir = Path(VID_DIRECTORY)
    vid_dir.mkdir(parents=True, exist_ok=True)
    for f in vid_dir.glob("*"):
        src_run_id = f.stem
        assert(len(f.stem) == 8)
        seen.add(f.stem)

    return seen

def get_timestamps():
    timestamps = {}
    with open(TS, "r") as f:
        f.readline()
        for line in f.readlines():
            src_run_id, start, end = line.strip().split(",")
            timestamps[src_run_id] = (start, end)

    return timestamps

def process_crop(prefix, func):
    seen = get_seen()
    timestamps = get_timestamps()

    raw_dir = Path(RAW_DIRECTORY)

    with open(CROP, "r") as f:
        f.readline()

        for line in f.readlines():
            src_run_id, crop = line.strip().split(",")
            if src_run_id in seen:
                continue
            if not crop.startswith(prefix):
                continue

            in_path = str(list(raw_dir.glob(f"{src_run_id}*"))[0])
            out_path = f"{VID_DIRECTORY}{src_run_id}.mp4"

            timestamp_start, timestamp_end = timestamps[src_run_id]

            func(in_path, out_path, crop, timestamp_start, timestamp_end)

src/size_runs.py
RetroTournaments/SMB-VLB-Update
#! python3
# import overtop of VLB-2023/Data (https://docs.google.com/spreadsheets/d/1_GwGMzxJuHCW60huQASQmNDknZlJMZCWh4LCEjRxUy0/edit)

from itertools import zip_longest
from pathlib import Path

FM2_DIR = "fm2/"
VID_DIR = "vid/"
OUT = "data.csv"

done = []
vids = []

for f in Path(FM2_DIR).glob("*.fm2"):
    assert(len(f.stem) == 8)
    done.append(f.stem)

for f in Path(VID_DIR).glob("*.mp4"):
    assert(len(f.stem) == 8)
    vids.append(f.stem)

with open(OUT, "w") as f:
    f.write("DONE,VIDEO_READY\n")
    for fm2, vid in zip_longest(done, vids):
        f.write(f"{fm2},{vid}\n")




update-sheet.py
RetroTournaments/SMB-VLB-Update
#! python3

from pathlib import Path
import shutil
import subprocess

SUBMIT_DIR = "fm2-submissions/"
FM2_DIR = "fm2/"
VID_DIR = "vid/"

seen = set()
for f in Path(FM2_DIR).glob("*.fm2"):
    assert(len(f.stem) == 8)
    seen.add(f.stem)

for f in Path(SUBMIT_DIR).glob("*.fm2"):
    src_id = f.stem[:8]
    if src_id in seen:
        continue

    print(src_id, f)

    shutil.copy(str(f), FM2_DIR + src_id + ".fm2")
    subprocess.call(["/home/matthew/repos/graphite/build/graphite/graphite", "/home/matthew/repos/rgms/data/roms/SMB.nes",
                    f"/home/matthew/repos/SMB-VLB-Update/vid/{src_id}.mp4", f"/home/matthew/repos/SMB-VLB-Update/fm2/{src_id}.fm2"])


for f in ["graphite.log", "graphite.json"]:
    p = Path(f)
    p.unlink(missing_ok=True)


verify-submissions.py
RetroTournaments/SMB-VLB-Update