add release dropdown

.github/scripts/check_dataset_update.py

@@ -27,8 +27,8 @@ def check_dataset_updates(dataset_id):
             {
                 "last_modified": last_modified,
                 "sha": current_sha,
-                "releases": ["a9b92c4"],
-                "whisperkit_version": "0.9.1",
+                "releases": ["a9b92c4", "112a023"],
+                "whisperkit_version": "0.10.1",
             },
             f,
         )

constants.py

@@ -9,6 +9,12 @@ BANNER_TEXT = """
 """
 
 
+SHA_TO_VERSION = {
+    "a9b92c4": "0.9.1",
+    "112a023": "0.10.1",
+}
+
+
 INTRO_LABEL = """We present comprehensive benchmarks for WhisperKit, our on-device ASR solution, compared against a reference implementation. These benchmarks aim to help developers and enterprises make informed decisions when choosing optimized or compressed variants of machine learning models for production use. Show more."""
 
 

dashboard_data/device_map.json

@@ -6,6 +6,8 @@
     "Mac15,3": "Apple M3",
     "Mac15,6": "Apple M3 Pro",
     "Mac15,9": "Apple M3 Max",
+    "Mac16,10": "Apple M4",
+    "Mac16,3": "Apple M4",
     "MacBookAir10,1": "Apple M1",
     "iPad13,16": "iPad Air (5th generation)",
     "iPad14,8": "iPad Air 11-inch (M2)",

dashboard_data/support_data.csv

@@ -1,23 +1,23 @@
-,Model,"Apple M2 Pro (Mac14,12)","Apple M2 Ultra (Mac14,14)","Apple M2 (Mac14,2)","Apple M3 Max (Mac15,10)","Apple M3 Pro (Mac15,6)","Apple M3 Max (Mac15,9)","Apple M1 (MacBookAir10,1)","iPad Air (5th generation) (iPad13,16)","iPad Air 11-inch (M2) (iPad14,8)","iPad mini (A17 Pro) (iPad16,1)","iPad Pro 11-inch (M4) (iPad16,3)","iPhone 11 (iPhone12,1)","iPhone 12 mini (iPhone13,1)","iPhone 12 (iPhone13,2)","iPhone 12 Pro Max (iPhone13,4)","iPhone 13 Pro (iPhone14,2)","iPhone 13 (iPhone14,5)","iPhone 14 (iPhone14,7)","iPhone 15 Pro (iPhone16,1)","iPhone 15 Pro Max (iPhone16,2)","iPhone 16 Pro (iPhone17,1)","iPhone 16 (iPhone17,3)","iPhone 16 Plus (iPhone17,4)"
-distil-whisper_distil-large-v3,distil-whisper_distil-large-v3,✅ macOS 15.0.1,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/MacBookAir10%2C1_summary_2024-10-30T012535.json>macOS 15.1</a>,Not Supported,✅ iPadOS 17.6.1,Not Supported,✅ iPadOS 18.1,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported
-distil-whisper_distil-large-v3_594MB,distil-whisper_distil-large-v3_594MB,✅ macOS 15.0.1,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,Not Supported,✅ iPadOS 17.6.1,✅ iPadOS 18.0.1,✅ iPadOS 18.1,Not Supported,Not Supported,Not Supported,Not Supported,✅ iOS 18.1,✅ iOS 17.3,✅ iOS 17.3,✅ iOS 18.1,✅ iOS 18.2,✅ iOS 18.0,✅ iOS 18.0.1,✅ iOS 18.0.1
-distil-whisper_distil-large-v3_turbo,distil-whisper_distil-large-v3_turbo,✅ macOS 15.0.1,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,Not Supported,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPad14%2C8_summary_2024-10-30T073931.json>iPadOS 17.6.1</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPad16%2C1_summary_2024-10-29T234233.json>iPadOS 18.0.1</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPad16%2C3_summary_2024-10-30T073931.json>iPadOS 18.1</a>,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported
-distil-whisper_distil-large-v3_turbo_600MB,distil-whisper_distil-large-v3_turbo_600MB,✅ macOS 15.0.1,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,Not Supported,Not Supported,✅ iPadOS 17.6.1,✅ iPadOS 18.0.1,✅ iPadOS 18.1,Not Supported,Not Supported,Not Supported,Not Supported,✅ iOS 18.1,✅ iOS 17.3,✅ iOS 17.3,✅ iOS 18.1,✅ iOS 18.2,✅ iOS 18.0,✅ iOS 18.0.1,✅ iOS 18.0.1
-openai_whisper-base,openai_whisper-base,✅ macOS 15.0.1,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,✅ iPadOS 18.1,✅ iPadOS 17.6.1,✅ iPadOS 18.0.1,✅ iPadOS 18.1,✅ iOS 17.6.1,✅ iOS 17.7,✅ iOS 17.2.1,✅ iOS 18.1,✅ iOS 18.1,✅ iOS 17.3,✅ iOS 17.3,✅ iOS 18.1,✅ iOS 18.2,✅ iOS 18.0,✅ iOS 18.0.1,✅ iOS 18.0.1
-openai_whisper-base.en,openai_whisper-base.en,✅ macOS 15.0.1,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,✅ iPadOS 18.1,✅ iPadOS 17.6.1,✅ iPadOS 18.0.1,✅ iPadOS 18.1,✅ iOS 17.6.1,✅ iOS 17.7,✅ iOS 17.2.1,✅ iOS 18.1,✅ iOS 18.1,✅ iOS 17.3,✅ iOS 17.3,✅ iOS 18.1,✅ iOS 18.2,✅ iOS 18.0,✅ iOS 18.0.1,✅ iOS 18.0.1
-openai_whisper-large-v2,openai_whisper-large-v2,✅ macOS 15.0.1,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/MacBookAir10%2C1_summary_2024-10-30T012535.json>macOS 15.1</a>,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported
-openai_whisper-large-v2_949MB,openai_whisper-large-v2_949MB,✅ macOS 15.0.1,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,Not Supported,✅ iPadOS 17.6.1,✅ iPadOS 18.0.1,✅ iPadOS 18.1,Not Supported,Not Supported,Not Supported,Not Supported,✅ iOS 18.1,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPhone14%2C5_summary_2024-10-30T073931.json>iOS 17.3</a>,✅ iOS 17.3,✅ iOS 18.1,✅ iOS 18.2,✅ iOS 18.0,✅ iOS 18.0.1,✅ iOS 18.0.1
-openai_whisper-large-v2_turbo,openai_whisper-large-v2_turbo,✅ macOS 15.0.1,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,Not Supported,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPad14%2C8_summary_2024-10-30T073931.json>iPadOS 17.6.1</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPad16%2C1_summary_2024-10-29T234233.json>iPadOS 18.0.1</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPad16%2C3_summary_2024-10-30T073931.json>iPadOS 18.1</a>,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported
-openai_whisper-large-v2_turbo_955MB,openai_whisper-large-v2_turbo_955MB,✅ macOS 15.0.1,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,Not Supported,Not Supported,✅ iPadOS 17.6.1,✅ iPadOS 18.0.1,✅ iPadOS 18.1,Not Supported,Not Supported,Not Supported,Not Supported,✅ iOS 18.1,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPhone14%2C5_summary_2024-10-30T073931.json>iOS 17.3</a>,✅ iOS 17.3,✅ iOS 18.1,✅ iOS 18.2,✅ iOS 18.0,✅ iOS 18.0.1,✅ iOS 18.0.1
-openai_whisper-large-v3,openai_whisper-large-v3,✅ macOS 15.0.1,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/MacBookAir10%2C1_summary_2024-10-30T012535.json>macOS 15.1</a>,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported
-openai_whisper-large-v3-v20240930,openai_whisper-large-v3-v20240930,✅ macOS 15.0.1,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/MacBookAir10%2C1_summary_2024-10-30T012535.json>macOS 15.1</a>,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPad14%2C8_summary_2024-10-30T073931.json>iPadOS 17.6.1</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPad16%2C1_summary_2024-10-29T234233.json>iPadOS 18.0.1</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPad16%2C3_summary_2024-10-30T073931.json>iPadOS 18.1</a>,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported
-openai_whisper-large-v3-v20240930_626MB,openai_whisper-large-v3-v20240930_626MB,✅ macOS 15.0.1,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,Not Supported,✅ iPadOS 17.6.1,✅ iPadOS 18.0.1,✅ iPadOS 18.1,Not Supported,Not Supported,Not Supported,Not Supported,✅ iOS 18.1,✅ iOS 17.3,✅ iOS 17.3,✅ iOS 18.1,✅ iOS 18.2,✅ iOS 18.0,✅ iOS 18.0.1,✅ iOS 18.0.1
-openai_whisper-large-v3-v20240930_turbo,openai_whisper-large-v3-v20240930_turbo,✅ macOS 15.0.1,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,Not Supported,Not Supported,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPad16%2C1_summary_2024-10-29T234233.json>iPadOS 18.0.1</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPad16%2C3_summary_2024-10-30T073931.json>iPadOS 18.1</a>,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported
-openai_whisper-large-v3-v20240930_turbo_632MB,openai_whisper-large-v3-v20240930_turbo_632MB,✅ macOS 15.0.1,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,Not Supported,Not Supported,✅ iPadOS 17.6.1,✅ iPadOS 18.0.1,✅ iPadOS 18.1,Not Supported,Not Supported,Not Supported,Not Supported,✅ iOS 18.1,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPhone14%2C5_summary_2024-10-30T073931.json>iOS 17.3</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPhone14%2C7_summary_2024-10-30T073931.json>iOS 17.3</a>,✅ iOS 18.1,✅ iOS 18.2,✅ iOS 18.0,✅ iOS 18.0.1,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPhone17%2C4_summary_2024-10-30T132649.json>iOS 18.0.1</a>
-openai_whisper-large-v3_947MB,openai_whisper-large-v3_947MB,✅ macOS 15.0.1,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,Not Supported,✅ iPadOS 17.6.1,✅ iPadOS 18.0.1,✅ iPadOS 18.1,Not Supported,Not Supported,Not Supported,Not Supported,✅ iOS 18.1,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPhone14%2C5_summary_2024-10-30T073931.json>iOS 17.3</a>,✅ iOS 17.3,✅ iOS 18.1,✅ iOS 18.2,✅ iOS 18.0,✅ iOS 18.0.1,✅ iOS 18.0.1
-openai_whisper-large-v3_turbo,openai_whisper-large-v3_turbo,✅ macOS 15.0.1,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,Not Supported,Not Supported,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPad16%2C1_summary_2024-10-29T234233.json>iPadOS 18.0.1</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPad16%2C3_summary_2024-10-30T073931.json>iPadOS 18.1</a>,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported
-openai_whisper-large-v3_turbo_954MB,openai_whisper-large-v3_turbo_954MB,✅ macOS 15.0.1,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,Not Supported,Not Supported,✅ iPadOS 17.6.1,✅ iPadOS 18.0.1,✅ iPadOS 18.1,Not Supported,Not Supported,Not Supported,Not Supported,✅ iOS 18.1,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPhone14%2C5_summary_2024-10-30T073931.json>iOS 17.3</a>,✅ iOS 17.3,✅ iOS 18.1,✅ iOS 18.2,✅ iOS 18.0,✅ iOS 18.0.1,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPhone17%2C4_summary_2024-10-30T132649.json>iOS 18.0.1</a>
-openai_whisper-small,openai_whisper-small,✅ macOS 15.0.1,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,✅ iPadOS 18.1,✅ iPadOS 17.6.1,✅ iPadOS 18.0.1,✅ iPadOS 18.1,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPhone13%2C1_summary_2024-10-30T022739.json>iOS 17.7</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPhone13%2C2_summary_2024-10-30T152729.json>iOS 17.2.1</a>,✅ iOS 18.1,✅ iOS 18.1,✅ iOS 17.3,✅ iOS 17.3,✅ iOS 18.1,✅ iOS 18.2,✅ iOS 18.0,✅ iOS 18.0.1,✅ iOS 18.0.1
-openai_whisper-small.en,openai_whisper-small.en,✅ macOS 15.0.1,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,✅ iPadOS 18.1,✅ iPadOS 17.6.1,✅ iPadOS 18.0.1,✅ iPadOS 18.1,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPhone13%2C1_summary_2024-10-30T022739.json>iOS 17.7</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPhone13%2C2_summary_2024-10-30T152729.json>iOS 17.2.1</a>,✅ iOS 18.1,✅ iOS 18.1,✅ iOS 17.3,✅ iOS 17.3,✅ iOS 18.1,✅ iOS 18.2,✅ iOS 18.0,✅ iOS 18.0.1,✅ iOS 18.0.1
-openai_whisper-tiny,openai_whisper-tiny,✅ macOS 15.0.1,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,✅ iPadOS 18.1,✅ iPadOS 17.6.1,✅ iPadOS 18.0.1,✅ iPadOS 18.1,Not Supported,✅ iOS 17.7,✅ iOS 17.2.1,✅ iOS 18.1,✅ iOS 18.1,✅ iOS 17.3,✅ iOS 17.3,✅ iOS 18.1,✅ iOS 18.2,✅ iOS 18.0,✅ iOS 18.0.1,✅ iOS 18.0.1
-openai_whisper-tiny.en,openai_whisper-tiny.en,✅ macOS 15.0.1,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,✅ iPadOS 18.1,✅ iPadOS 17.6.1,✅ iPadOS 18.0.1,✅ iPadOS 18.1,✅ iOS 17.6.1,✅ iOS 17.7,✅ iOS 17.2.1,✅ iOS 18.1,✅ iOS 18.1,✅ iOS 17.3,✅ iOS 17.3,✅ iOS 18.1,✅ iOS 18.2,✅ iOS 18.0,✅ iOS 18.0.1,✅ iOS 18.0.1
+,Model,"Apple M2 Pro (Mac14,12)","Apple M2 Ultra (Mac14,14)","Apple M2 (Mac14,2)","Apple M3 Max (Mac15,10)","Apple M3 Pro (Mac15,6)","Apple M3 Max (Mac15,9)","Apple M4 (Mac16,10)","Apple M4 (Mac16,3)","Apple M1 (MacBookAir10,1)","iPad Air (5th generation) (iPad13,16)","iPad Air 11-inch (M2) (iPad14,8)","iPad mini (A17 Pro) (iPad16,1)","iPad Pro 11-inch (M4) (iPad16,3)","iPhone 11 (iPhone12,1)","iPhone 12 mini (iPhone13,1)","iPhone 12 (iPhone13,2)","iPhone 12 Pro Max (iPhone13,4)","iPhone 13 Pro (iPhone14,2)","iPhone 13 (iPhone14,5)","iPhone 14 (iPhone14,7)","iPhone 15 Pro (iPhone16,1)","iPhone 15 Pro Max (iPhone16,2)","iPhone 16 Pro (iPhone17,1)","iPhone 16 (iPhone17,3)","iPhone 16 Plus (iPhone17,4)"
+distil-whisper_distil-large-v3,distil-whisper_distil-large-v3,✅ macOS 15.0.1<p>✅ macOS 15.1.1</p>,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.0,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/MacBookAir10%2C1_summary_2024-10-30T012535.json>macOS 15.1</a>,Not Supported,✅ iPadOS 17.6.1,Not Supported,✅ iPadOS 18.1,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported
+distil-whisper_distil-large-v3_594MB,distil-whisper_distil-large-v3_594MB,✅ macOS 15.0.1<p>✅ macOS 15.1.1</p>,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.0,✅ macOS 15.1,Not Supported,✅ iPadOS 17.6.1,✅ iPadOS 18.0.1,✅ iPadOS 18.1,Not Supported,Not Supported,Not Supported,Not Supported,✅ iOS 18.1,✅ iOS 17.3,✅ iOS 17.3,✅ iOS 18.1,✅ iOS 18.2,✅ iOS 18.0,✅ iOS 18.0.1,✅ iOS 18.0.1
+distil-whisper_distil-large-v3_turbo,distil-whisper_distil-large-v3_turbo,✅ macOS 15.0.1<p>✅ macOS 15.1.1</p>,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.0,Not Supported,Not Supported,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPad16%2C1_summary_2024-10-29T234233.json>iPadOS 18.0.1</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPad16%2C3_summary_2024-10-30T073931.json>iPadOS 18.1</a>,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported
+distil-whisper_distil-large-v3_turbo_600MB,distil-whisper_distil-large-v3_turbo_600MB,✅ macOS 15.0.1<p>✅ macOS 15.1.1</p>,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.0,Not Supported,Not Supported,✅ iPadOS 17.6.1,✅ iPadOS 18.0.1,✅ iPadOS 18.1,Not Supported,Not Supported,Not Supported,Not Supported,✅ iOS 18.1,✅ iOS 17.3,✅ iOS 17.3,✅ iOS 18.1,✅ iOS 18.2,✅ iOS 18.0,✅ iOS 18.0.1,✅ iOS 18.0.1
+openai_whisper-base,openai_whisper-base,✅ macOS 15.0.1<p>✅ macOS 15.1.1</p>,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.0,✅ macOS 15.1,✅ iPadOS 18.1,✅ iPadOS 17.6.1,✅ iPadOS 18.0.1,✅ iPadOS 18.1,✅ iOS 17.6.1,✅ iOS 17.7,✅ iOS 17.2.1,✅ iOS 18.1,✅ iOS 18.1,✅ iOS 17.3,✅ iOS 17.3,✅ iOS 18.1,✅ iOS 18.2,✅ iOS 18.0,✅ iOS 18.0.1,✅ iOS 18.0.1
+openai_whisper-base.en,openai_whisper-base.en,✅ macOS 15.0.1<p>✅ macOS 15.1.1</p>,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.0,✅ macOS 15.1,✅ iPadOS 18.1,✅ iPadOS 17.6.1,✅ iPadOS 18.0.1,✅ iPadOS 18.1,✅ iOS 17.6.1,✅ iOS 17.7,✅ iOS 17.2.1,✅ iOS 18.1,✅ iOS 18.1,✅ iOS 17.3,✅ iOS 17.3,✅ iOS 18.1,✅ iOS 18.2,✅ iOS 18.0,✅ iOS 18.0.1,✅ iOS 18.0.1
+openai_whisper-large-v2,openai_whisper-large-v2,✅ macOS 15.0.1<p>✅ macOS 15.1.1</p>,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.0,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/MacBookAir10%2C1_summary_2024-10-30T012535.json>macOS 15.1</a>,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported
+openai_whisper-large-v2_949MB,openai_whisper-large-v2_949MB,✅ macOS 15.0.1<p>✅ macOS 15.1.1</p>,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.0,✅ macOS 15.1,Not Supported,✅ iPadOS 17.6.1,✅ iPadOS 18.0.1,✅ iPadOS 18.1,Not Supported,Not Supported,Not Supported,Not Supported,✅ iOS 18.1,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPhone14%2C5_summary_2024-10-30T073931.json>iOS 17.3</a>,✅ iOS 17.3,✅ iOS 18.1,✅ iOS 18.2,✅ iOS 18.0,✅ iOS 18.0.1,✅ iOS 18.0.1
+openai_whisper-large-v2_turbo,openai_whisper-large-v2_turbo,✅ macOS 15.0.1<p>✅ macOS 15.1.1</p>,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.0,Not Supported,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-12-21T053728_112a023/iPad14%2C8_summary_2024-12-21T170131.json>iPadOS 17.6.1</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPad16%2C1_summary_2024-10-29T234233.json>iPadOS 18.0.1</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPad16%2C3_summary_2024-10-30T073931.json>iPadOS 18.1</a>,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported
+openai_whisper-large-v2_turbo_955MB,openai_whisper-large-v2_turbo_955MB,✅ macOS 15.0.1<p>✅ macOS 15.1.1</p>,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.0,Not Supported,Not Supported,✅ iPadOS 17.6.1,✅ iPadOS 18.0.1,✅ iPadOS 18.1,Not Supported,Not Supported,Not Supported,Not Supported,✅ iOS 18.1,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPhone14%2C5_summary_2024-10-30T073931.json>iOS 17.3</a>,✅ iOS 17.3,✅ iOS 18.1,✅ iOS 18.2,✅ iOS 18.0,✅ iOS 18.0.1,✅ iOS 18.0.1
+openai_whisper-large-v3,openai_whisper-large-v3,✅ macOS 15.0.1<p>✅ macOS 15.1.1</p>,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.0,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/MacBookAir10%2C1_summary_2024-10-30T012535.json>macOS 15.1</a>,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported
+openai_whisper-large-v3-v20240930,openai_whisper-large-v3-v20240930,✅ macOS 15.0.1<p>✅ macOS 15.1.1</p>,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.0,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/MacBookAir10%2C1_summary_2024-10-30T012535.json>macOS 15.1</a>,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-12-21T053728_112a023/iPad14%2C8_summary_2024-12-21T170131.json>iPadOS 17.6.1</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPad16%2C1_summary_2024-10-29T234233.json>iPadOS 18.0.1</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPad16%2C3_summary_2024-10-30T073931.json>iPadOS 18.1</a>,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported
+openai_whisper-large-v3-v20240930_626MB,openai_whisper-large-v3-v20240930_626MB,✅ macOS 15.0.1<p>✅ macOS 15.1.1</p>,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.0,✅ macOS 15.1,Not Supported,✅ iPadOS 17.6.1,✅ iPadOS 18.0.1,✅ iPadOS 18.1,Not Supported,Not Supported,Not Supported,Not Supported,✅ iOS 18.1,✅ iOS 17.3,✅ iOS 17.3,✅ iOS 18.1,✅ iOS 18.2,✅ iOS 18.0,✅ iOS 18.0.1,✅ iOS 18.0.1
+openai_whisper-large-v3-v20240930_turbo,openai_whisper-large-v3-v20240930_turbo,✅ macOS 15.0.1<p>✅ macOS 15.1.1</p>,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.0,Not Supported,Not Supported,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPad16%2C1_summary_2024-10-29T234233.json>iPadOS 18.0.1</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPad16%2C3_summary_2024-10-30T073931.json>iPadOS 18.1</a>,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported
+openai_whisper-large-v3-v20240930_turbo_632MB,openai_whisper-large-v3-v20240930_turbo_632MB,✅ macOS 15.0.1<p>✅ macOS 15.1.1</p>,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.0,Not Supported,Not Supported,✅ iPadOS 17.6.1,✅ iPadOS 18.0.1,✅ iPadOS 18.1,Not Supported,Not Supported,Not Supported,Not Supported,✅ iOS 18.1,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPhone14%2C5_summary_2024-10-30T073931.json>iOS 17.3</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPhone14%2C7_summary_2024-10-30T073931.json>iOS 17.3</a>,✅ iOS 18.1,✅ iOS 18.2,✅ iOS 18.0,✅ iOS 18.0.1,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPhone17%2C4_summary_2024-10-30T132649.json>iOS 18.0.1</a>
+openai_whisper-large-v3_947MB,openai_whisper-large-v3_947MB,✅ macOS 15.0.1<p>✅ macOS 15.1.1</p>,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.0,✅ macOS 15.1,Not Supported,✅ iPadOS 17.6.1,✅ iPadOS 18.0.1,✅ iPadOS 18.1,Not Supported,Not Supported,Not Supported,Not Supported,✅ iOS 18.1,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPhone14%2C5_summary_2024-10-30T073931.json>iOS 17.3</a>,✅ iOS 17.3,✅ iOS 18.1,✅ iOS 18.2,✅ iOS 18.0,✅ iOS 18.0.1,✅ iOS 18.0.1
+openai_whisper-large-v3_turbo,openai_whisper-large-v3_turbo,✅ macOS 15.0.1<p>✅ macOS 15.1.1</p>,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.0,Not Supported,Not Supported,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPad16%2C1_summary_2024-10-29T234233.json>iPadOS 18.0.1</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPad16%2C3_summary_2024-10-30T073931.json>iPadOS 18.1</a>,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported
+openai_whisper-large-v3_turbo_954MB,openai_whisper-large-v3_turbo_954MB,✅ macOS 15.0.1<p>✅ macOS 15.1.1</p>,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.0,Not Supported,Not Supported,✅ iPadOS 17.6.1,✅ iPadOS 18.0.1,✅ iPadOS 18.1,Not Supported,Not Supported,Not Supported,Not Supported,✅ iOS 18.1,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPhone14%2C5_summary_2024-10-30T073931.json>iOS 17.3</a>,✅ iOS 17.3,✅ iOS 18.1,✅ iOS 18.2,✅ iOS 18.0,✅ iOS 18.0.1,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPhone17%2C4_summary_2024-10-30T132649.json>iOS 18.0.1</a>
+openai_whisper-small,openai_whisper-small,✅ macOS 15.0.1<p>✅ macOS 15.1.1</p>,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.0,✅ macOS 15.1,✅ iPadOS 18.1,✅ iPadOS 17.6.1,✅ iPadOS 18.0.1,✅ iPadOS 18.1,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPhone13%2C1_summary_2024-10-30T022739.json>iOS 17.7</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPhone13%2C2_summary_2024-10-30T152729.json>iOS 17.2.1</a>,✅ iOS 18.1,✅ iOS 18.1,✅ iOS 17.3,✅ iOS 17.3,✅ iOS 18.1,✅ iOS 18.2,✅ iOS 18.0,✅ iOS 18.0.1,✅ iOS 18.0.1
+openai_whisper-small.en,openai_whisper-small.en,✅ macOS 15.0.1<p>✅ macOS 15.1.1</p>,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.0,✅ macOS 15.1,✅ iPadOS 18.1,✅ iPadOS 17.6.1,✅ iPadOS 18.0.1,✅ iPadOS 18.1,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPhone13%2C1_summary_2024-10-30T022739.json>iOS 17.7</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPhone13%2C2_summary_2024-10-30T152729.json>iOS 17.2.1</a>,✅ iOS 18.1,✅ iOS 18.1,✅ iOS 17.3,✅ iOS 17.3,✅ iOS 18.1,✅ iOS 18.2,✅ iOS 18.0,✅ iOS 18.0.1,✅ iOS 18.0.1
+openai_whisper-tiny,openai_whisper-tiny,✅ macOS 15.0.1<p>✅ macOS 15.1.1</p>,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.0,✅ macOS 15.1,✅ iPadOS 18.1,✅ iPadOS 17.6.1,✅ iPadOS 18.0.1,✅ iPadOS 18.1,Not Supported,✅ iOS 17.7,✅ iOS 17.2.1,✅ iOS 18.1,✅ iOS 18.1,✅ iOS 17.3,✅ iOS 17.3,✅ iOS 18.1,✅ iOS 18.2,✅ iOS 18.0,✅ iOS 18.0.1,✅ iOS 18.0.1
+openai_whisper-tiny.en,openai_whisper-tiny.en,✅ macOS 15.0.1<p>✅ macOS 15.1.1</p>,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.0,✅ macOS 15.1,✅ iPadOS 18.1,✅ iPadOS 17.6.1,✅ iPadOS 18.0.1,✅ iPadOS 18.1,✅ iOS 17.6.1,✅ iOS 17.7,✅ iOS 17.2.1,✅ iOS 18.1,✅ iOS 18.1,✅ iOS 17.3,✅ iOS 17.3,✅ iOS 18.1,✅ iOS 18.2,✅ iOS 18.0,✅ iOS 18.0.1,✅ iOS 18.0.1

dashboard_data/support_data_112a023.csv

@@ -0,0 +1,23 @@
+,Model,"Apple M2 Pro (Mac14,12)","Apple M2 Ultra (Mac14,14)","Apple M3 Max (Mac15,10)","iPad Air 11-inch (M2) (iPad14,8)","iPad Pro 11-inch (M4) (iPad16,3)","iPhone 11 (iPhone12,1)","iPhone 14 (iPhone14,7)"
+distil-whisper_distil-large-v3,distil-whisper_distil-large-v3,✅ macOS 15.1.1,✅ macOS 15.0.1,✅ macOS 15.2,✅ iPadOS 17.6.1,✅ iPadOS 18.1,Not Supported,Not Supported
+distil-whisper_distil-large-v3_594MB,distil-whisper_distil-large-v3_594MB,✅ macOS 15.1.1,✅ macOS 15.0.1,✅ macOS 15.2,✅ iPadOS 17.6.1,✅ iPadOS 18.1,Not Supported,✅ iOS 17.7.1
+distil-whisper_distil-large-v3_turbo,distil-whisper_distil-large-v3_turbo,✅ macOS 15.1.1,✅ macOS 15.0.1,✅ macOS 15.2,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-12-21T053728_112a023/iPad14%2C8_summary_2024-12-21T223438.json>iPadOS 17.6.1</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-12-21T053728_112a023/iPad16%2C3_summary_2024-12-21T223438.json>iPadOS 18.1</a>,Not Supported,Not Supported
+distil-whisper_distil-large-v3_turbo_600MB,distil-whisper_distil-large-v3_turbo_600MB,✅ macOS 15.1.1,✅ macOS 15.0.1,✅ macOS 15.2,✅ iPadOS 17.6.1,✅ iPadOS 18.1,Not Supported,✅ iOS 17.7.1
+openai_whisper-base,openai_whisper-base,✅ macOS 15.1.1,✅ macOS 15.0.1,✅ macOS 15.2,✅ iPadOS 17.6.1,✅ iPadOS 18.1,✅ iOS 17.6.1,✅ iOS 17.7.1
+openai_whisper-base.en,openai_whisper-base.en,✅ macOS 15.1.1,✅ macOS 15.0.1,✅ macOS 15.2,✅ iPadOS 17.6.1,✅ iPadOS 18.1,✅ iOS 17.6.1,✅ iOS 17.7.1
+openai_whisper-large-v2,openai_whisper-large-v2,✅ macOS 15.1.1,✅ macOS 15.0.1,✅ macOS 15.2,✅ iPadOS 17.6.1,✅ iPadOS 18.1,Not Supported,Not Supported
+openai_whisper-large-v2_949MB,openai_whisper-large-v2_949MB,✅ macOS 15.1.1,✅ macOS 15.0.1,✅ macOS 15.2,✅ iPadOS 17.6.1,✅ iPadOS 18.1,Not Supported,✅ iOS 17.7.1
+openai_whisper-large-v2_turbo,openai_whisper-large-v2_turbo,✅ macOS 15.1.1,✅ macOS 15.0.1,✅ macOS 15.2,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-12-21T053728_112a023/iPad14%2C8_summary_2024-12-21T223438.json>iPadOS 17.6.1</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-12-21T053728_112a023/iPad16%2C3_summary_2024-12-21T223438.json>iPadOS 18.1</a>,Not Supported,Not Supported
+openai_whisper-large-v2_turbo_955MB,openai_whisper-large-v2_turbo_955MB,✅ macOS 15.1.1,✅ macOS 15.0.1,✅ macOS 15.2,✅ iPadOS 17.6.1,✅ iPadOS 18.1,Not Supported,✅ iOS 17.7.1
+openai_whisper-large-v3,openai_whisper-large-v3,✅ macOS 15.1.1,✅ macOS 15.0.1,✅ macOS 15.2,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-12-21T053728_112a023/iPad14%2C8_summary_2024-12-21T223438.json>iPadOS 17.6.1</a>,✅ iPadOS 18.1,Not Supported,Not Supported
+openai_whisper-large-v3-v20240930,openai_whisper-large-v3-v20240930,✅ macOS 15.1.1,✅ macOS 15.0.1,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-12-21T053728_112a023/Mac15%2C10_summary_2024-12-24T151240.json>macOS 15.2</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-12-21T053728_112a023/iPad14%2C8_summary_2024-12-21T223438.json>iPadOS 17.6.1</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-12-21T053728_112a023/iPad16%2C3_summary_2024-12-21T223438.json>iPadOS 18.1</a>,Not Supported,Not Supported
+openai_whisper-large-v3-v20240930_626MB,openai_whisper-large-v3-v20240930_626MB,✅ macOS 15.1.1,✅ macOS 15.0.1,✅ macOS 15.2,✅ iPadOS 17.6.1,✅ iPadOS 18.1,Not Supported,✅ iOS 17.7.1
+openai_whisper-large-v3-v20240930_turbo,openai_whisper-large-v3-v20240930_turbo,✅ macOS 15.1.1,✅ macOS 15.0.1,✅ macOS 15.2,✅ iPadOS 17.6.1,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-12-21T053728_112a023/iPad16%2C3_summary_2024-12-21T223438.json>iPadOS 18.1</a>,Not Supported,Not Supported
+openai_whisper-large-v3-v20240930_turbo_632MB,openai_whisper-large-v3-v20240930_turbo_632MB,✅ macOS 15.1.1,✅ macOS 15.0.1,✅ macOS 15.2,✅ iPadOS 17.6.1,✅ iPadOS 18.1,Not Supported,✅ iOS 17.7.1
+openai_whisper-large-v3_947MB,openai_whisper-large-v3_947MB,✅ macOS 15.1.1,✅ macOS 15.0.1,✅ macOS 15.2,✅ iPadOS 17.6.1,✅ iPadOS 18.1,Not Supported,✅ iOS 17.7.1
+openai_whisper-large-v3_turbo,openai_whisper-large-v3_turbo,✅ macOS 15.1.1,✅ macOS 15.0.1,✅ macOS 15.2,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-12-21T053728_112a023/iPad14%2C8_summary_2024-12-21T223438.json>iPadOS 17.6.1</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-12-21T053728_112a023/iPad16%2C3_summary_2024-12-21T223438.json>iPadOS 18.1</a>,Not Supported,Not Supported
+openai_whisper-large-v3_turbo_954MB,openai_whisper-large-v3_turbo_954MB,✅ macOS 15.1.1,✅ macOS 15.0.1,✅ macOS 15.2,✅ iPadOS 17.6.1,✅ iPadOS 18.1,Not Supported,✅ iOS 17.7.1
+openai_whisper-small,openai_whisper-small,✅ macOS 15.1.1,✅ macOS 15.0.1,✅ macOS 15.2,✅ iPadOS 17.6.1,✅ iPadOS 18.1,Not Supported,✅ iOS 17.7.1
+openai_whisper-small.en,openai_whisper-small.en,✅ macOS 15.1.1,✅ macOS 15.0.1,✅ macOS 15.2,✅ iPadOS 17.6.1,✅ iPadOS 18.1,Not Supported,✅ iOS 17.7.1
+openai_whisper-tiny,openai_whisper-tiny,✅ macOS 15.1.1,✅ macOS 15.0.1,✅ macOS 15.2,✅ iPadOS 17.6.1,✅ iPadOS 18.1,✅ iOS 17.6.1,✅ iOS 17.7.1
+openai_whisper-tiny.en,openai_whisper-tiny.en,✅ macOS 15.1.1,✅ macOS 15.0.1,✅ macOS 15.2,✅ iPadOS 17.6.1,✅ iPadOS 18.1,✅ iOS 17.6.1,✅ iOS 17.7.1

dashboard_data/support_data_a9b92c4.csv

@@ -0,0 +1,23 @@
+,Model,"Apple M2 Pro (Mac14,12)","Apple M2 Ultra (Mac14,14)","Apple M2 (Mac14,2)","Apple M3 Max (Mac15,10)","Apple M3 Pro (Mac15,6)","Apple M3 Max (Mac15,9)","Apple M4 (Mac16,10)","Apple M4 (Mac16,3)","Apple M1 (MacBookAir10,1)","iPad Air (5th generation) (iPad13,16)","iPad Air 11-inch (M2) (iPad14,8)","iPad mini (A17 Pro) (iPad16,1)","iPad Pro 11-inch (M4) (iPad16,3)","iPhone 11 (iPhone12,1)","iPhone 12 mini (iPhone13,1)","iPhone 12 (iPhone13,2)","iPhone 12 Pro Max (iPhone13,4)","iPhone 13 Pro (iPhone14,2)","iPhone 13 (iPhone14,5)","iPhone 14 (iPhone14,7)","iPhone 15 Pro (iPhone16,1)","iPhone 15 Pro Max (iPhone16,2)","iPhone 16 Pro (iPhone17,1)","iPhone 16 (iPhone17,3)","iPhone 16 Plus (iPhone17,4)"
+distil-whisper_distil-large-v3,distil-whisper_distil-large-v3,✅ macOS 15.0.1,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.0,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/MacBookAir10%2C1_summary_2024-10-30T012535.json>macOS 15.1</a>,Not Supported,✅ iPadOS 17.6.1,✅ iPadOS 18.0.1,✅ iPadOS 18.1,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported
+distil-whisper_distil-large-v3_594MB,distil-whisper_distil-large-v3_594MB,✅ macOS 15.0.1,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.0,✅ macOS 15.1,Not Supported,✅ iPadOS 17.6.1,✅ iPadOS 18.0.1,✅ iPadOS 18.1,Not Supported,Not Supported,Not Supported,Not Supported,✅ iOS 18.1,✅ iOS 17.3,✅ iOS 17.3,✅ iOS 18.1,✅ iOS 18.2,✅ iOS 18.0,✅ iOS 18.0.1,✅ iOS 18.0.1
+distil-whisper_distil-large-v3_turbo,distil-whisper_distil-large-v3_turbo,✅ macOS 15.0.1,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.0,Not Supported,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPad14%2C8_summary_2024-10-30T073931.json>iPadOS 17.6.1</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPad16%2C1_summary_2024-10-29T234233.json>iPadOS 18.0.1</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPad16%2C3_summary_2024-10-30T073931.json>iPadOS 18.1</a>,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported
+distil-whisper_distil-large-v3_turbo_600MB,distil-whisper_distil-large-v3_turbo_600MB,✅ macOS 15.0.1,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.0,Not Supported,Not Supported,✅ iPadOS 17.6.1,✅ iPadOS 18.0.1,✅ iPadOS 18.1,Not Supported,Not Supported,Not Supported,Not Supported,✅ iOS 18.1,✅ iOS 17.3,✅ iOS 17.3,✅ iOS 18.1,✅ iOS 18.2,✅ iOS 18.0,✅ iOS 18.0.1,✅ iOS 18.0.1
+openai_whisper-base,openai_whisper-base,✅ macOS 15.0.1,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.0,✅ macOS 15.1,✅ iPadOS 18.1,✅ iPadOS 17.6.1,✅ iPadOS 18.0.1,✅ iPadOS 18.1,✅ iOS 17.6.1,✅ iOS 17.7,✅ iOS 17.2.1,✅ iOS 18.1,✅ iOS 18.1,✅ iOS 17.3,✅ iOS 17.3,✅ iOS 18.1,✅ iOS 18.2,✅ iOS 18.0,✅ iOS 18.0.1,✅ iOS 18.0.1
+openai_whisper-base.en,openai_whisper-base.en,✅ macOS 15.0.1,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.0,✅ macOS 15.1,✅ iPadOS 18.1,✅ iPadOS 17.6.1,✅ iPadOS 18.0.1,✅ iPadOS 18.1,✅ iOS 17.6.1,✅ iOS 17.7,✅ iOS 17.2.1,✅ iOS 18.1,✅ iOS 18.1,✅ iOS 17.3,✅ iOS 17.3,✅ iOS 18.1,✅ iOS 18.2,✅ iOS 18.0,✅ iOS 18.0.1,✅ iOS 18.0.1
+openai_whisper-large-v2,openai_whisper-large-v2,✅ macOS 15.0.1,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.0,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/MacBookAir10%2C1_summary_2024-10-30T012535.json>macOS 15.1</a>,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPad14%2C8_summary_2024-10-30T073931.json>iPadOS 17.6.1</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPad16%2C1_summary_2024-10-29T234233.json>iPadOS 18.0.1</a>,✅ iPadOS 18.1,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported
+openai_whisper-large-v2_949MB,openai_whisper-large-v2_949MB,✅ macOS 15.0.1,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.0,✅ macOS 15.1,Not Supported,✅ iPadOS 17.6.1,✅ iPadOS 18.0.1,✅ iPadOS 18.1,Not Supported,Not Supported,Not Supported,Not Supported,✅ iOS 18.1,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPhone14%2C5_summary_2024-10-30T073931.json>iOS 17.3</a>,✅ iOS 17.3,✅ iOS 18.1,✅ iOS 18.2,✅ iOS 18.0,✅ iOS 18.0.1,✅ iOS 18.0.1
+openai_whisper-large-v2_turbo,openai_whisper-large-v2_turbo,✅ macOS 15.0.1,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.0,Not Supported,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPad14%2C8_summary_2024-10-30T073931.json>iPadOS 17.6.1</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPad16%2C1_summary_2024-10-29T234233.json>iPadOS 18.0.1</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPad16%2C3_summary_2024-10-30T073931.json>iPadOS 18.1</a>,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported
+openai_whisper-large-v2_turbo_955MB,openai_whisper-large-v2_turbo_955MB,✅ macOS 15.0.1,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.0,Not Supported,Not Supported,✅ iPadOS 17.6.1,✅ iPadOS 18.0.1,✅ iPadOS 18.1,Not Supported,Not Supported,Not Supported,Not Supported,✅ iOS 18.1,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPhone14%2C5_summary_2024-10-30T073931.json>iOS 17.3</a>,✅ iOS 17.3,✅ iOS 18.1,✅ iOS 18.2,✅ iOS 18.0,✅ iOS 18.0.1,✅ iOS 18.0.1
+openai_whisper-large-v3,openai_whisper-large-v3,✅ macOS 15.0.1,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.0,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/MacBookAir10%2C1_summary_2024-10-30T012535.json>macOS 15.1</a>,Not Supported,✅ iPadOS 17.6.1,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPad16%2C1_summary_2024-10-29T234233.json>iPadOS 18.0.1</a>,✅ iPadOS 18.1,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported
+openai_whisper-large-v3-v20240930,openai_whisper-large-v3-v20240930,✅ macOS 15.0.1,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.0,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/MacBookAir10%2C1_summary_2024-10-30T012535.json>macOS 15.1</a>,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPad14%2C8_summary_2024-10-30T073931.json>iPadOS 17.6.1</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPad16%2C1_summary_2024-10-29T234233.json>iPadOS 18.0.1</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPad16%2C3_summary_2024-10-30T073931.json>iPadOS 18.1</a>,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported
+openai_whisper-large-v3-v20240930_626MB,openai_whisper-large-v3-v20240930_626MB,✅ macOS 15.0.1,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.0,✅ macOS 15.1,Not Supported,✅ iPadOS 17.6.1,✅ iPadOS 18.0.1,✅ iPadOS 18.1,Not Supported,Not Supported,Not Supported,Not Supported,✅ iOS 18.1,✅ iOS 17.3,✅ iOS 17.3,✅ iOS 18.1,✅ iOS 18.2,✅ iOS 18.0,✅ iOS 18.0.1,✅ iOS 18.0.1
+openai_whisper-large-v3-v20240930_turbo,openai_whisper-large-v3-v20240930_turbo,✅ macOS 15.0.1,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.0,Not Supported,Not Supported,✅ iPadOS 17.6.1,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPad16%2C1_summary_2024-10-29T234233.json>iPadOS 18.0.1</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPad16%2C3_summary_2024-10-30T073931.json>iPadOS 18.1</a>,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported
+openai_whisper-large-v3-v20240930_turbo_632MB,openai_whisper-large-v3-v20240930_turbo_632MB,✅ macOS 15.0.1,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.0,Not Supported,Not Supported,✅ iPadOS 17.6.1,✅ iPadOS 18.0.1,✅ iPadOS 18.1,Not Supported,Not Supported,Not Supported,Not Supported,✅ iOS 18.1,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPhone14%2C5_summary_2024-10-30T073931.json>iOS 17.3</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPhone14%2C7_summary_2024-10-30T073931.json>iOS 17.3</a>,✅ iOS 18.1,✅ iOS 18.2,✅ iOS 18.0,✅ iOS 18.0.1,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPhone17%2C4_summary_2024-10-30T132649.json>iOS 18.0.1</a>
+openai_whisper-large-v3_947MB,openai_whisper-large-v3_947MB,✅ macOS 15.0.1,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.0,✅ macOS 15.1,Not Supported,✅ iPadOS 17.6.1,✅ iPadOS 18.0.1,✅ iPadOS 18.1,Not Supported,Not Supported,Not Supported,Not Supported,✅ iOS 18.1,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPhone14%2C5_summary_2024-10-30T073931.json>iOS 17.3</a>,✅ iOS 17.3,✅ iOS 18.1,✅ iOS 18.2,✅ iOS 18.0,✅ iOS 18.0.1,✅ iOS 18.0.1
+openai_whisper-large-v3_turbo,openai_whisper-large-v3_turbo,✅ macOS 15.0.1,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.0,Not Supported,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPad14%2C8_summary_2024-10-30T073931.json>iPadOS 17.6.1</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPad16%2C1_summary_2024-10-29T234233.json>iPadOS 18.0.1</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPad16%2C3_summary_2024-10-30T073931.json>iPadOS 18.1</a>,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported
+openai_whisper-large-v3_turbo_954MB,openai_whisper-large-v3_turbo_954MB,✅ macOS 15.0.1,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.0,Not Supported,Not Supported,✅ iPadOS 17.6.1,✅ iPadOS 18.0.1,✅ iPadOS 18.1,Not Supported,Not Supported,Not Supported,Not Supported,✅ iOS 18.1,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPhone14%2C5_summary_2024-10-30T073931.json>iOS 17.3</a>,✅ iOS 17.3,✅ iOS 18.1,✅ iOS 18.2,✅ iOS 18.0,✅ iOS 18.0.1,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPhone17%2C4_summary_2024-10-30T132649.json>iOS 18.0.1</a>
+openai_whisper-small,openai_whisper-small,✅ macOS 15.0.1,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.0,✅ macOS 15.1,✅ iPadOS 18.1,✅ iPadOS 17.6.1,✅ iPadOS 18.0.1,✅ iPadOS 18.1,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPhone13%2C1_summary_2024-10-30T022739.json>iOS 17.7</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPhone13%2C2_summary_2024-10-30T152729.json>iOS 17.2.1</a>,✅ iOS 18.1,✅ iOS 18.1,✅ iOS 17.3,✅ iOS 17.3,✅ iOS 18.1,✅ iOS 18.2,✅ iOS 18.0,✅ iOS 18.0.1,✅ iOS 18.0.1
+openai_whisper-small.en,openai_whisper-small.en,✅ macOS 15.0.1,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.0,✅ macOS 15.1,✅ iPadOS 18.1,✅ iPadOS 17.6.1,✅ iPadOS 18.0.1,✅ iPadOS 18.1,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPhone13%2C1_summary_2024-10-30T022739.json>iOS 17.7</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPhone13%2C2_summary_2024-10-30T152729.json>iOS 17.2.1</a>,✅ iOS 18.1,✅ iOS 18.1,✅ iOS 17.3,✅ iOS 17.3,✅ iOS 18.1,✅ iOS 18.2,✅ iOS 18.0,✅ iOS 18.0.1,✅ iOS 18.0.1
+openai_whisper-tiny,openai_whisper-tiny,✅ macOS 15.0.1,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.0,✅ macOS 15.1,✅ iPadOS 18.1,✅ iPadOS 17.6.1,✅ iPadOS 18.0.1,✅ iPadOS 18.1,✅ iOS 17.6.1,✅ iOS 17.7,✅ iOS 17.2.1,✅ iOS 18.1,✅ iOS 18.1,✅ iOS 17.3,✅ iOS 17.3,✅ iOS 18.1,✅ iOS 18.2,✅ iOS 18.0,✅ iOS 18.0.1,✅ iOS 18.0.1
+openai_whisper-tiny.en,openai_whisper-tiny.en,✅ macOS 15.0.1,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.0,✅ macOS 15.1,✅ iPadOS 18.1,✅ iPadOS 17.6.1,✅ iPadOS 18.0.1,✅ iPadOS 18.1,✅ iOS 17.6.1,✅ iOS 17.7,✅ iOS 17.2.1,✅ iOS 18.1,✅ iOS 18.1,✅ iOS 17.3,✅ iOS 17.3,✅ iOS 18.1,✅ iOS 18.2,✅ iOS 18.0,✅ iOS 18.0.1,✅ iOS 18.0.1

dashboard_data/version.json

@@ -1 +1 @@
-{"last_modified": "2024-11-03T19:01:20+00:00", "sha": "025ca7f259a8f554806d26f4d08e6f75b92941d1", "releases": ["a9b92c4"], "whisperkit_version": "0.9.1"}
+{"last_modified": "2024-11-03T19:01:20+00:00", "sha": "025ca7f259a8f554806d26f4d08e6f75b92941d1", "releases": ["a9b92c4", "112a023"], "whisperkit_version": "0.10.1"}

main.py

@@ -27,6 +27,7 @@ from constants import (
     METHODOLOGY_TEXT,
     PERFORMANCE_TEXT,
     QUALITY_TEXT,
+    SHA_TO_VERSION,
 )
 from utils import (
     add_datasets_to_performance_columns,
@@ -61,10 +62,13 @@ local_dir = ""
 # Load benchmark data from JSON files
 PERFORMANCE_DATA = read_json_line_by_line("dashboard_data/performance_data.json")
 QUALITY_DATA = read_json_line_by_line("dashboard_data/quality_data.json")
+with open("dashboard_data/version.json", "r") as file:
+    VERSION_DATA = json.load(file)
 
 # Convert JSON data to pandas DataFrames
 quality_df = pd.json_normalize(QUALITY_DATA)
 benchmark_df = pd.json_normalize(PERFORMANCE_DATA)
+releases = VERSION_DATA["releases"]
 
 # Process timestamp data
 benchmark_df["timestamp"] = pd.to_datetime(benchmark_df["timestamp"]).dt.tz_localize(
@@ -174,6 +178,7 @@ performance_df = sorted_performance_df[
         "speed",
         "tokens_per_second",
         "timestamp",
+        "commit_hash"
     ]
     + dataset_speed_columns
     + dataset_toks_columns
@@ -220,8 +225,9 @@ performance_df["Model"] = performance_df["Model"].apply(
 )
 
 # Extract unique devices and OS versions
-PERFORMANCE_DEVICES = performance_df["Device"].unique().tolist()
-PERFORMANCE_OS = performance_df["OS"].apply(get_os_name_and_version).unique().tolist()
+initial_release_df = benchmark_df[benchmark_df["commit_hash"] == releases[-1]]
+PERFORMANCE_DEVICES = initial_release_df["device"].unique().tolist()
+PERFORMANCE_OS = initial_release_df["os"].apply(get_os_name_and_version).unique().tolist()
 PERFORMANCE_OS.sort()
 
 # Create initial column dictionaries and update with dataset information
@@ -254,6 +260,15 @@ QUALITY_TOGGLE_COLS = quality_column_info["TOGGLE_COLS"]
 PERFORMANCE_SELECTED_COLS = performance_column_info["SELECTED_COLS"]
 QUALITY_SELECTED_COLS = quality_column_info["SELECTED_COLS"]
 
+def get_release_devices(release):
+    """
+    Get the list of devices for a specific release.
+    
+    :param release: Selected release hash
+    :return: List of devices available in the release
+    """
+    release_df = benchmark_df[benchmark_df["commit_hash"] == release]
+    return release_df["device"].unique().tolist()
 
 def performance_filter(
     df,
@@ -266,6 +281,7 @@ def performance_filter(
     long_speed_slider,
     short_toks_slider,
     long_toks_slider,
+    release,
 ):
     """
     Filters the performance DataFrame based on specified criteria.
@@ -281,8 +297,10 @@ def performance_filter(
     :param long_toks_slider: The range of values to filter the 'Long-Form Tok/s' column.
     :return: The filtered DataFrame.
     """
+    filtered_df = df[df["commit_hash"] == release]
+
     # Select columns based on input and always-present columns
-    filtered_df = df[
+    filtered_df = filtered_df[
         PERFORMANCE_ALWAYS_HERE_COLS
         + [c for c in PERFORMANCE_COLS if c in df.columns and c in columns]
     ]
@@ -405,6 +423,66 @@ def quality_filter(df, columns, model_query, wer_slider, qoi_slider, exclude_mod
     return filtered_df
 
 
+def update_performance_filters(release):
+    """
+    Updates the performance filters (devices and OS) based on the selected release.
+    
+    :param release: Selected release hash
+    :return: Tuple containing updated device and OS choices
+    """
+    # Filter benchmark data for the selected release
+    release_df = benchmark_df[benchmark_df["commit_hash"] == release]
+    
+    # Get unique devices and OS versions for this release
+    release_devices = release_df["device"].unique().tolist()
+    release_os = release_df["os"].apply(get_os_name_and_version).unique().tolist()
+    release_os.sort()
+    
+    return (
+        gr.update(choices=release_devices, value=release_devices),
+        gr.update(choices=release_os, value=release_os)
+    )
+
+
+def update_support_table(release):
+    """
+    Updates the support table and its column configuration for a given release.
+    
+    :param release: Selected release hash
+    :return: Tuple containing (updated DataFrame, updated column choices, updated column values)
+    """
+    # Load new support data
+    support_data = pd.read_csv(f"dashboard_data/support_data_{release[:7]}.csv")
+    support_data.set_index(support_data.columns[0], inplace=True)
+    
+    # Process model names
+    support_data["Model"] = support_data["Model"].apply(
+        lambda x: x.replace("_", "/")
+    )
+    support_data["Model"] = support_data["Model"].apply(
+        lambda x: make_model_name_clickable_link(x)
+    )
+    
+    # Sort by model name length
+    support_data = (
+        support_data.assign(model_len=support_data["Model"].str.len())
+        .sort_values(
+            by=["model_len"],
+            ascending=[True],
+        )
+        .drop(columns=["model_len"])
+    )
+    
+    # Get new columns (excluding 'Model')
+    new_columns = support_data.columns.tolist()[1:]
+    
+    return (
+        gr.update(value=support_data, datatype=["html" for _ in support_data.columns]),
+        gr.update(choices=new_columns, value=new_columns),
+        gr.update(value=support_data)
+    )
+
+
 diff_tab = gr.TabItem("Difference Checker", elem_id="diff_checker", id=2)
 text_diff_elems = []
 
@@ -513,6 +591,14 @@ with gr.Blocks(css=css, theme=gr.themes.Base(font=font)) as demo:
     # Add header and banner to the interface
     gr.HTML(HEADER)
     gr.HTML(BANNER_TEXT, elem_classes="markdown-text")
+    gr.Markdown("### Release")
+    release_dropdown = gr.Dropdown(
+        choices=[(f"{release} v{SHA_TO_VERSION[release]}", release) for release in releases],
+        label="Select Release",
+        value=releases[-1] if releases else None,
+        elem_id="release-dropdown",
+        container=False,
+    )
 
     # Create tabs for different sections of the dashboard
     with tabs.render():
@@ -583,8 +669,8 @@ with gr.Blocks(css=css, theme=gr.themes.Base(font=font)) as demo:
                                     scale=9, elem_id="filter_devices_column"
                                 ):
                                     performance_shown_devices = gr.CheckboxGroup(
-                                        choices=PERFORMANCE_DEVICES,
-                                        value=PERFORMANCE_DEVICES,
+                                        choices=get_release_devices(releases[-1]),
+                                        value=get_release_devices(releases[-1]),
                                         label="Filter Devices",
                                         interactive=True,
                                     )
@@ -605,15 +691,17 @@ with gr.Blocks(css=css, theme=gr.themes.Base(font=font)) as demo:
                                             interactive=True,
                                         )
 
-                            def select_all_devices():
-                                return PERFORMANCE_DEVICES
+                            def select_all_devices(release):
+                                """Returns all devices available in the current release"""
+                                return get_release_devices(release)
 
                             def deselect_all_devices():
+                                """Returns an empty list for deselecting all devices"""
                                 return []
 
                             select_all_devices_button.click(
                                 select_all_devices,
-                                inputs=[],
+                                inputs=[release_dropdown],
                                 outputs=performance_shown_devices,
                             )
                             deselect_all_devices_button.click(
@@ -679,8 +767,9 @@ with gr.Blocks(css=css, theme=gr.themes.Base(font=font)) as demo:
                     with gr.Row():
                         gr.Markdown(PERFORMANCE_TEXT, elem_classes="markdown-text")
             with gr.Row():
+                initial_df = performance_df[performance_df["commit_hash"] == releases[-1]]
                 leaderboard_df = gr.components.Dataframe(
-                    value=performance_df[
+                    value=initial_df[
                         PERFORMANCE_ALWAYS_HERE_COLS + performance_shown_columns.value
                     ],
                     headers=[
@@ -720,6 +809,7 @@ with gr.Blocks(css=css, theme=gr.themes.Base(font=font)) as demo:
                     long_speed_slider,
                     short_toks_slider,
                     long_toks_slider,
+                    release_dropdown,
                 ]
 
                 filter_output = leaderboard_df
@@ -750,6 +840,16 @@ with gr.Blocks(css=css, theme=gr.themes.Base(font=font)) as demo:
                 long_toks_slider.change(
                     performance_filter, performance_filter_inputs, filter_output
                 )
+                release_dropdown.change(
+                    fn=update_performance_filters,
+                    inputs=[release_dropdown],
+                    outputs=[performance_shown_devices, performance_shown_os],
+                    queue=False
+                ).then(
+                    fn=performance_filter,
+                    inputs=performance_filter_inputs,
+                    outputs=filter_output
+                )
 
         # English Quality Tab
         with gr.TabItem("English Quality", elem_id="timeline", id=1):
@@ -1163,7 +1263,7 @@ with gr.Blocks(css=css, theme=gr.themes.Base(font=font)) as demo:
         # Device Support Tab
         with gr.TabItem("Device Support", elem_id="device_support", id=6):
             # Load device support data from CSV
-            support_data = pd.read_csv("dashboard_data/support_data.csv")
+            support_data = pd.read_csv(f"dashboard_data/support_data_{releases[-1][:7]}.csv")
             support_data.set_index(support_data.columns[0], inplace=True)
             support_data["Model"] = support_data["Model"].apply(
                 lambda x: x.replace("_", "/")
@@ -1198,9 +1298,7 @@ with gr.Blocks(css=css, theme=gr.themes.Base(font=font)) as demo:
                             with gr.Row():
                                 with gr.Column(scale=9):
                                     support_shown_columns = gr.CheckboxGroup(
-                                        choices=support_data.columns.tolist()[
-                                            1:
-                                        ],  # Exclude 'Model' column
+                                        choices=support_data.columns.tolist()[1:],  # Exclude 'Model' column
                                         value=support_data.columns.tolist()[1:],
                                         label="Toggle Columns",
                                         elem_id="support-column-select",
@@ -1218,6 +1316,7 @@ with gr.Blocks(css=css, theme=gr.themes.Base(font=font)) as demo:
                                             elem_id="deselect-all-support-button",
                                             interactive=True,
                                         )
+
             with gr.Column():
                 gr.Markdown(
                     """
@@ -1243,6 +1342,15 @@ with gr.Blocks(css=css, theme=gr.themes.Base(font=font)) as demo:
             hidden_support_df = gr.Dataframe(value=support_data, visible=False)
 
             def filter_support_data(df, columns, model_query, exclude_models):
+                """
+                Filters the device support data based on specified criteria.
+                
+                :param df: The DataFrame to be filtered
+                :param columns: Columns to include in the output
+                :param model_query: Query string to filter models
+                :param exclude_models: Models to exclude
+                :return: Filtered DataFrame
+                """
                 filtered_df = df.copy()
 
                 # Filter models based on query
@@ -1257,9 +1365,7 @@ with gr.Blocks(css=css, theme=gr.themes.Base(font=font)) as demo:
 
                 # Exclude specified models
                 if exclude_models:
-                    exclude_list = [
-                        re.escape(m.strip()) for m in exclude_models.split(";")
-                    ]
+                    exclude_list = [re.escape(m.strip()) for m in exclude_models.split(";")]
                     filtered_df = filtered_df[
                         ~filtered_df["Model"].str.contains(
                             "|".join(exclude_list), case=False, regex=True
@@ -1267,40 +1373,31 @@ with gr.Blocks(css=css, theme=gr.themes.Base(font=font)) as demo:
                     ]
 
                 # Select columns
-                selected_columns = ["Model"] + [
-                    col for col in columns if col in df.columns
-                ]
+                selected_columns = ["Model"] + [col for col in columns if col in df.columns]
                 filtered_df = filtered_df[selected_columns]
 
                 return filtered_df
 
-            def select_all_support_columns():
-                return support_data.columns.tolist()[1:]  # Exclude 'Model' column
+            def select_all_support_columns(release):
+                """
+                Returns all current columns from the support shown columns.
+                
+                :param release: Selected release hash
+                :return: List of all available choices
+                """
+                # Load new support data for the current release
+                support_data = pd.read_csv(f"dashboard_data/support_data_{release[:7]}.csv")
+                support_data.set_index(support_data.columns[0], inplace=True)
+                # Return all columns except 'Model'
+                return [col for col in support_data.columns if col != "Model"]
 
             def deselect_all_support_columns():
                 return []
 
-            # Connect the filter function to the input components
-            filter_inputs = [
-                hidden_support_df,
-                support_shown_columns,
-                filter_support_models,
-                exclude_support_models,
-            ]
-            filter_support_models.change(
-                filter_support_data, filter_inputs, device_support_table
-            )
-            exclude_support_models.change(
-                filter_support_data, filter_inputs, device_support_table
-            )
-            support_shown_columns.change(
-                filter_support_data, filter_inputs, device_support_table
-            )
-
             # Connect select all and deselect all buttons
             select_all_support_button.click(
                 select_all_support_columns,
-                inputs=[],
+                inputs=[release_dropdown],
                 outputs=support_shown_columns,
             )
             deselect_all_support_button.click(
@@ -1309,6 +1406,25 @@ with gr.Blocks(css=css, theme=gr.themes.Base(font=font)) as demo:
                 outputs=support_shown_columns,
             )
 
+            # Connect release dropdown to support data update
+            release_dropdown.change(
+                update_support_table,
+                inputs=[release_dropdown],
+                outputs=[device_support_table, support_shown_columns, hidden_support_df]
+            ).then(
+                filter_support_data,
+                inputs=[hidden_support_df, support_shown_columns, filter_support_models, exclude_support_models],
+                outputs=device_support_table
+            )
+
+            # Also connect the filter inputs to update the table
+            for input_elem in [filter_support_models, exclude_support_models, support_shown_columns]:
+                input_elem.change(
+                    filter_support_data,
+                    inputs=[hidden_support_df, support_shown_columns, filter_support_models, exclude_support_models],
+                    outputs=device_support_table
+                )
+
         # Methodology Tab
         with gr.TabItem("Methodology", elem_id="methodology", id=7):
             gr.Markdown(METHODOLOGY_TEXT, elem_id="methodology-text")

performance_generate.py

@@ -173,10 +173,11 @@ def process_summary_file(file_path, results, releases):
 
     :param file_path: Path to the summary JSON file.
     :param results: Dictionary to store the processed results.
+    :param releases: Set of release commit hashes to process.
 
     This function reads a summary JSON file, extracts information about supported
     and failed models for a specific device and OS combination, and updates the
-    results dictionary accordingly.
+    results dictionary accordingly. It creates separate entries for each release.
     """
     with open(file_path, "r") as file:
         summary_data = json.load(file)
@@ -186,11 +187,12 @@ def process_summary_file(file_path, results, releases):
 
     device = summary_data["deviceIdentifier"]
     os = f"{'iPadOS' if 'iPad' in device else summary_data['osType']} {summary_data['osVersion']}"
+    commit_hash = summary_data["commitHash"]
     commit_timestamp = summary_data["commitTimestamp"]
     test_file_name = file_path.split("/")[-1]
     test_timestamp = test_file_name.split("_")[-1].replace(".json", "")
     
-    key = (device, os) 
+    key = (device, os, commit_hash)
     if key in results:
         existing_commit_timestamp = results[key]["commitTimestamp"]
         existing_test_timestamp = results[key]["testTimestamp"]
@@ -224,6 +226,7 @@ def process_summary_file(file_path, results, releases):
         failed_models.add(failed_model)
 
     results[key]["supportedModels"] = supported_models
+    results[key]["commitHash"] = commit_hash
     results[key]["commitTimestamp"] = commit_timestamp
     results[key]["testTimestamp"] = test_timestamp
     results[key]["failedModels"] = (failed_models, file_path)
@@ -294,70 +297,98 @@ def calculate_and_save_support_results(
     support_results, not_supported, support_output_path
 ):
     """
-    Calculates device support results and saves them to a CSV file.
+    Calculates device support results and saves them to separate CSV files for each release.
 
     :param support_results: Dictionary containing device support information.
-    :param support_output_path: Path to save the processed support results.
+    :param support_output_path: Base path to save the processed support results.
+    :param not_supported: List of (model, device, os) tuples that are not supported.
 
-    This function processes the device support data and creates a CSV file
+    This function processes the device support data and creates separate CSV files
     showing which models are supported on different devices and OS versions,
-    using checkmarks, warning signs, quesiton marks or Not supported to
+    using checkmarks, warning signs, question marks or Not supported to
     indicate support status.
     """
     all_models = sorted(support_results["modelsTested"])
-    all_devices = sorted(support_results["devices"])
 
-    df = pd.DataFrame(index=all_models, columns=["Model"] + all_devices)
-
-    for model in all_models:
-        row = {"Model": model}
-        for device in all_devices:
-            row[device] = ""
-
-        for key, data in support_results.items():
-            if key in ["modelsTested", "devices"]:
-                continue
-            (device, os) = key
-            supported_models = data["supportedModels"]
-            failed_models, file_path = data["failedModels"]
-            directories = file_path.split("/")
-            commit_file, summary_file = directories[-2], directories[-1]
-            url = f"{BASE_WHISPERKIT_BENCHMARK_URL}/{commit_file}/{urllib.parse.quote(summary_file)}"
-
-            if model in supported_models:
-                current_value = row[device]
-                new_value = (
-                    f"✅ {os}"
-                    if current_value == ""
-                    else f"{current_value}<p>✅ {os}</p>"
-                )
-            elif model in failed_models:
-                current_value = row[device]
-                new_value = (
-                    f"""⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href={url}>{os}</a>"""
-                    if current_value == ""
-                    else f"""{current_value}<p>⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href={url}>{os}</a></p>"""
-                )
-            else:
-                current_value = row[device]
-                new_value = (
-                    f"? {os}"
-                    if current_value == ""
-                    else f"{current_value}<p>? {os}</p>"
-                )
-            row[device] = new_value
-
-        df.loc[model] = row
-
-    remove_unsupported_cells(df, not_supported)
-
-    cols = df.columns.tolist()
-    cols = ["Model"] + [
-        f"""{get_device_name(col).replace("_", " ")} ({col})""" for col in cols if col != "Model"
-    ]
-    df.columns = cols
-
-    df.to_csv(support_output_path, index=True)
+    # Group results by commit hash
+    results_by_commit = {}
+    for key, data in support_results.items():
+        if key in ["modelsTested", "devices"]:
+            continue
+        device, os, commit_hash = key
+        if commit_hash not in results_by_commit:
+            results_by_commit[commit_hash] = {
+                "data": {},
+                "devices": set(),
+                "timestamp": data["commitTimestamp"]
+            }
+        results_by_commit[commit_hash]["data"][key] = data
+        results_by_commit[commit_hash]["devices"].add(device)
+
+    # Generate separate CSV for each commit
+    for commit_hash, commit_data in results_by_commit.items():
+        commit_devices = sorted(commit_data["devices"])
+        df = pd.DataFrame(index=all_models, columns=["Model"] + commit_devices)
+        
+        for model in all_models:
+            row = {"Model": model}
+            for device in commit_devices:
+                row[device] = ""
+
+            for key, data in commit_data["data"].items():
+                device, os, _ = key
+                supported_models = data["supportedModels"]
+                failed_models, file_path = data["failedModels"]
+                directories = file_path.split("/")
+                commit_file, summary_file = directories[-2], directories[-1]
+                url = f"{BASE_WHISPERKIT_BENCHMARK_URL}/{commit_file}/{urllib.parse.quote(summary_file)}"
+
+                if model in supported_models:
+                    current_value = row[device]
+                    new_value = (
+                        f"✅ {os}"
+                        if current_value == ""
+                        else f"{current_value}<p>✅ {os}</p>"
+                    )
+                elif model in failed_models:
+                    current_value = row[device]
+                    new_value = (
+                        f"""⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href={url}>{os}</a>"""
+                        if current_value == ""
+                        else f"""{current_value}<p>⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href={url}>{os}</a></p>"""
+                    )
+                else:
+                    current_value = row[device]
+                    new_value = (
+                        f"? {os}"
+                        if current_value == ""
+                        else f"{current_value}<p>? {os}</p>"
+                    )
+                row[device] = new_value
+
+            df.loc[model] = row
+
+        # Mark unsupported combinations for this commit
+        commit_not_supported = [
+            (model, device, os) 
+            for model, device, os in not_supported 
+            if any(key[2] == commit_hash for key in support_results if key not in ["modelsTested", "devices"] and model == key[0])
+        ]
+        remove_unsupported_cells(df, commit_not_supported)
+
+        # Format column headers
+        cols = df.columns.tolist()
+        cols = ["Model"] + [
+            f"""{get_device_name(col).replace("_", " ")} ({col})""" for col in cols if col != "Model"
+        ]
+        df.columns = cols
+
+        # Save to commit-specific file
+        output_path = support_output_path.replace(
+            ".csv", 
+            f"_{commit_hash[:7]}.csv"
+        )
+        df.to_csv(output_path, index=True)
 
 
 def remove_unsupported_cells(df, not_supported):

utils.py

@@ -908,6 +908,12 @@ strong, b {
     letter-spacing: -0.02em;
 }
 
+#release-dropdown {
+    width: 14%;
+    margin-left: 0px;
+    margin-right: auto;
+}
+
 /* Global Zwizz styles */
 :root {
     --zwizz-spacing: -0.02em;