Spaces:

illuin
/

grouse

Running

App Files Files Community

smuller commited on Sep 12, 2024

Commit

ded6e1f

verified ·

1 Parent(s): 70a59c3

Sync local Space with Hub

Browse files

This view is limited to 50 files because it contains too many changes. See raw diff

Files changed (50) hide show

.gitattributes +1 -0
index.html +336 -369
static/images/confusion_matrixes.png +0 -0
static/images/correlation_table.png +0 -0
static/images/favicon.svg +0 -37
static/images/grounded_qa_cases.png +0 -0
static/images/interpolate_end.jpg +0 -0
static/images/interpolate_start.jpg +0 -0
static/images/matrices.png +0 -0
static/images/meme.jpeg +0 -0
static/images/{steve.webm → one_sample.png} +2 -2
static/images/result_table.png +0 -0
static/images/schema_unit_test.png +0 -0
static/images/schema_unit_test_blog.png +0 -0
static/interpolation/stacked/000000.jpg +0 -0
static/interpolation/stacked/000001.jpg +0 -0
static/interpolation/stacked/000002.jpg +0 -0
static/interpolation/stacked/000003.jpg +0 -0
static/interpolation/stacked/000004.jpg +0 -0
static/interpolation/stacked/000005.jpg +0 -0
static/interpolation/stacked/000006.jpg +0 -0
static/interpolation/stacked/000007.jpg +0 -0
static/interpolation/stacked/000008.jpg +0 -0
static/interpolation/stacked/000009.jpg +0 -0
static/interpolation/stacked/000010.jpg +0 -0
static/interpolation/stacked/000011.jpg +0 -0
static/interpolation/stacked/000012.jpg +0 -0
static/interpolation/stacked/000013.jpg +0 -0
static/interpolation/stacked/000014.jpg +0 -0
static/interpolation/stacked/000015.jpg +0 -0
static/interpolation/stacked/000016.jpg +0 -0
static/interpolation/stacked/000017.jpg +0 -0
static/interpolation/stacked/000018.jpg +0 -0
static/interpolation/stacked/000019.jpg +0 -0
static/interpolation/stacked/000020.jpg +0 -0
static/interpolation/stacked/000021.jpg +0 -0
static/interpolation/stacked/000022.jpg +0 -0
static/interpolation/stacked/000023.jpg +0 -0
static/interpolation/stacked/000024.jpg +0 -0
static/interpolation/stacked/000025.jpg +0 -0
static/interpolation/stacked/000026.jpg +0 -0
static/interpolation/stacked/000027.jpg +0 -0
static/interpolation/stacked/000028.jpg +0 -0
static/interpolation/stacked/000029.jpg +0 -0
static/interpolation/stacked/000030.jpg +0 -0
static/interpolation/stacked/000031.jpg +0 -0
static/interpolation/stacked/000032.jpg +0 -0
static/interpolation/stacked/000033.jpg +0 -0
static/interpolation/stacked/000034.jpg +0 -0
static/interpolation/stacked/000035.jpg +0 -0

.gitattributes CHANGED Viewed

@@ -46,3 +46,4 @@ static/videos/shiba.mp4 filter=lfs diff=lfs merge=lfs -text
 static/videos/steve.mp4 filter=lfs diff=lfs merge=lfs -text
 static/videos/teaser.mp4 filter=lfs diff=lfs merge=lfs -text
 static/videos/toby.mp4 filter=lfs diff=lfs merge=lfs -text

 static/videos/steve.mp4 filter=lfs diff=lfs merge=lfs -text
 static/videos/teaser.mp4 filter=lfs diff=lfs merge=lfs -text
 static/videos/toby.mp4 filter=lfs diff=lfs merge=lfs -text
+static/images/one_sample.png filter=lfs diff=lfs merge=lfs -text

index.html CHANGED Viewed

@@ -1,435 +1,402 @@
 <!DOCTYPE html>
 <html>
 <head>
   <meta charset="utf-8">
   <meta name="description"
-        content="Deformable Neural Radiance Fields creates free-viewpoint portraits (nerfies) from casually captured videos.">
   <meta name="keywords" content="Nerfies, D-NeRF, NeRF">
   <meta name="viewport" content="width=device-width, initial-scale=1">
-  <title>Nerfies: Deformable Neural Radiance Fields</title>
-  <link href="https://fonts.googleapis.com/css?family=Google+Sans|Noto+Sans|Castoro"
-        rel="stylesheet">
   <link rel="stylesheet" href="./static/css/bulma.min.css">
-  <link rel="stylesheet" href="./static/css/bulma-carousel.min.css">
-  <link rel="stylesheet" href="./static/css/bulma-slider.min.css">
   <link rel="stylesheet" href="./static/css/fontawesome.all.min.css">
-  <link rel="stylesheet"
-        href="https://cdn.jsdelivr.net/gh/jpswalsh/academicons@1/css/academicons.min.css">
   <link rel="stylesheet" href="./static/css/index.css">
   <link rel="icon" href="./static/images/favicon.svg">
   <script src="https://ajax.googleapis.com/ajax/libs/jquery/3.5.1/jquery.min.js"></script>
   <script defer src="./static/js/fontawesome.all.min.js"></script>
   <script src="./static/js/bulma-carousel.min.js"></script>
   <script src="./static/js/bulma-slider.min.js"></script>
   <script src="./static/js/index.js"></script>
 </head>
 <body>
-<section class="hero">
-  <div class="hero-body">
-    <div class="container is-max-desktop">
-      <div class="columns is-centered">
-        <div class="column has-text-centered">
-          <h1 class="title is-1 publication-title">Nerfies: Deformable Neural Radiance Fields</h1>
-          <div class="is-size-5 publication-authors">
-            <span class="author-block">
-              <a href="https://keunhong.com" target="_blank">Keunhong Park</a><sup>1</sup>,</span>
-            <span class="author-block">
-              <a href="https://utkarshsinha.com" target="_blank">Utkarsh Sinha</a><sup>2</sup>,</span>
-            <span class="author-block">
-              <a href="https://jonbarron.info" target="_blank">Jonathan T. Barron</a><sup>2</sup>,
-            </span>
-            <span class="author-block">
-              <a href="http://sofienbouaziz.com" target="_blank">Sofien Bouaziz</a><sup>2</sup>,
-            </span>
-            <span class="author-block">
-              <a href="https://www.danbgoldman.com" target="_blank">Dan B Goldman</a><sup>2</sup>,
-            </span>
-            <span class="author-block">
-              <a href="https://homes.cs.washington.edu/~seitz/" target="_blank">Steven M. Seitz</a><sup>1,2</sup>,
-            </span>
-            <span class="author-block">
-              <a href="http://www.ricardomartinbrualla.com" target="_blank">Ricardo Martin-Brualla</a><sup>2</sup>
-            </span>
-          </div>
-          <div class="is-size-5 publication-authors">
-            <span class="author-block"><sup>1</sup>University of Washington,</span>
-            <span class="author-block"><sup>2</sup>Google Research</span>
-          </div>
-          <div class="column has-text-centered">
-            <div class="publication-links">
-              <!-- PDF Link. -->
-              <span class="link-block">
-                <a href="https://arxiv.org/pdf/2011.12948" target="_blank"
-                   class="external-link button is-normal is-rounded is-dark">
-                  <span class="icon">
                       <i class="fas fa-file-pdf"></i>
-                  </span>
-                  <span>Paper</span>
-                </a>
-              </span>
-              <span class="link-block">
-                <a href="https://arxiv.org/abs/2011.12948" target="_blank"
-                   class="external-link button is-normal is-rounded is-dark">
-                  <span class="icon">
                       <i class="ai ai-arxiv"></i>
-                  </span>
-                  <span>arXiv</span>
-                </a>
-              </span>
-              <!-- Video Link. -->
-              <span class="link-block">
-                <a href="https://www.youtube.com/watch?v=MrKrnHhk8IA" target="_blank"
-                   class="external-link button is-normal is-rounded is-dark">
-                  <span class="icon">
-                      <i class="fab fa-youtube"></i>
-                  </span>
-                  <span>Video</span>
-                </a>
-              </span>
-              <!-- Code Link. -->
-              <span class="link-block">
-                <a href="https://github.com/google/nerfies" target="_blank"
-                   class="external-link button is-normal is-rounded is-dark">
-                  <span class="icon">
                       <i class="fab fa-github"></i>
-                  </span>
-                  <span>Code</span>
                   </a>
-              </span>
-              <!-- Dataset Link. -->
-              <span class="link-block">
-                <a href="https://github.com/google/nerfies/releases/tag/0.1" target="_blank"
-                   class="external-link button is-normal is-rounded is-dark">
-                  <span class="icon">
-                      <i class="far fa-images"></i>
-                  </span>
-                  <span>Data</span>
                   </a>
-            </div>
           </div>
         </div>
       </div>
     </div>
-  </div>
-</section>
-<section class="hero teaser">
-  <div class="container is-max-desktop">
-    <div class="hero-body">
-      <video id="teaser" autoplay muted loop playsinline height="100%">
-        <source src="./static/videos/teaser.mp4"
-                type="video/mp4">
-      </video>
-      <h2 class="subtitle has-text-centered">
-        <span class="dnerf">Nerfies</span> turns selfie videos from your phone into
-        free-viewpoint
-        portraits.
-      </h2>
-    </div>
-  </div>
-</section>
-<section class="hero is-light is-small">
-  <div class="hero-body">
-    <div class="container">
-      <div id="results-carousel" class="carousel results-carousel">
-        <div class="item item-steve">
-          <video poster="" id="steve" autoplay controls muted loop playsinline height="100%">
-            <source src="./static/videos/steve.mp4"
-                    type="video/mp4">
-          </video>
-        </div>
-        <div class="item item-chair-tp">
-          <video poster="" id="chair-tp" autoplay controls muted loop playsinline height="100%">
-            <source src="./static/videos/chair-tp.mp4"
-                    type="video/mp4">
-          </video>
-        </div>
-        <div class="item item-shiba">
-          <video poster="" id="shiba" autoplay controls muted loop playsinline height="100%">
-            <source src="./static/videos/shiba.mp4"
-                    type="video/mp4">
-          </video>
-        </div>
-        <div class="item item-fullbody">
-          <video poster="" id="fullbody" autoplay controls muted loop playsinline height="100%">
-            <source src="./static/videos/fullbody.mp4"
-                    type="video/mp4">
-          </video>
-        </div>
-        <div class="item item-blueshirt">
-          <video poster="" id="blueshirt" autoplay controls muted loop playsinline height="100%">
-            <source src="./static/videos/blueshirt.mp4"
-                    type="video/mp4">
-          </video>
-        </div>
-        <div class="item item-mask">
-          <video poster="" id="mask" autoplay controls muted loop playsinline height="100%">
-            <source src="./static/videos/mask.mp4"
-                    type="video/mp4">
-          </video>
-        </div>
-        <div class="item item-coffee">
-          <video poster="" id="coffee" autoplay controls muted loop playsinline height="100%">
-            <source src="./static/videos/coffee.mp4"
-                    type="video/mp4">
-          </video>
-        </div>
-        <div class="item item-toby">
-          <video poster="" id="toby" autoplay controls muted loop playsinline height="100%">
-            <source src="./static/videos/toby2.mp4"
-                    type="video/mp4">
-          </video>
         </div>
       </div>
-    </div>
-  </div>
-</section>
-<section class="section">
   <div class="container is-max-desktop">
-    <!-- Abstract. -->
     <div class="columns is-centered has-text-centered">
       <div class="column is-four-fifths">
-        <h2 class="title is-3">Abstract</h2>
         <div class="content has-text-justified">
           <p>
-            We present the first method capable of photorealistically reconstructing a non-rigidly
-            deforming scene using photos/videos captured casually from mobile phones.
           </p>
-          <p>
-            Our approach augments neural radiance fields
-            (NeRF) by optimizing an
-            additional continuous volumetric deformation field that warps each observed point into a
-            canonical 5D NeRF.
-            We observe that these NeRF-like deformation fields are prone to local minima, and
-            propose a coarse-to-fine optimization method for coordinate-based models that allows for
-            more robust optimization.
-            By adapting principles from geometry processing and physical simulation to NeRF-like
-            models, we propose an elastic regularization of the deformation field that further
-            improves robustness.
           </p>
           <p>
-            We show that <span class="dnerf">Nerfies</span> can turn casually captured selfie
-            photos/videos into deformable NeRF
-            models that allow for photorealistic renderings of the subject from arbitrary
-            viewpoints, which we dub <i>"nerfies"</i>. We evaluate our method by collecting data
-            using a
-            rig with two mobile phones that take time-synchronized photos, yielding train/validation
-            images of the same pose at different viewpoints. We show that our method faithfully
-            reconstructs non-rigidly deforming scenes and reproduces unseen views with high
-            fidelity.
           </p>
-        </div>
-      </div>
-    </div>
-    <!--/ Abstract. -->
-    <!-- Paper video. -->
-    <div class="columns is-centered has-text-centered">
-      <div class="column is-four-fifths">
-        <h2 class="title is-3">Video</h2>
-        <div class="publication-video">
-          <iframe src="https://www.youtube.com/embed/MrKrnHhk8IA?rel=0&amp;showinfo=0"
-                  frameborder="0" allow="autoplay; encrypted-media" allowfullscreen></iframe>
-        </div>
-      </div>
-    </div>
-    <!--/ Paper video. -->
-  </div>
-</section>
-<section class="section">
-  <div class="container is-max-desktop">
-    <div class="columns is-centered">
-      <!-- Visual Effects. -->
-      <div class="column">
-        <div class="content">
-          <h2 class="title is-3">Visual Effects</h2>
           <p>
-            Using <i>nerfies</i> you can create fun visual effects. This Dolly zoom effect
-            would be impossible without nerfies since it would require going through a wall.
           </p>
-          <video id="dollyzoom" autoplay controls muted loop playsinline height="100%">
-            <source src="./static/videos/dollyzoom-stacked.mp4"
-                    type="video/mp4">
-          </video>
-        </div>
-      </div>
-      <!--/ Visual Effects. -->
-      <!-- Matting. -->
-      <div class="column">
-        <h2 class="title is-3">Matting</h2>
-        <div class="columns is-centered">
-          <div class="column content">
-            <p>
-              As a byproduct of our method, we can also solve the matting problem by ignoring
-              samples that fall outside of a bounding box during rendering.
-            </p>
-            <video id="matting-video" controls playsinline height="100%">
-              <source src="./static/videos/matting.mp4"
-                      type="video/mp4">
-            </video>
-          </div>
         </div>
       </div>
     </div>
-    <!--/ Matting. -->
-    <!-- Animation. -->
-    <div class="columns is-centered">
-      <div class="column is-full-width">
-        <h2 class="title is-3">Animation</h2>
-        <!-- Interpolating. -->
-        <h3 class="title is-4">Interpolating states</h3>
-        <div class="content has-text-justified">
-          <p>
-            We can also animate the scene by interpolating the deformation latent codes of two input
-            frames. Use the slider here to linearly interpolate between the left frame and the right
-            frame.
-          </p>
-        </div>
-        <div class="columns is-vcentered interpolation-panel">
-          <div class="column is-3 has-text-centered">
-            <img src="./static/images/interpolate_start.jpg"
-                 class="interpolation-image"
-                 alt="Interpolate start reference image."/>
-            <p>Start Frame</p>
-          </div>
-          <div class="column interpolation-video-column">
-            <div id="interpolation-image-wrapper">
-              Loading...
             </div>
-            <input class="slider is-fullwidth is-large is-info"
-                   id="interpolation-slider"
-                   step="1" min="0" max="100" value="0" type="range">
-          </div>
-          <div class="column is-3 has-text-centered">
-            <img src="./static/images/interpolate_end.jpg"
-                 class="interpolation-image"
-                 alt="Interpolation end reference image."/>
-            <p class="is-bold">End Frame</p>
           </div>
         </div>
-        <br/>
-        <!--/ Interpolating. -->
-        <!-- Re-rendering. -->
-        <h3 class="title is-4">Re-rendering the input video</h3>
-        <div class="content has-text-justified">
-          <p>
-            Using <span class="dnerf">Nerfies</span>, you can re-render a video from a novel
-            viewpoint such as a stabilized camera by playing back the training deformations.
-          </p>
         </div>
-        <div class="content has-text-centered">
-          <video id="replay-video"
-                 controls
-                 muted
-                 preload
-                 playsinline
-                 width="75%">
-            <source src="./static/videos/replay.mp4"
-                    type="video/mp4">
-          </video>
         </div>
-        <!--/ Re-rendering. -->
       </div>
-    </div>
-    <!--/ Animation. -->
-    <!-- Concurrent Work. -->
-    <div class="columns is-centered">
-      <div class="column is-full-width">
-        <h2 class="title is-3">Related Links</h2>
-        <div class="content has-text-justified">
-          <p>
-            There's a lot of excellent work that was introduced around the same time as ours.
-          </p>
-          <p>
-            <a href="https://arxiv.org/abs/2104.09125" target="_blank">Progressive Encoding for Neural Optimization</a> introduces an idea similar to our windowed position encoding for coarse-to-fine optimization.
-          </p>
-          <p>
-            <a href="https://www.albertpumarola.com/research/D-NeRF/index.html" target="_blank">D-NeRF</a> and <a href="https://gvv.mpi-inf.mpg.de/projects/nonrigid_nerf/" target="_blank">NR-NeRF</a>
-            both use deformation fields to model non-rigid scenes.
-          </p>
-          <p>
-            Some works model videos with a NeRF by directly modulating the density, such as <a href="https://video-nerf.github.io/" target="_blank">Video-NeRF</a>, <a href="https://www.cs.cornell.edu/~zl548/NSFF/" target="_blank">NSFF</a>, and <a href="https://neural-3d-video.github.io/" target="_blank">DyNeRF</a>
-          </p>
-          <p>
-            There are probably many more by the time you are reading this. Check out <a href="https://dellaert.github.io/NeRF/" target="_blank">Frank Dellart's survey on recent NeRF papers</a>, and <a href="https://github.com/yenchenlin/awesome-NeRF" target="_blank">Yen-Chen Lin's curated list of NeRF papers</a>.
-          </p>
         </div>
       </div>
-    </div>
-    <!--/ Concurrent Work. -->
-  </div>
-</section>
-<section class="section" id="BibTeX">
-  <div class="container is-max-desktop content">
-    <h2 class="title">BibTeX</h2>
-    <pre><code>@article{park2021nerfies,
-  author    = {Park, Keunhong and Sinha, Utkarsh and Barron, Jonathan T. and Bouaziz, Sofien and Goldman, Dan B and Seitz, Steven M. and Martin-Brualla, Ricardo},
-  title     = {Nerfies: Deformable Neural Radiance Fields},
-  journal   = {ICCV},
-  year      = {2021},
-}</code></pre>
-  </div>
-</section>
-<footer class="footer">
-  <div class="container">
-    <div class="content has-text-centered">
-      <a class="icon-link" target="_blank"
-         href="./static/videos/nerfies_paper.pdf">
-        <i class="fas fa-file-pdf"></i>
-      </a>
-      <a class="icon-link" href="https://github.com/keunhong" target="_blank" class="external-link" disabled>
-        <i class="fab fa-github"></i>
-      </a>
-    </div>
-    <div class="columns is-centered">
-      <div class="column is-8">
-        <div class="content">
-          <p>
-            This website is licensed under a <a rel="license" target="_blank"
-                                                href="http://creativecommons.org/licenses/by-sa/4.0/">Creative
-            Commons Attribution-ShareAlike 4.0 International License</a>.
-          </p>
-          <p>
-            This means you are free to borrow the <a target="_blank"
-              href="https://github.com/nerfies/nerfies.github.io">source code</a> of this website,
-            we just ask that you link back to this page in the footer.
-            Please remember to remove the analytics code included in the header of the website which
-            you do not want on your website.
-          </p>
-        </div>
-      </div>
-    </div>
-  </div>
-</footer>
 </body>
-</html>

 <!DOCTYPE html>
 <html>
 <head>
   <meta charset="utf-8">
   <meta name="description"
+    content="Deformable Neural Radiance Fields creates free-viewpoint portraits (nerfies) from casually captured videos.">
   <meta name="keywords" content="Nerfies, D-NeRF, NeRF">
   <meta name="viewport" content="width=device-width, initial-scale=1">
+  <title>GroUSE: A Benchmark to Evaluate Evaluators in Grounded Question
+    Answering</title>
+  <link href="https://fonts.googleapis.com/css?family=Google+Sans|Noto+Sans|Castoro" rel="stylesheet">
   <link rel="stylesheet" href="./static/css/bulma.min.css">
+  <!-- <link rel="stylesheet" href="./static/css/bulma-carousel.min.css"> -->
+  <!-- <link rel="stylesheet" href="./static/css/bulma-slider.min.css"> -->
   <link rel="stylesheet" href="./static/css/fontawesome.all.min.css">
+  <link rel="stylesheet" href="https://cdn.jsdelivr.net/gh/jpswalsh/academicons@1/css/academicons.min.css">
   <link rel="stylesheet" href="./static/css/index.css">
   <link rel="icon" href="./static/images/favicon.svg">
   <script src="https://ajax.googleapis.com/ajax/libs/jquery/3.5.1/jquery.min.js"></script>
   <script defer src="./static/js/fontawesome.all.min.js"></script>
   <script src="./static/js/bulma-carousel.min.js"></script>
   <script src="./static/js/bulma-slider.min.js"></script>
   <script src="./static/js/index.js"></script>
 </head>
 <body>
+  <section class="hero">
+    <div class="hero-body">
+      <div class="container is-max-desktop">
+        <div class="columns is-centered">
+          <div class="column has-text-centered">
+            <h1 class="title is-1 publication-title">GroUSE: A Benchmark to Evaluate Evaluators in Grounded Question
+              Answering</h1>
+            <div class="is-size-5 publication-authors">
+              <span class="author-block">
+                <a href="https://www.linkedin.com/in/sacha-muller/" target="_blank">Sacha Muller</a>,</span>
+              <span class="author-block">
+                <a href="https://www.linkedin.com/in/antonio-loison/" target="_blank">António
+                  Loison</a>,</span>
+              <span class="author-block">
+                <a href="https://www.linkedin.com/in/bilel-omrani/" target="_blank">Bilel Omrani</a>,
+              </span>
+              <span class="author-block">
+                <a href="https://www.linkedin.com/in/gautier-viaud/" target="_blank">Gautier Viaud</a>,
+              </span>
+            </div>
+            <div class="is-size-5 publication-authors">
+              <span class="author-block">Illuin Technology</span>
+            </div>
+            <div class="column has-text-centered">
+              <div class="publication-links">
+                <span class="link-block">
+                  <a href="https://arxiv.org/pdf/2409.06595" target="_blank"
+                    class="external-link button is-normal is-rounded is-dark">
+                    <span class="icon">
                       <i class="fas fa-file-pdf"></i>
+                    </span>
+                    <span>Paper</span>
+                  </a>
+                </span>
+                <span class="link-block">
+                  <a href="https://arxiv.org/abs/2409.06595" target="_blank"
+                    class="external-link button is-normal is-rounded is-dark">
+                    <span class="icon">
                       <i class="ai ai-arxiv"></i>
+                    </span>
+                    <span>arXiv</span>
+                  </a>
+                </span>
+                <!-- Code Link. -->
+                <span class="link-block">
+                  <a href="https://github.com/illuin-tech/grouse" target="_blank"
+                    class="external-link button is-normal is-rounded is-dark">
+                    <span class="icon">
                       <i class="fab fa-github"></i>
+                    </span>
+                    <span>Code</span>
                   </a>
+                </span>
+                <!-- Dataset Link. -->
+                <span class="link-block">
+                  <a href="https://github.com/google/nerfies/releases/tag/0.1" target="_blank"
+                    class="external-link button is-normal is-rounded is-dark">
+                    <span class="icon">
+                      <i class="fas fa-database"></i>
+                    </span>
+                    <span>Dataset</span>
                   </a>
+                </span>
+                <!-- Dataset Link. -->
+                <span class="link-block">
+                  <a href="https://huggingface.co/illuin/llama-3-grouse" target="_blank"
+                    class="external-link button is-normal is-rounded is-dark">
+                    <span class="icon">
+                      <i class="fas fa-robot"></i>
+                    </span>
+                    <span>Model</span>
+                  </a>
+              </div>
+            </div>
           </div>
         </div>
       </div>
     </div>
+  </section>
+  <section class="section">
+    <div class="container is-max-desktop">
+      <!-- Abstract. -->
+      <div class="columns is-centered has-text-centered">
+        <div class="column is-four-fifths">
+          <div class="content has-text-justified">
+            <h2 class="title is-3">Context</h2>
+            <p>
+              Grounded Question Answering (QA) is usually the last step of a RAG pipeline: given a question and a set of
+              documents retrieved from the corpus, an LLM must generate an answer. We expect the LLM to cite which
+              document each piece of information is coming from, as depicted below. When no precise answer is in the
+              documents, the LLM should indicate it in its answer. In that case, if some related information is
+              available in the documents, the LLM can add it to the answer to show the corpus is not completely
+              off-topic with respect to the question.
+            </p>
+            <p align="center">
+              <img src="./static/images/grounded_qa_cases.png"
+                alt="Schema showing an example depending on whether the references contain a precise answer, only related information or no information. For each case there is an example of references and ground truth answer. The question is common to the three cases : What is the relationship between Pluto and Neptune. Case 1 : the references contain a precise answer. Reference 1 :  More than 200 objects in 2:3 resonance are known (meaning they complete exactly 2 revolutions around the Sun when Neptune completes 3), among which are Pluto and its moons. Reference 2 : Pluto’s axis of rotation is tilted at 57.5 degrees relative to its orbital plane, which is quite high and unusual in the Solar System. Reference 3 : On the left: view of a cardiac cycle, of a systolic-diastolic oscillating flow, characteristic of circulatory arrest. Ground truth answer : The 3:2 orbital resonance relationship between Pluto and Neptune means that for every 3 revolutions of Neptune around the Sun, Pluto completes 2 [reference 1 citation]. Case 2 : References only contain related information. The reference 1 containing a precise information was removed, the two others are left. Ground truth answer : No document seems to precisely answer your question. However, the documents indicate that : Pluto’s axis of rotation is tilted at 57.5 degrees [reference 2 citation]. Case 3 : References contain no answer nor related information. Reference 1 and 2 were removed, only reference 3 which is off topic if left. Ground truth answer : No document seems to precisely answer your question."
+                style="width:800px;" />
+            </p>
+            <p>
+              This task is difficult to evaluate due to the wide variety of errors an answer can contain, such as
+              superfluous information, missing relevant details from references, incorrect claims that no document
+              answers the question, citation mistakes and so on. Some attempts to define metrics and automatize the
+              evaluation of this task have been made (RAGAS, DeepEval), however, these approaches didn't cover all the
+              failure modes we were interested in.
+            </p>
+            <p>
+              Most of these approaches rely heavily on the <em>LLM-as-a-Judge</em> method. While this technique can be
+              powerful, it is crucial to first assess the ability of an LLM to accurately evaluate this Grounded QA task
+              with respect to our metrics.
+            </p>
+            <p>
+              It is tempting to consider using an LLM to verify the evaluations generated by an evaluator LLM (which was
+              assessing LLM's answers). However, this could quickly lead down a rabbit hole of endless AI-on-AI
+              evaluations. This is why we developed <strong>GroUSE: a unit testing suite designed to evaluate the
+                evaluators</strong>
+              <em>(pronounced "graouse")</em>.
+            </p>
+            <p align="center">
+              <img src="./static/images/meme.jpeg"
+                alt="This image is a two-panel meme: on the left panel, a man labeled 'Grounded QA LLM,' stands confidently. Behind him, a menacing muscular figure labeled 'Evaluator LLM' is holding a bat. On the right panel: the same 'Grounded QA LLM' and 'Evaluator LLM' appear again, but now they are dwarfed by an even larger, more muscular figure labeled 'Grouse'."
+                style="width:450px;" />
+            </p>
+          </div>
         </div>
       </div>
+  </section>
+  <section class="section">
+    <div class="container is-max-desktop">
+      <div class="columns is-centered has-text-centered">
+        <div class="column is-four-fifths">
+          <div class="content has-text-justified">
+            <h2 class="title is-3">The GroUSE dataset</h2>
+            <p>
+              GroUSE (Grounded QA Unitary Scoring of Evaluators) is a dataset of unitary tests used to check if a
+              Grounded QA evaluator is giving the scores we expect. Each test contains:
+            </p>
+            <ul>
+              <li>a Grounded QA sample (consisting of a question and references),</li>
+              <li>a ground truth answer to the question,</li>
+              <li>an answer to evaluate (which may or may not contain an error),</li>
+              <li>a list of expected grades.</li>
+            </ul>
+            <p align="center">
+              <img src="./static/images/one_sample.png"
+                alt="This image is titled 'A simplified sample of GroUSE' and showcases an example of how to evaluate the accuracy and relevance of an answer. The structure is divided into five main sections: 1) Question: 'What is the relationship between Pluto and Neptune?'. 2) A list of two references. Reference 1: 'More than 200 objects in 2:3 resonance are known (meaning they complete exactly 2 revolutions around the Sun when Neptune completes 3), among which are Pluto and its moons.' Reference 2:  'Pluto rotates on its axis every 6.387 days, which is 6 days, 9 hours, and 17 minutes. Its axis of rotation is tilted at 57.5 degrees relative to its orbital plane, which is quite high and unusual in the Solar System.' 3) Ground Truth Answer: 'The 3:2 orbital resonance relationship between Pluto and Neptune means that for every 3 revolutions of Neptune around the Sun, Pluto completes 2 [reference 1 citation].' 4) Answer to Evaluate: 'The 3:2 orbital resonance relationship between Pluto and Neptune means that for every 3 revolutions of Neptune around the Sun, Pluto completes 2 [reference 1 citation]. Pluto’s axis of rotation is tilted at 57.5 degrees [reference 2 citation].' 5) Expected Notes: A rating system different evaluation criteria: Answer Relevancy: less than 4 (indicated in orange). Completeness: 5 (green). Useful: undefined (grey). Faithfulness: 1 (green). Positive Acceptance: undefined (grey). Negative Rejection: undefined (grey)."
+                style="width:600px;" />
+            </p>
+            <p>
+              In our framework, judge LLMs evaluate the quality of a grounded QA answer according to 6 metrics intended
+              to capture all the failure modes of the task:
+            </p>
+            <p>
+            <ul>
+              <li><strong>Answer relevancy</strong> assesses the relevance of the information provided in the answer
+                regarding the question, using a Likert scale (1 to 5).</li>
+              <li><strong>Completeness</strong> also uses a Likert scale to evaluate whether all relevant information
+                from
+                the documents is present in the answer.</li>
+              <li><strong>Faithfulness</strong> is a binary score that checks if all facts in the answer are accurate
+                and
+                correctly attributed to the corresponding document.</li>
+              <li><strong>Usefulness</strong> is only evaluated when the answer states there is no precise answer in the
+                references and provides related information. It is a binary score that determines if the additional
+                information is indeed useful and relevant to the question.</li>
+              <li><strong>Positive Acceptance</strong> and <strong>Negative Rejection</strong> are binary scores
+                indicating a true positive and a true negative respectively in identifying whether the question is
+                answerable.</li>
+            </ul>
+            </p>
+            <p>
+              The GroUSE dataset comprises <strong>144 samples organized into 9 sets</strong>. Each set addresses the
+              same question
+              and draws from largely similar references, with slight variations in the answers. These small
+              modifications are tailored to fit a predefined typology of 16 test types, which are designed to assess
+              whether an evaluator correctly penalizes all failure modes and rewards accurate answers across a diverse
+              range of scenarios. The image below displays four samples along with their corresponding test types. For
+              instance, test type 14 assesses whether the faithfulness score is set to 0 when there is a citation
+              mistake.
+            </p>
+            <p align="center">
+              <img src="./static/images/schema_unit_test_blog.png"
+                alt="Schema showing four type of tests. Type 1: A perfect answer should get the highest notes. Type 2: Related information is not mandatory to get the highest notes. Type 9: Answering in an adversarial situation should result in low negative rejection. Type 14: An error in the citations leads to minimal faithfulness."
+                style="width:1000px;" />
+            </p>
+            <p>
+              GroUSE includes an additional set of tests meant to help users engineer their prompts and try to obtain
+              the best evaluator possible before checking its performances on the 9 other sets. Using this "train set",
+              we iterated on the prompts, making our best effort to craft the best prompts possible for each of the
+              tested models before measuring how many tests they passed. The "train set" is kept small to imitate the
+              real-world scenario where the user has a limited number of samples to optimize its prompts.
+            </p>
+          </div>
+        </div>
+      </div>
+  </section>
   <div class="container is-max-desktop">
     <div class="columns is-centered has-text-centered">
       <div class="column is-four-fifths">
         <div class="content has-text-justified">
+          <h2 class="title is-3">Benchmarking the evaluation abilities of models</h2>
           <p>
+            The structure of the GroUSE dataset allows for presenting a model's results in a matrix format, where each
+            row represents the model's performance on a specific test type, and each column corresponds to its
+            performance on a particular question. This format reveals, for example, that GPT-4 struggles with test type
+            16, which involves an answer containing information that distorts one of the references, leading to a low
+            expected faithfulness but good relevancy and good completeness. Moreover, Llama-3 70B struggles the most
+            with test type 7, a test in which we include an *absurd* fact in the references and mention this fact in the
+            answer. Despite the fact seeming incorrect, since it's present in the references, high scores are expected.
+            Test type 7 allows to check that the model doesn't use its internal knowledge and refers solely to the
+            references to evaluate the metrics.
           </p>
+          <p align="center">
+            <img src="./static/images/matrices.png"
+              alt="12 matrices arranged in two rows of six. The two rows correspond to GPT-4 and LLama-3 70b, the six columns represent the six metrics: answer relevancy, completeness, usefulness, faithfulness, positive acceptance and negative rejection. Each matrix contains 144 small squares, blue if the test passed, red if the test failed, orange if the output was not parsable. We can see that there are more red squares on Llama-3 70b row, especially in the completeness column."
+              style="width:1000px;" />
           </p>
           <p>
+            For a more compact view, we can also calculate the percentage of tests each model passes for each metric:
+          </p>
+          <p align="center">
+            <img src="./static/images/result_table.png"
+              alt="A table with the agreement rate of each metric, for a list of models." style="width:800px;" />
           </p>
           <p>
+            The strongest evaluator models are GPT-4 for closed-weights models, with a pass rate of 95%, and Llama-3 70b
+            for open-weights with 79%. The human performance on this dataset is 98%. The hardest metric to evaluate is
+            completeness, for LLMs and humans alike.
           </p>
         </div>
       </div>
     </div>
+    </section>
+    <section class="section">
+      <div class="container is-max-desktop">
+        <div class="columns is-centered has-text-centered">
+          <div class="column is-four-fifths">
+            <div class="content has-text-justified">
+              <h2 class="title is-3">Improving an open source model</h2>
+              <p>
+                To demonstrate the gap between open-weights and closed-weights models can be narrowed, we finetuned a
+                Llama-3 8b model on traces of evaluations by GPT-4. Aiming to develop a model capable of solving the
+                task in a single call, we concatenated the metric-specific responses from GPT-4 into a single output and
+                followed a similar process for the input, resulting in a dataset of 1200 samples. We finetuned the
+                Llama-3 8b on 1k samples of this dataset, and used the rest as a test set. We measured the model's
+                progression both on GroUSE and by measuring the correlation between GPT-4's grades and the finetuned
+                model's grades on the test set.
+              </p>
+              <p align="center">
+                <img src="./static/images/correlation_table.png"
+                  alt="A table with the correlation between a list of models and GPT-4's evaluation."
+                  style="width:700px;" />
+              </p>
+              <p>
+                Finetuning significantly enhances the evaluation capabilities of Llama-3, as evidenced by the
+                substantial improvement in pass rates, going from a 40% to a 83% test pass rate. A similar progress can
+                be seen on the correlation measures, however it is worth noting that the finetuned model has similar
+                correlation levels than the 0-shot Llama-3 8b with evaluating one metric per prompt. Although this
+                approach demonstrated significant improvements, it would be beneficial to explore the effects of
+                finetuning larger models, which could potentially yield even better performance.
+              </p>
             </div>
           </div>
         </div>
+    </section>
+    <section class="section">
+      <div class="container is-max-desktop">
+        <div class="columns is-centered has-text-centered">
+          <div class="column is-four-fifths">
+            <div class="content has-text-justified">
+              <h2 class="title is-3">Improving an open source model</h2>
+              <p>
+                Our results reveal a discrepancy between GroUSE pass rates and correlation with GPT-4's grades. While
+                Prometheus 2 7b and finetuned Llama-3 8b show similar correlations with GPT-4 on answer relevancy, their
+                GroUSE pass rates differ significantly, with Llama-3 8b outperforming Prometheus 2 7b.
+                Confusion matrices reveal that Prometheus 2 has better overall agreement with GPT-4 but struggles with
+                extreme cases (1, 5 and NaN cases), while finetuned Llama-3 excels in extreme cases but lacks
+                correlation in intermediate ones.
+              </p>
+              <p align="center">
+                <img src="./static/images/confusion_matrixes.png" alt="" style="width:350px;" />
+              </p>
+              <p>
+                This finding suggests that a high correlation with GPT-4's judgments does not necessarily equate to a
+                high unit test pass rate. A judge model can share the same relative preferences as GPT-4 (indicated by
+                strong rank correlation) but still lack the same calibration on precise reference cases (very good
+                answers, subtle mistakes, etc.), resulting in poor performance on judgment unit tests.
+              </p>
+            </div>
+          </div>
         </div>
+    </section>
+    <section class="section">
+      <div class="container is-max-desktop">
+        <div class="columns is-centered has-text-centered">
+          <div class="column is-four-fifths">
+            <div class="content has-text-justified">
+              <h2 class="title is-3">Conclusion</h2>
+              <p>
+                To conclude briefly:
+              </p>
+              <ul>
+                <li>GroUSE is a dataset that allows to check if a model attributes the expected score on a wide range of
+                  cases.</li>
+                <li>Using the LLM-as-a-Judge approach, GPT-4 was the strongest closed weights evaluator and Llama-3 70B
+                  the best open weights evaluator.</li>
+                <li>We demonstrated that a model evaluation abilities can improve with finetuning on a stronger model's
+                  evaluations. </li>
+                <li>We showed that correlation with a strong evaluator does not necessarily imply a good score on the
+                  unit tests. These measures are complementary: correlation with GPT-4 indicates agreement in relative
+                  preference, while GroUSE pass rate measures precise calibration on practical reference cases.</li>
+              </ul>
+              <p>
+                If you want to evaluate your RAG pipeline with our GPT-4 prompts, or even meta-evaluate your RAG
+                evaluator on GroUSE, a python package is available at
+                <a href="https://github.com/illuin-tech/grouse">github.com/illuin-tech/grouse</a> !
+              </p>
+            </div>
+          </div>
         </div>
+    </section>
+    <section class="section" id="BibTeX">
+      <div class="container is-max-desktop content">
+        <h2 class="title">BibTeX</h2>
+        <pre><code>@misc{muller2024grouse,
+          title={GroUSE: A Benchmark to Evaluate Evaluators in Grounded Question Answering},
+          author={Sacha Muller and António Loison and Bilel Omrani and Gautier Viaud},
+          year={2024},
+          eprint={2409.06595},
+          archivePrefix={arXiv},
+          primaryClass={cs.CL},
+          url={https://arxiv.org/abs/2409.06595},
+}</code></pre>
       </div>
+    </section>
+    <!-- <section class="section"></section>
+    <div class="container is-max-desktop">
+      <div class="columns is-centered has-text-centered">
+        <div class="column is-four-fifths">
+          <div class="content has-text-justified">
+            <h2 class="title is-3">Acknowledgments</h2>
+          </div>
         </div>
       </div>
+      </section> -->
+    <footer class="footer">
+    </footer>
 </body>
+</html>

static/images/confusion_matrixes.png ADDED Viewed

static/images/correlation_table.png ADDED Viewed

static/images/favicon.svg DELETED Viewed

static/images/grounded_qa_cases.png ADDED Viewed

static/images/interpolate_end.jpg DELETED Viewed

Binary file (113 kB)

static/images/interpolate_start.jpg DELETED Viewed

Binary file (117 kB)

static/images/matrices.png ADDED Viewed

static/images/meme.jpeg ADDED Viewed

static/images/{steve.webm → one_sample.png} RENAMED Viewed

File without changes

static/images/result_table.png ADDED Viewed

static/images/schema_unit_test.png ADDED Viewed

static/images/schema_unit_test_blog.png ADDED Viewed

static/interpolation/stacked/000000.jpg DELETED Viewed

Binary file (128 kB)

static/interpolation/stacked/000001.jpg DELETED Viewed

Binary file (128 kB)

static/interpolation/stacked/000002.jpg DELETED Viewed

Binary file (128 kB)

static/interpolation/stacked/000003.jpg DELETED Viewed

Binary file (128 kB)

static/interpolation/stacked/000004.jpg DELETED Viewed

Binary file (128 kB)

static/interpolation/stacked/000005.jpg DELETED Viewed

Binary file (129 kB)

static/interpolation/stacked/000006.jpg DELETED Viewed

Binary file (129 kB)

static/interpolation/stacked/000007.jpg DELETED Viewed

Binary file (129 kB)

static/interpolation/stacked/000008.jpg DELETED Viewed

Binary file (129 kB)

static/interpolation/stacked/000009.jpg DELETED Viewed

Binary file (129 kB)

static/interpolation/stacked/000010.jpg DELETED Viewed

Binary file (129 kB)

static/interpolation/stacked/000011.jpg DELETED Viewed

Binary file (129 kB)

static/interpolation/stacked/000012.jpg DELETED Viewed

Binary file (130 kB)

static/interpolation/stacked/000013.jpg DELETED Viewed

Binary file (130 kB)

static/interpolation/stacked/000014.jpg DELETED Viewed

Binary file (130 kB)

static/interpolation/stacked/000015.jpg DELETED Viewed

Binary file (130 kB)

static/interpolation/stacked/000016.jpg DELETED Viewed

Binary file (130 kB)

static/interpolation/stacked/000017.jpg DELETED Viewed

Binary file (131 kB)

static/interpolation/stacked/000018.jpg DELETED Viewed

Binary file (131 kB)

static/interpolation/stacked/000019.jpg DELETED Viewed

Binary file (130 kB)

static/interpolation/stacked/000020.jpg DELETED Viewed

Binary file (130 kB)

static/interpolation/stacked/000021.jpg DELETED Viewed

Binary file (131 kB)

static/interpolation/stacked/000022.jpg DELETED Viewed

Binary file (131 kB)

static/interpolation/stacked/000023.jpg DELETED Viewed

Binary file (131 kB)

static/interpolation/stacked/000024.jpg DELETED Viewed

Binary file (131 kB)

static/interpolation/stacked/000025.jpg DELETED Viewed

Binary file (131 kB)

static/interpolation/stacked/000026.jpg DELETED Viewed

Binary file (131 kB)

static/interpolation/stacked/000027.jpg DELETED Viewed

Binary file (131 kB)

static/interpolation/stacked/000028.jpg DELETED Viewed

Binary file (131 kB)

static/interpolation/stacked/000029.jpg DELETED Viewed

Binary file (131 kB)

static/interpolation/stacked/000030.jpg DELETED Viewed

Binary file (131 kB)

static/interpolation/stacked/000031.jpg DELETED Viewed

Binary file (131 kB)

static/interpolation/stacked/000032.jpg DELETED Viewed

Binary file (131 kB)

static/interpolation/stacked/000033.jpg DELETED Viewed

Binary file (131 kB)

static/interpolation/stacked/000034.jpg DELETED Viewed

Binary file (131 kB)

static/interpolation/stacked/000035.jpg DELETED Viewed

Binary file (131 kB)