Merge pull request #5 from TianyiQ/main

feat: add automatic downloading
PKU-Alignment · Sep 26, 2024 · 5d8dd9f · 5d8dd9f
2 parents b03dc6b + 1dfd6a9
commit 5d8dd9f
Show file tree

Hide file tree

Showing 28 changed files with 115 additions and 88 deletions.
diff --git a/build_dataset.py b/build_dataset.py
@@ -54,6 +54,12 @@ def build_pile_of_law():
 
 if __name__ == "__main__":
     tw.write_log(f"\n\n\n\n\n\n=========== NEW RUN ============\n\n")
+    print('This script is NOT meant to be run as part of the benchmarking process. Unless you would like to replicate the dataset building & model training process, you could directly run `run_benchmark.py` instead, which will automatically download the pre-built dataset and/or models on demand.')
+
+    # Checking if the user wants to proceed
+    if "y" not in input("Start running this script? (y/n)").lower():
+        print("Exiting.")
+        exit()
 
     try:
         proceed = False

diff --git a/doc_generation/build/doctrees/documentation.doctree b/doc_generation/build/doctrees/documentation.doctree
diff --git a/doc_generation/build/doctrees/running.doctree b/doc_generation/build/doctrees/running.doctree
diff --git a/doc_generation/build/doctrees/usage.doctree b/doc_generation/build/doctrees/usage.doctree
diff --git a/doc_generation/source/conf.py b/doc_generation/source/conf.py
@@ -13,7 +13,7 @@
 project = "ProgressGym"
 copyright = "2024 PKU Alignment, Tianyi Qiu, Yang Zhang, Xuchuan Huang, Xinze Li"
 author = "Tianyi Qiu, Yang Zhang, Xuchuan Huang, Xinze Li"
-release = "0.1"
+release = "v0.1"
 
 # -- General configuration ---------------------------------------------------
 # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration

diff --git a/doc_generation/source/documentation.rst b/doc_generation/source/documentation.rst
@@ -1,4 +1,4 @@
-Documentation
+API reference
 ==============
 
 This is the documentation page containing documentation of all key classes and their members of ProgressGym, namely the abstraction classes (Model, Data), the Judge classes and the Examinee classes.

diff --git a/doc_generation/source/running.rst b/doc_generation/source/running.rst
@@ -39,7 +39,7 @@ Basic Usage
 
 **Building Dataset and Training Models**
 
-*Note that you do not need to run this part yourself unless you want to replicate our findings, as the HisText dataset has already been utilized to train the HistLlama model sequence, and is available on* `Huggingface <https://huggingface.co/collections/PKU-Alignment/progressgym-666735fcf3e4efa276226eaa>`_.
+*Note that you do not need to run this part yourself unless you want to replicate our findings, as the HisText dataset has already been utilized to train the HistLlama model sequence, and is available on* `Huggingface <https://huggingface.co/collections/PKU-Alignment/progressgym-666735fcf3e4efa276226eaa>`_, *and the benchmark script will automatically download the HistLlama models when needed.*
 
 Running 
 

diff --git a/doc_generation/source/usage.rst b/doc_generation/source/usage.rst
@@ -1,4 +1,4 @@
-Implementing Your Own Experiment
+Implementing your own experiment
 ================================
 
 To run the benchmark for your own alignment algorithm and assess its temporal alignment

diff --git a/docs/.buildinfo b/docs/.buildinfo
@@ -1,4 +1,4 @@
 # Sphinx build info version 1
 # This file hashes the configuration used when building these files. When it is not found, a full rebuild will be done.
-config: b108215ce31d25eea92f3317b2d0c513
+config: c80b35d0dee5afa2de2271ca68cd2c7d
 tags: 645f666f9bcd5a90fca523b33c5a78b7
diff --git a/docs/Data.html b/docs/Data.html
@@ -5,8 +5,8 @@
     <meta name="color-scheme" content="light dark"><meta name="viewport" content="width=device-width, initial-scale=1" />
 <link rel="index" title="Index" href="genindex.html" /><link rel="search" title="Search" href="search.html" /><link rel="next" title="Judge" href="Judge.html" /><link rel="prev" title="Model" href="Model.html" />
 
-    <!-- Generated with Sphinx 7.1.2 and Furo 2024.07.18 -->
-        <title>Data - ProgressGym 0.1 documentation</title>
+    <!-- Generated with Sphinx 7.1.2 and Furo 2024.08.06 -->
+        <title>Data - ProgressGym v0.1 documentation</title>
       <link rel="stylesheet" type="text/css" href="_static/pygments.css?v=dc19f415" />
     <link rel="stylesheet" type="text/css" href="_static/styles/furo.css?v=387cc868" />
     <link rel="stylesheet" type="text/css" href="_static/styles/furo-extensions.css?v=36a5483c" />
@@ -166,7 +166,7 @@
       </label>
     </div>
     <div class="header-center">
-      <a href="index.html"><div class="brand">ProgressGym 0.1 documentation</div></a>
+      <a href="index.html"><div class="brand">ProgressGym v0.1 documentation</div></a>
     </div>
     <div class="header-right">
       <div class="theme-toggle-container theme-toggle-header">
@@ -190,7 +190,7 @@
       <div class="sidebar-sticky"><a class="sidebar-brand" href="index.html">
 
 
-  <span class="sidebar-brand-text">ProgressGym 0.1 documentation</span>
+  <span class="sidebar-brand-text">ProgressGym v0.1 documentation</span>
 
 </a><form class="sidebar-search-container" method="get" action="search.html" role="search">
   <input class="sidebar-search" placeholder="Search" name="q" aria-label="Search">
@@ -200,8 +200,8 @@
 <div id="searchbox"></div><div class="sidebar-scroll"><div class="sidebar-tree">
   <ul class="current">
 <li class="toctree-l1"><a class="reference internal" href="running.html">Running the benchmark</a></li>
-<li class="toctree-l1"><a class="reference internal" href="usage.html">Implementing Your Own Experiment</a></li>
-<li class="toctree-l1 current has-children"><a class="reference internal" href="documentation.html">Documentation</a><input checked="" class="toctree-checkbox" id="toctree-checkbox-1" name="toctree-checkbox-1" role="switch" type="checkbox"/><label for="toctree-checkbox-1"><div class="visually-hidden">Toggle navigation of Documentation</div><i class="icon"><svg><use href="#svg-arrow-right"></use></svg></i></label><ul class="current">
+<li class="toctree-l1"><a class="reference internal" href="usage.html">Implementing your own experiment</a></li>
+<li class="toctree-l1 current has-children"><a class="reference internal" href="documentation.html">API reference</a><input checked="" class="toctree-checkbox" id="toctree-checkbox-1" name="toctree-checkbox-1" role="switch" type="checkbox"/><label for="toctree-checkbox-1"><div class="visually-hidden">Toggle navigation of API reference</div><i class="icon"><svg><use href="#svg-arrow-right"></use></svg></i></label><ul class="current">
 <li class="toctree-l2"><a class="reference internal" href="Model.html">Model</a></li>
 <li class="toctree-l2 current current-page"><a class="current reference internal" href="#">Data</a></li>
 <li class="toctree-l2"><a class="reference internal" href="Judge.html">Judge</a></li>

diff --git a/docs/Examinee.html b/docs/Examinee.html
@@ -5,8 +5,8 @@
     <meta name="color-scheme" content="light dark"><meta name="viewport" content="width=device-width, initial-scale=1" />
 <link rel="index" title="Index" href="genindex.html" /><link rel="search" title="Search" href="search.html" /><link rel="prev" title="Judge" href="Judge.html" />
 
-    <!-- Generated with Sphinx 7.1.2 and Furo 2024.07.18 -->
-        <title>Examinee - ProgressGym 0.1 documentation</title>
+    <!-- Generated with Sphinx 7.1.2 and Furo 2024.08.06 -->
+        <title>Examinee - ProgressGym v0.1 documentation</title>
       <link rel="stylesheet" type="text/css" href="_static/pygments.css?v=dc19f415" />
     <link rel="stylesheet" type="text/css" href="_static/styles/furo.css?v=387cc868" />
     <link rel="stylesheet" type="text/css" href="_static/styles/furo-extensions.css?v=36a5483c" />
@@ -166,7 +166,7 @@
       </label>
     </div>
     <div class="header-center">
-      <a href="index.html"><div class="brand">ProgressGym 0.1 documentation</div></a>
+      <a href="index.html"><div class="brand">ProgressGym v0.1 documentation</div></a>
     </div>
     <div class="header-right">
       <div class="theme-toggle-container theme-toggle-header">
@@ -190,7 +190,7 @@
       <div class="sidebar-sticky"><a class="sidebar-brand" href="index.html">
 
 
-  <span class="sidebar-brand-text">ProgressGym 0.1 documentation</span>
+  <span class="sidebar-brand-text">ProgressGym v0.1 documentation</span>
 
 </a><form class="sidebar-search-container" method="get" action="search.html" role="search">
   <input class="sidebar-search" placeholder="Search" name="q" aria-label="Search">
@@ -200,8 +200,8 @@
 <div id="searchbox"></div><div class="sidebar-scroll"><div class="sidebar-tree">
   <ul class="current">
 <li class="toctree-l1"><a class="reference internal" href="running.html">Running the benchmark</a></li>
-<li class="toctree-l1"><a class="reference internal" href="usage.html">Implementing Your Own Experiment</a></li>
-<li class="toctree-l1 current has-children"><a class="reference internal" href="documentation.html">Documentation</a><input checked="" class="toctree-checkbox" id="toctree-checkbox-1" name="toctree-checkbox-1" role="switch" type="checkbox"/><label for="toctree-checkbox-1"><div class="visually-hidden">Toggle navigation of Documentation</div><i class="icon"><svg><use href="#svg-arrow-right"></use></svg></i></label><ul class="current">
+<li class="toctree-l1"><a class="reference internal" href="usage.html">Implementing your own experiment</a></li>
+<li class="toctree-l1 current has-children"><a class="reference internal" href="documentation.html">API reference</a><input checked="" class="toctree-checkbox" id="toctree-checkbox-1" name="toctree-checkbox-1" role="switch" type="checkbox"/><label for="toctree-checkbox-1"><div class="visually-hidden">Toggle navigation of API reference</div><i class="icon"><svg><use href="#svg-arrow-right"></use></svg></i></label><ul class="current">
 <li class="toctree-l2"><a class="reference internal" href="Model.html">Model</a></li>
 <li class="toctree-l2"><a class="reference internal" href="Data.html">Data</a></li>
 <li class="toctree-l2"><a class="reference internal" href="Judge.html">Judge</a></li>

diff --git a/docs/Judge.html b/docs/Judge.html
@@ -5,8 +5,8 @@
     <meta name="color-scheme" content="light dark"><meta name="viewport" content="width=device-width, initial-scale=1" />
 <link rel="index" title="Index" href="genindex.html" /><link rel="search" title="Search" href="search.html" /><link rel="next" title="Examinee" href="Examinee.html" /><link rel="prev" title="Data" href="Data.html" />
 
-    <!-- Generated with Sphinx 7.1.2 and Furo 2024.07.18 -->
-        <title>Judge - ProgressGym 0.1 documentation</title>
+    <!-- Generated with Sphinx 7.1.2 and Furo 2024.08.06 -->
+        <title>Judge - ProgressGym v0.1 documentation</title>
       <link rel="stylesheet" type="text/css" href="_static/pygments.css?v=dc19f415" />
     <link rel="stylesheet" type="text/css" href="_static/styles/furo.css?v=387cc868" />
     <link rel="stylesheet" type="text/css" href="_static/styles/furo-extensions.css?v=36a5483c" />
@@ -166,7 +166,7 @@
       </label>
     </div>
     <div class="header-center">
-      <a href="index.html"><div class="brand">ProgressGym 0.1 documentation</div></a>
+      <a href="index.html"><div class="brand">ProgressGym v0.1 documentation</div></a>
     </div>
     <div class="header-right">
       <div class="theme-toggle-container theme-toggle-header">
@@ -190,7 +190,7 @@
       <div class="sidebar-sticky"><a class="sidebar-brand" href="index.html">
 
 
-  <span class="sidebar-brand-text">ProgressGym 0.1 documentation</span>
+  <span class="sidebar-brand-text">ProgressGym v0.1 documentation</span>
 
 </a><form class="sidebar-search-container" method="get" action="search.html" role="search">
   <input class="sidebar-search" placeholder="Search" name="q" aria-label="Search">
@@ -200,8 +200,8 @@
 <div id="searchbox"></div><div class="sidebar-scroll"><div class="sidebar-tree">
   <ul class="current">
 <li class="toctree-l1"><a class="reference internal" href="running.html">Running the benchmark</a></li>
-<li class="toctree-l1"><a class="reference internal" href="usage.html">Implementing Your Own Experiment</a></li>
-<li class="toctree-l1 current has-children"><a class="reference internal" href="documentation.html">Documentation</a><input checked="" class="toctree-checkbox" id="toctree-checkbox-1" name="toctree-checkbox-1" role="switch" type="checkbox"/><label for="toctree-checkbox-1"><div class="visually-hidden">Toggle navigation of Documentation</div><i class="icon"><svg><use href="#svg-arrow-right"></use></svg></i></label><ul class="current">
+<li class="toctree-l1"><a class="reference internal" href="usage.html">Implementing your own experiment</a></li>
+<li class="toctree-l1 current has-children"><a class="reference internal" href="documentation.html">API reference</a><input checked="" class="toctree-checkbox" id="toctree-checkbox-1" name="toctree-checkbox-1" role="switch" type="checkbox"/><label for="toctree-checkbox-1"><div class="visually-hidden">Toggle navigation of API reference</div><i class="icon"><svg><use href="#svg-arrow-right"></use></svg></i></label><ul class="current">
 <li class="toctree-l2"><a class="reference internal" href="Model.html">Model</a></li>
 <li class="toctree-l2"><a class="reference internal" href="Data.html">Data</a></li>
 <li class="toctree-l2 current current-page"><a class="current reference internal" href="#">Judge</a></li>

diff --git a/docs/Model.html b/docs/Model.html
@@ -3,10 +3,10 @@
   <head><meta charset="utf-8"/>
     <meta name="viewport" content="width=device-width,initial-scale=1"/>
     <meta name="color-scheme" content="light dark"><meta name="viewport" content="width=device-width, initial-scale=1" />
-<link rel="index" title="Index" href="genindex.html" /><link rel="search" title="Search" href="search.html" /><link rel="next" title="Data" href="Data.html" /><link rel="prev" title="Documentation" href="documentation.html" />
+<link rel="index" title="Index" href="genindex.html" /><link rel="search" title="Search" href="search.html" /><link rel="next" title="Data" href="Data.html" /><link rel="prev" title="API reference" href="documentation.html" />
 
-    <!-- Generated with Sphinx 7.1.2 and Furo 2024.07.18 -->
-        <title>Model - ProgressGym 0.1 documentation</title>
+    <!-- Generated with Sphinx 7.1.2 and Furo 2024.08.06 -->
+        <title>Model - ProgressGym v0.1 documentation</title>
       <link rel="stylesheet" type="text/css" href="_static/pygments.css?v=dc19f415" />
     <link rel="stylesheet" type="text/css" href="_static/styles/furo.css?v=387cc868" />
     <link rel="stylesheet" type="text/css" href="_static/styles/furo-extensions.css?v=36a5483c" />
@@ -166,7 +166,7 @@
       </label>
     </div>
     <div class="header-center">
-      <a href="index.html"><div class="brand">ProgressGym 0.1 documentation</div></a>
+      <a href="index.html"><div class="brand">ProgressGym v0.1 documentation</div></a>
     </div>
     <div class="header-right">
       <div class="theme-toggle-container theme-toggle-header">
@@ -190,7 +190,7 @@
       <div class="sidebar-sticky"><a class="sidebar-brand" href="index.html">
 
 
-  <span class="sidebar-brand-text">ProgressGym 0.1 documentation</span>
+  <span class="sidebar-brand-text">ProgressGym v0.1 documentation</span>
 
 </a><form class="sidebar-search-container" method="get" action="search.html" role="search">
   <input class="sidebar-search" placeholder="Search" name="q" aria-label="Search">
@@ -200,8 +200,8 @@
 <div id="searchbox"></div><div class="sidebar-scroll"><div class="sidebar-tree">
   <ul class="current">
 <li class="toctree-l1"><a class="reference internal" href="running.html">Running the benchmark</a></li>
-<li class="toctree-l1"><a class="reference internal" href="usage.html">Implementing Your Own Experiment</a></li>
-<li class="toctree-l1 current has-children"><a class="reference internal" href="documentation.html">Documentation</a><input checked="" class="toctree-checkbox" id="toctree-checkbox-1" name="toctree-checkbox-1" role="switch" type="checkbox"/><label for="toctree-checkbox-1"><div class="visually-hidden">Toggle navigation of Documentation</div><i class="icon"><svg><use href="#svg-arrow-right"></use></svg></i></label><ul class="current">
+<li class="toctree-l1"><a class="reference internal" href="usage.html">Implementing your own experiment</a></li>
+<li class="toctree-l1 current has-children"><a class="reference internal" href="documentation.html">API reference</a><input checked="" class="toctree-checkbox" id="toctree-checkbox-1" name="toctree-checkbox-1" role="switch" type="checkbox"/><label for="toctree-checkbox-1"><div class="visually-hidden">Toggle navigation of API reference</div><i class="icon"><svg><use href="#svg-arrow-right"></use></svg></i></label><ul class="current">
 <li class="toctree-l2 current current-page"><a class="current reference internal" href="#">Model</a></li>
 <li class="toctree-l2"><a class="reference internal" href="Data.html">Data</a></li>
 <li class="toctree-l2"><a class="reference internal" href="Judge.html">Judge</a></li>
@@ -421,7 +421,7 @@ <h1>Model<a class="headerlink" href="#model" title="Permalink to this heading">
                   <span>Previous</span>
                 </div>
 
-                <div class="title">Documentation</div>
+                <div class="title">API reference</div>
 
               </div>
             </a>

diff --git a/docs/_sources/documentation.rst.txt b/docs/_sources/documentation.rst.txt
@@ -1,4 +1,4 @@
-Documentation
+API reference
 ==============
 
 This is the documentation page containing documentation of all key classes and their members of ProgressGym, namely the abstraction classes (Model, Data), the Judge classes and the Examinee classes.

diff --git a/docs/_sources/running.rst.txt b/docs/_sources/running.rst.txt
@@ -39,7 +39,7 @@ Basic Usage
 
 **Building Dataset and Training Models**
 
-*Note that you do not need to run this part yourself unless you want to replicate our findings, as the HisText dataset has already been utilized to train the HistLlama model sequence, and is available on* `Huggingface <https://huggingface.co/collections/PKU-Alignment/progressgym-666735fcf3e4efa276226eaa>`_.
+*Note that you do not need to run this part yourself unless you want to replicate our findings, as the HisText dataset has already been utilized to train the HistLlama model sequence, and is available on* `Huggingface <https://huggingface.co/collections/PKU-Alignment/progressgym-666735fcf3e4efa276226eaa>`_, *and the benchmark script will automatically download the HistLlama models when needed.*
 
 Running 
 

diff --git a/docs/_sources/usage.rst.txt b/docs/_sources/usage.rst.txt
@@ -1,4 +1,4 @@
-Implementing Your Own Experiment
+Implementing your own experiment
 ================================
 
 To run the benchmark for your own alignment algorithm and assess its temporal alignment