update

llm.py

@@ -11,6 +11,8 @@ TKNZ_RATIO = 1
 GEMINI_MODEL = 'gemini-1.5-pro-002'
 FLASH_MODEL = 'gemini-1.5-flash-002'
 
+MAX_OUTPUT_TOKENS = 1024*8
+
 # https://github.com/google-gemini/cookbook/blob/main/quickstarts/Prompting.ipynb
 # https://github.com/google-gemini/cookbook/blob/main/quickstarts/Streaming.ipynb
 import google.generativeai as genai # pip install -U -q google-generativeai
@@ -20,9 +22,9 @@ llm_log_filename = f"{location__}/.cache/llm.log"
 genai.configure(api_key="AIzaSyAUeHVWLkYioIGk6PMbCTqk73PowHCIyPM")
 
 GEMINI_CLIENT = genai.GenerativeModel(GEMINI_MODEL, \
-    generation_config=genai.GenerationConfig(
-        max_output_tokens=1024*4, 
-        temperature=TEMPERATURE
+    generation_config = genai.GenerationConfig(
+        max_output_tokens = MAX_OUTPUT_TOKENS, 
+        temperature = TEMPERATURE,
     ))
 
 def chat(prompt, history=[], use_cache=False, stream=False):
@@ -78,20 +80,16 @@ elif thinker in "70b|405b":
 
     # https://docs.together.ai/docs/chat-models#hosted-models
     model = {
-        "405b": "meta-llama/Meta-Llama-3.1-405B-Instruct-Turbo  128k 4k 1.2", # $3.50 / 1m tokens(*)
-         "70b": "meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo   128k 4k 1.2", # $0.88 / 1m tokens(*)
+        "405b": "meta-llama/Meta-Llama-3.1-405B-Instruct-Turbo  128k", # $3.50 / 1m tokens(*)
+         "70b": "meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo   128k", # $0.88 / 1m tokens(*)
     }[thinker]
 
-    model, CTXLEN, MAX_TOKENS, TKNZ_RATIO = model.strip().split()
+    model, CTXLEN = model.strip().split()
     LLM_HOST = model
 
-    MAX_TOKENS = int(MAX_TOKENS[:-1])*1024
-    TKNZ_RATIO = float(TKNZ_RATIO)
-
     CTXLEN = int(CTXLEN[:-1])
-    if CTXLEN > 64: CTXLEN = 64 # max 32k ctxlen
-    CTXLEN = CTXLEN*1024 - MAX_TOKENS
-    # print(model, CTXLEN, MAX_TOKENS, TKNZ_RATIO); input(); # DEBUG
+    if CTXLEN > 64: CTXLEN = 64 # max 64k ctxlen
+    CTXLEN = CTXLEN*1024 - MAX_OUTPUT_TOKENS
 
     from together import Together
     together_client = Together(api_key='adc0db56b77fe6508bdeadb4d8253771750a50639f8e87313153e49d4599f6ea')
@@ -103,7 +101,7 @@ elif thinker in "70b|405b":
             return together_client.chat.completions.create(
                 model=model,
                 messages=[{"role": "user", "content": prompt}],
-                max_tokens=MAX_TOKENS,
+                max_tokens=MAX_OUTPUT_TOKENS,
                 temperature=TEMPERATURE,
                 top_p=0.7, top_k=50,
                 repetition_penalty=1.2, stop=stops,
@@ -129,7 +127,7 @@ elif thinker in "70b|405b":
                 response = Together(api_key=os.environ.get('TOGETHER_API_KEY')).chat.completions.create(
                     model=model,
                     messages=messages,
-                    max_tokens=MAX_TOKENS,
+                    max_tokens=MAX_OUTPUT_TOKENS,
                     temperature=TEMPERATURE,
                     top_p=0.7, top_k=50,
                     repetition_penalty=1.2, stop=stops,

pages_helpers.py

@@ -569,4 +569,5 @@ https://arxiv.org/html/2409.10516v2
 https://rlhflow.github.io/posts/2024-05-29-multi-objective-reward-modeling
 https://arxiv.org/html/2405.07863v2
 https://arxiv.org/html/2406.12845
+https://eugeneyan.com/writing/llm-evaluators
 """.strip()