# credit to: https://github.com/KoushikNavuluri/Claude-API/blob/main/claude-api/claude_api.py

import json
import os
import uuid
from curl_cffi import requests
import requests as req
import re


class Client:

  def __init__(self, cookie):
    self.cookie = cookie
    self.organization_id = self.get_organization_id()

  def get_organization_id(self):
    url = "https://claude.ai/api/organizations"

    headers = {
        'User-Agent':
        'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/115.0',
        'Accept-Language': 'en-US,en;q=0.5',
        'Referer': 'https://claude.ai/chats',
        'Content-Type': 'application/json',
        'Sec-Fetch-Dest': 'empty',
        'Sec-Fetch-Mode': 'cors',
        'Sec-Fetch-Site': 'same-origin',
        'Connection': 'keep-alive',
        'Cookie': f'{self.cookie}'
    }

    response = requests.get(url, headers=headers,impersonate="chrome110")
    res = json.loads(response.text)
    uuid = res[0]['uuid']

    return uuid

  def get_content_type(self, file_path):
    # Function to determine content type based on file extension
    extension = os.path.splitext(file_path)[-1].lower()
    if extension == '.pdf':
      return 'application/pdf'
    elif extension == '.txt':
      return 'text/plain'
    elif extension == '.csv':
      return 'text/csv'
    # Add more content types as needed for other file types
    else:
      return 'application/octet-stream'

  # Lists all the conversations you had with Claude
  def list_all_conversations(self):
    url = f"https://claude.ai/api/organizations/{self.organization_id}/chat_conversations"

    headers = {
        'User-Agent':
        'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/115.0',
        'Accept-Language': 'en-US,en;q=0.5',
        'Referer': 'https://claude.ai/chats',
        'Content-Type': 'application/json',
        'Sec-Fetch-Dest': 'empty',
        'Sec-Fetch-Mode': 'cors',
        'Sec-Fetch-Site': 'same-origin',
        'Connection': 'keep-alive',
        'Cookie': f'{self.cookie}'
    }

    response = requests.get(url, headers=headers,impersonate="chrome110")
    conversations = response.json()

    # Returns all conversation information in a list
    if response.status_code == 200:
      return conversations
    else:
      print(f"Error: {response.status_code} - {response.text}")

  # Send Message to Claude
  def send_message(self, prompt, conversation_id, attachment=None,timeout=500):
    url = "https://claude.ai/api/append_message"

    # Upload attachment if provided
    attachments = []
    if attachment:
      attachment_response = self.upload_attachment(attachment)
      if attachment_response:
        attachments = [attachment_response]
      else:
        return {"Error: Invalid file format. Please try again."}

    # Ensure attachments is an empty list when no attachment is provided
    if not attachment:
      attachments = []

    payload = json.dumps({
      "completion": {
        "prompt": f"{prompt}",
        "timezone": "Asia/Kolkata",
        "model": "claude-2"
      },
      "organization_uuid": f"{self.organization_id}",
      "conversation_uuid": f"{conversation_id}",
      "text": f"{prompt}",
      "attachments": attachments
    })

    headers = {
      'User-Agent':
      'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/115.0',
      'Accept': 'text/event-stream, text/event-stream',
      'Accept-Language': 'en-US,en;q=0.5',
      'Referer': 'https://claude.ai/chats',
      'Content-Type': 'application/json',
      'Origin': 'https://claude.ai',
      'DNT': '1',
      'Connection': 'keep-alive',
      'Cookie': f'{self.cookie}',
      'Sec-Fetch-Dest': 'empty',
      'Sec-Fetch-Mode': 'cors',
      'Sec-Fetch-Site': 'same-origin',
      'TE': 'trailers'
    }

    response = requests.post( url, headers=headers, data=payload, impersonate="chrome110", timeout=500)
    account = self.cookie.split("--")[-1]
    if response.status_code == 429:    
        print(f"account {account} is out of usage. (status code: 429)")
        return None
    decoded_data = response.content.decode("utf-8")
    # if "error" in decoded_data:
    #     print(f"account {account} is out of usage.")
    #     print(f"error msg: {decoded_data}")
    #     return None
    decoded_data = re.sub('\n+', '\n', decoded_data).strip()
    data_strings = decoded_data.split('\n')
    completions = []
    for data_string in data_strings:
      json_str = data_string[6:].strip()
      data = json.loads(json_str)
      if 'completion' in data:
        completions.append(data['completion'])

    answer = ''.join(completions)

    # Returns answer
    return answer

  # Deletes the conversation
  def delete_conversation(self, conversation_id):
    url = f"https://claude.ai/api/organizations/{self.organization_id}/chat_conversations/{conversation_id}"

    payload = json.dumps(f"{conversation_id}")
    headers = {
        'User-Agent':
        'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/115.0',
        'Accept-Language': 'en-US,en;q=0.5',
        'Content-Type': 'application/json',
        'Content-Length': '38',
        'Referer': 'https://claude.ai/chats',
        'Origin': 'https://claude.ai',
        'Sec-Fetch-Dest': 'empty',
        'Sec-Fetch-Mode': 'cors',
        'Sec-Fetch-Site': 'same-origin',
        'Connection': 'keep-alive',
        'Cookie': f'{self.cookie}',
        'TE': 'trailers'
    }

    response = requests.delete( url, headers=headers, data=payload,impersonate="chrome110")

    # Returns True if deleted or False if any error in deleting
    if response.status_code == 204:
      return True
    else:
      return False

  # Returns all the messages in conversation
  def chat_conversation_history(self, conversation_id):
    url = f"https://claude.ai/api/organizations/{self.organization_id}/chat_conversations/{conversation_id}"

    headers = {
        'User-Agent':
        'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/115.0',
        'Accept-Language': 'en-US,en;q=0.5',
        'Referer': 'https://claude.ai/chats',
        'Content-Type': 'application/json',
        'Sec-Fetch-Dest': 'empty',
        'Sec-Fetch-Mode': 'cors',
        'Sec-Fetch-Site': 'same-origin',
        'Connection': 'keep-alive',
        'Cookie': f'{self.cookie}'
    }

    response = requests.get( url, headers=headers,impersonate="chrome110")
    

    # List all the conversations in JSON
    return response.json()

  def generate_uuid(self):
    random_uuid = uuid.uuid4()
    random_uuid_str = str(random_uuid)
    formatted_uuid = f"{random_uuid_str[0:8]}-{random_uuid_str[9:13]}-{random_uuid_str[14:18]}-{random_uuid_str[19:23]}-{random_uuid_str[24:]}"
    return formatted_uuid

  def create_new_chat(self):
    url = f"https://claude.ai/api/organizations/{self.organization_id}/chat_conversations"
    uuid = self.generate_uuid()

    payload = json.dumps({"uuid": uuid, "name": ""})
    headers = {
        'User-Agent':
        'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/115.0',
        'Accept-Language': 'en-US,en;q=0.5',
        'Referer': 'https://claude.ai/chats',
        'Content-Type': 'application/json',
        'Origin': 'https://claude.ai',
        'DNT': '1',
        'Connection': 'keep-alive',
        'Cookie': self.cookie,
        'Sec-Fetch-Dest': 'empty',
        'Sec-Fetch-Mode': 'cors',
        'Sec-Fetch-Site': 'same-origin',
        'TE': 'trailers'
    }

    response = requests.post( url, headers=headers, data=payload,impersonate="chrome110")

    # Returns JSON of the newly created conversation information
    return response.json()

  # Resets all the conversations
  def reset_all(self):
    conversations = self.list_all_conversations()

    for conversation in conversations:
      conversation_id = conversation['uuid']
      delete_id = self.delete_conversation(conversation_id)

    return True

  def upload_attachment(self, file_path):
    if file_path.endswith('.txt'):
      file_name = os.path.basename(file_path)
      file_size = os.path.getsize(file_path)
      file_type = "text/plain"
      with open(file_path, 'r', encoding='utf-8') as file:
        file_content = file.read()

      return {
          "file_name": file_name,
          "file_type": file_type,
          "file_size": file_size,
          "extracted_content": file_content
      }
    url = 'https://claude.ai/api/convert_document'
    headers = {
        'User-Agent':
        'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/115.0',
        'Accept-Language': 'en-US,en;q=0.5',
        'Referer': 'https://claude.ai/chats',
        'Origin': 'https://claude.ai',
        'Sec-Fetch-Dest': 'empty',
        'Sec-Fetch-Mode': 'cors',
        'Sec-Fetch-Site': 'same-origin',
        'Connection': 'keep-alive',
        'Cookie': f'{self.cookie}',
        'TE': 'trailers'
    }

    file_name = os.path.basename(file_path)
    content_type = self.get_content_type(file_path)

    files = {
        'file': (file_name, open(file_path, 'rb'), content_type),
        'orgUuid': (None, self.organization_id)
    }

    response = req.post(url, headers=headers, files=files)
    if response.status_code == 200:
      return response.json()
    else:
      return False
      

    
  # Renames the chat conversation title
  def rename_chat(self, title, conversation_id):
    url = "https://claude.ai/api/rename_chat"

    payload = json.dumps({
        "organization_uuid": f"{self.organization_id}",
        "conversation_uuid": f"{conversation_id}",
        "title": f"{title}"
    })
    headers = {
        'User-Agent':
        'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/115.0',
        'Accept-Language': 'en-US,en;q=0.5',
        'Content-Type': 'application/json',
        'Referer': 'https://claude.ai/chats',
        'Origin': 'https://claude.ai',
        'Sec-Fetch-Dest': 'empty',
        'Sec-Fetch-Mode': 'cors',
        'Sec-Fetch-Site': 'same-origin',
        'Connection': 'keep-alive',
        'Cookie': f'{self.cookie}',
        'TE': 'trailers'
    }

    response = requests.post(url, headers=headers, data=payload,impersonate="chrome110")

    if response.status_code == 200:
      return True
    else:
      return False
claude.py
dinobby/ReConcile
# credit to https://github.com/swarnaHub/ExplanationIntervention/blob/main/src/data_utils.py
import os
import re
import json
import random
import pickle
import pandas

random.seed(1234)

class StrategyQA:
    def __init__(self, data_dir):
        self.dev_path = os.path.join(data_dir, "dev.json")

    def get_samples(self, file_path):
        samples = []
        with open(file_path, "r", encoding="utf-8-sig") as f:
            json_inputs = json.load(f)
            for i, json_input in enumerate(json_inputs):
                samples.append({
                    "index": i,
                    "qid": json_input["qid"],
                    "question": json_input["question"],
                    "answer": "yes" if json_input["answer"] else "no",
                    "gold_explanation": " ".join(json_input["facts"])
                })

        return samples

    def get_test_samples(self):
        return self.get_samples(self.dev_path)

class GSM8k:
    def __init__(self, data_dir):
        self.test_path = os.path.join(data_dir, "test.jsonl")

    def get_samples(self, file_path):
        samples = []
        count = 0
        with open(file_path, "r") as f:
            jsonlines = f.read().splitlines()
            for i, jsonline in enumerate(jsonlines):
                sample = json.loads(jsonline)
                answer = re.sub(r"[^0-9.]", "",sample["answer"].split("#### ")[1].strip())
                gold_explanation = re.sub('<<.*>>', '', sample["answer"].split("#### ")[0].replace("\n\n", "\n").strip())
                gold_explanation_sents = gold_explanation.split("\n")
                gold_explanation_sents = [gold_explanation_sent + "." if gold_explanation_sent[-1] != "." else gold_explanation_sent for gold_explanation_sent in gold_explanation_sents]
                gold_explanation = " ".join(gold_explanation_sents)
                sample_json = {
                    "index": i,
                    "question": sample["question"],
                    "answer": answer,
                    "gold_explanation": gold_explanation
                }
                samples.append(sample_json)

        return samples

    def get_test_samples(self):
        return self.get_samples(self.test_path)
    
class Aqua:
    def __init__(self, data_dir):
        self.test_path = os.path.join(data_dir, "test.json")

    def get_samples(self, file_path):
        samples = []
        data = [json.loads(line) for line in open(file_path, 'r')]
        for i, json_input in enumerate(data):
            samples.append({
                "index": i,
                "question": json_input["question"],
                "options": json_input["options"],
                "answer": json_input["correct"],
                "gold_explanation": json_input["rationale"]
            })

        return samples

    def get_test_samples(self):
        return self.get_samples(self.test_path)
        
class ECQA:
    def __init__(self, data_dir):
        self.test_path = os.path.join(data_dir, "cqa_data_test.csv")

    def get_samples(self, file_path):
        samples = []
        df = pandas.read_csv(file_path)
        for index, row in df.iterrows():
            options = [row["q_op1"], row["q_op2"], row["q_op3"], row["q_op4"], row["q_op5"]]
            samples.append({
                "index": index,
                "question": row["q_text"],
                "options": options,
                "answer": str(options.index(row["q_ans"]) + 1),
                "gold_explanation": row["taskB"]
            })

        return samples

    def get_test_samples(self):
        return self.get_samples(self.test_path)
    
data_utils.py
dinobby/ReConcile
import os
import time
import backoff
from tqdm import tqdm
from dotenv import load_dotenv
from utils import prepare_context, prepare_context_for_chat_assistant, prepare_context_for_bard, parse_json, invalid_result

import openai
from claude import Client
import google.generativeai as palm

from curl_cffi import CurlError
from json import JSONDecodeError
from requests.exceptions import RequestException
from google.api_core.exceptions import ServiceUnavailable
from openai.error import RateLimitError, APIError, ServiceUnavailableError, APIConnectionError, InvalidRequestError

load_dotenv()

openai.api_type = "azure"
openai.api_base = os.environ['OPEN_AI_API_BASE']
openai.api_version = os.environ['OPEN_AI_API_VERSION']
openai.api_key = os.environ['OPEN_AI_API_KEY']
palm.configure(api_key=os.environ['PALM_API_KEY'])
# Note: you can add more account in .env and here
claude_coockies = [c for c in [os.environ['CLAUDE_COOCKIE1'], os.environ['CLAUDE_COOCKIE2'], os.environ['CLAUDE_COOCKIE3'], os.environ['CLAUDE_COOCKIE4'], os.environ['CLAUDE_COOCKIE5']] if c]

class ClaudeModel:
    def __init__(self):
        self.coockies = claude_coockies
        self.claude_api = self.connect()
        
    @backoff.on_exception(backoff.expo, (CurlError, RequestException), max_tries=5)   
    def connect(self):
        for coockie in self.coockies:
            claude_api = Client(coockie)
            uuid = claude_api.create_new_chat()['uuid']
            output = claude_api.send_message("Hi, what is your name and where are you from?", uuid)
            if output:
                print(output)
                print("claude connected successfully.")
                return claude_api
        print("all coockies are not available now.")
        return None
        
    @backoff.on_exception(backoff.expo, (CurlError, RequestException, ValueError), max_tries=3)
    def claude_gen_ans(self, sample, convincing_samples=None, additional_instruc=None, intervene=False, dataset="SQA"):
        contexts = prepare_context(sample, convincing_samples, intervene, dataset)
    
        if additional_instruc:
            contexts += " ".join(additional_instruc)
              
        if not self.claude_api:
            print("claude is not connected. trying to connect...")
            self.claude_api = self.connect()
            print("claude is connected.")
        uuid = self.claude_api.create_new_chat()['uuid']
        output = self.claude_api.send_message(contexts, uuid)
            
        if output:
            result = parse_json(output)
            if result == "ERR_SYNTAX":
                print("incomplete JSON format.")
                print(output)
                print("-----------------------")
                raise ValueError("incomplete JSON format.")
        else:
            print("reconnecting claude using different coockie...")
            self.claude_api = self.connect()
            if self.claude_api:
                uuid = self.claude_api.create_new_chat()['uuid']
                output = self.claude_api.send_message(contexts, uuid)
                result = parse_json(output)
            else:
                print("take a rest for the count down...")
                return 403
                
        if not result:
            result = invalid_result(dataset)
            
        if dataset == "SQA":
            result['answer'] = result['answer'].lower()
        elif dataset == "Aqua":
            result['answer'] = result['answer'].upper()
        elif dataset in ["GSM8k", "ECQA"]:
            result['answer'] = str(result['answer'])
      
        return result

    def claude_debate(self, test_samples, all_results, rounds, convincing_samples, dataset):
        r = '_' + str(rounds-1)
        result = None
        for i, s in tqdm(enumerate(all_results)):
            if 'claude_output_'+str(rounds) not in s and 'debate_prompt'+ r in s and len(s['debate_prompt'+r]):
                additional_instruc = ["\n\nCarefully review the following solutions from other agents as additional information, and provide your own answer and step-by-step reasoning to the question."]
                additional_instruc.append("Clearly states that which pointview do you agree or disagree and why.\n\n")
                additional_instruc.append(s['debate_prompt'+r])
                additional_instruc.append("Output your answer in json format, with the format as follows: {\"reasoning\": \"\", \"answer\": \"\", \"confidence_level\": \"\"}. Please strictly output in JSON format.")
                try:
                    result = self.claude_gen_ans(test_samples[i],
                                                convincing_samples=convincing_samples,
                                                additional_instruc=additional_instruc,
                                                intervene=False,
                                                dataset=dataset)
                except ValueError:
                    print(result)
                    s['claude_output_'+str(rounds)] = s['claude_output'+r]
                if result != 403:
                    s['claude_output_'+str(rounds)] = result
                else:
                    print("taking a rest for the count down...")
                    break
        return all_results

@backoff.on_exception(backoff.expo, (RateLimitError, APIError, ServiceUnavailableError, APIConnectionError, ValueError), max_tries=5)
def gpt_gen_ans(sample, convincing_samples=None, additional_instruc=None, intervene=False, dataset="SQA"):
    contexts = prepare_context_for_chat_assistant(sample, convincing_samples, intervene, dataset)
    if additional_instruc:
        contexts[-1]['content'] += " ".join(additional_instruc)
    # print(contexts)
    completion = openai.ChatCompletion.create(
              engine="gpt-35-turbo",
              messages=contexts)
    
    output = completion['choices'][0]['message']['content']
    if output:
        if "{" not in output or "}" not in output:
            raise ValueError("cannot find { or } in the model output.")
        result = parse_json(output)
        if result == "ERR_SYNTAX":
            raise ValueError("incomplete JSON format.")
            
    if not result:
        result = invalid_result(dataset)
        
    if dataset == "SQA":
        result['answer'] = result['answer'].lower()
    elif dataset == "Aqua":
        result['answer'] = result['answer'].upper()
    elif dataset in ["GSM8k", "ECQA"]:
        result['answer'] = str(result['answer'])

    return result

@backoff.on_exception(backoff.expo, (ServiceUnavailable, ValueError, TypeError), max_tries=5)
def bard_gen_ans(sample, convincing_samples=None, additional_instruc=None, intervene=False, dataset="SQA"):
    msg, cs, us = prepare_context_for_bard(sample, convincing_samples, intervene, dataset)
    
    if additional_instruc:
        msg += " ".join(additional_instruc) 

    response = palm.chat(
        examples=cs+us,
        messages=msg)
    
    if not response.last:
        raise ValueError
    if "{" not in response.last and "}" not in response.last: 
        # Bard sometimes doesn't follow the instruction of generate a JSON format output
        print("parsing the output into json format using bard...")
        bard_json = bard_transform_json(response.last, dataset)
        result = parse_json(bard_json)
    else:
        result = parse_json(response.last)
        
    if result == "ERR_SYNTAX":
        raise ValueError("incomplete JSON format.") 
        
    if dataset == "SQA":
        result['answer'] = result['answer'].lower()
    elif dataset == "Aqua":
        result['answer'] = result['answer'].upper()
    elif dataset in ["GSM8k", "ECQA"]:
        result['answer'] = str(result['answer'])
    return result

def gpt_debate(test_samples, all_results, rounds, convincing_samples, dataset):
    r = '_' + str(rounds-1)
    for i, s in tqdm(enumerate(all_results)):
        if 'gpt3_output_'+str(rounds) not in s and 'debate_prompt'+ r in s and len(s['debate_prompt'+r]):
            additional_instruc = ["\n\nCarefully review the following solutions from other agents as additional information, and provide your own answer and step-by-step reasoning to the question."]
            additional_instruc.append("Clearly states that which pointview do you agree or disagree and why.\n\n")
            additional_instruc.append(s['debate_prompt'+r])
            additional_instruc.append("Output your answer in json format, with the format as follows: {\"reasoning\": \"\", \"answer\": \"\", \"confidence_level\": \"\"}. Please strictly output in JSON format.")
            result = gpt_gen_ans(test_samples[i],
                                 convincing_samples=convincing_samples,
                                 additional_instruc=additional_instruc,
                                 intervene=False,
                                 dataset=dataset)            
            s['gpt3_output_'+str(rounds)] = result
    return all_results

def bard_debate(test_samples, all_results, rounds, convincing_samples, dataset):
    r = '_' + str(rounds-1)
    for i, s in tqdm(enumerate(all_results)):
        if 'bard_output_'+str(rounds) not in s and 'debate_prompt'+ r in s and len(s['debate_prompt'+r]):
            additional_instruc = ["\n\nCarefully review the following solutions from other agents as additional information, and provide your own answer and step-by-step reasoning to the question."]
            additional_instruc.append("Clearly states that which pointview do you agree or disagree and why.\n\n")
            additional_instruc.append(s['debate_prompt'+r])
            additional_instruc.append("Output your answer in json format, with the format as follows: {\"reasoning\": \"\", \"answer\": \"\", \"confidence_level\": \"\"}. Please strictly output in JSON format.")
            try:
                result = bard_gen_ans(test_samples[i],
                                      convincing_samples=convincing_samples,
                                      additional_instruc=additional_instruc,
                                      intervene=False,
                                      dataset=dataset)
            except ValueError:
                print("cannot generate valid answer for this sample.")
                result = invalid_result(dataset)
            
            s['bard_output_'+str(rounds)] = result
            time.sleep(1)
    return all_results

def bard_transform_json(model_output, dataset):
    prompt = "Transform the following paragraph to fit in the JSON format: {\"reasoning\": \"\", \"answer\": \"\", \"confidence_level\": \"\"}"
    prompt += "Place all of the reasoning of why the answer is derived in \"reasoning\" field."
    if dataset == "SQA":
        prompt += "Place only yes or no in the \"answer\" field."
    elif dataset=="GSM8k":
        prompt += "Place only a single numeric value in the \"answer\" field."
    elif dataset=="ECQA":
        prompt += "Place only 1,2,3,4,5 representing your choice in the \"answer\" field."
    elif dataset=="Aqua":
        prompt += "Place only A,B,C,D,E representing your choice in the \"answer\" field."
    prompt += "Place the confidence level in the \"confidence_level\" field."
    prompt += model_output
    
    response = palm.chat(messages=prompt)
    return response.last


generation.py
dinobby/ReConcile
import re
import time
import json
import pickle
import random
import argparse
import numpy as np
from utils import *
from generation import *
from tqdm import tqdm
from data_utils import StrategyQA, GSM8k, Aqua, ECQA

if __name__ == '__main__':
    parser = argparse.ArgumentParser()

    parser.add_argument('--dataset', default='SQA', type=str)
    parser.add_argument('--num_samples', default=100, type=int)
    parser.add_argument('--round', default=2, type=int)
    args = parser.parse_args()

    if args.dataset == "SQA":
        data = StrategyQA(data_dir=f'./dataset/{args.dataset}')
    elif args.dataset == "ECQA":
        data = ECQA(data_dir=f'./dataset/{args.dataset}')
    elif args.dataset == "GSM8k":
        data = GSM8k(data_dir=f'./dataset/{args.dataset}')
    elif args.dataset == "Aqua":
        data = Aqua(data_dir=f'./dataset/{args.dataset}')

    test_samples = data.get_test_samples()[:args.num_samples]
    print(f"Number of test samples={len(test_samples)}")

    with open(f'convincing/{args.dataset}/chatgpt.json', 'r') as f:
        convincing_gpt = json.load(f)
    with open(f'convincing/{args.dataset}/claude.json', 'r') as f:
        convincing_claude = json.load(f)
    with open(f'convincing/{args.dataset}/bard.json', 'r') as f:
        convincing_bard = json.load(f)

    claude = ClaudeModel()

    # Phase1: Initial Response Generation

    claude_result = []
    while True:
        for test_sample in tqdm(test_samples[len(claude_result):]):
            tmp = {}
            tmp['gold_answer'] = test_sample['answer']
            try:
                result = claude.claude_gen_ans(test_sample,
                                            convincing_samples=convincing_gpt+convincing_bard,
                                            additional_instruc=None,
                                            intervene=False,
                                            dataset=args.dataset)
            except ValueError:
                print("cannot generate valid response for this sample.")
                result = invalid_result(args.dataset)
            if result == 403:
                pause = input("rate limit: let's take a break. enter anything to resume: ")
                if pause: break

            tmp['prediction'] = result
            claude_result.append(tmp)
            time.sleep(1)
        break

    gpt_result = []
    for test_sample in tqdm(test_samples[len(gpt_result):]):
        tmp = {}
        tmp['gold_answer'] = test_sample['answer']
        try:
            result = gpt_gen_ans(test_sample,
                                convincing_samples=convincing_claude+convincing_bard,
                                additional_instruc=None,
                                intervene=False,
                                dataset=args.dataset)
        except InvalidRequestError:
            print("blocked by Azure OpenAI’s content management policy.")
            result = invalid_result(args.dataset)
        tmp['prediction'] = result
        gpt_result.append(tmp)
        time.sleep(1)
   
    bard_result = []
    for test_sample in tqdm(test_samples[len(bard_result):]):
        tmp = {}
        tmp['gold_answer'] = test_sample['answer']
        try:
            result = bard_gen_ans(test_sample,
                                convincing_samples=convincing_claude+convincing_gpt,
                                additional_instruc=None,
                                intervene=False,
                                dataset=args.dataset)   
            tmp['prediction'] = result
        except ValueError:
            tmp['prediction'] = invalid_result(args.dataset)
        
        bard_result.append(tmp)
        time.sleep(1)

    # Evaluation for the initial round

    all_results = []
    for c, g, b in zip(claude_result, gpt_result, bard_result):
        tmp = {}
        tmp['gold_answer'] = c['gold_answer']
        tmp['claude_output_0'] = c['prediction']
        tmp['gpt3_output_0'] = g['prediction']
        tmp['bard_output_0'] = b['prediction']
        all_results.append(tmp)

    all_results = clean_output(all_results, 0, dataset=args.dataset)
    all_results = parse_output(all_results, 0)
    print(f"Initial Round Performance: {evaluate_all(all_results, 0)}")

    # Phase2: Multi-Round Discussion

    for r in range(1, args.round+1):
        print(f"----- Round {r} Discussion -----")
        all_results = claude.claude_debate(test_samples,
                                        all_results,
                                        rounds=r,
                                        convincing_samples=convincing_gpt+convincing_bard,
                                        dataset=args.dataset)

        all_results = gpt_debate(test_samples,
                            all_results,
                            rounds=r,
                            convincing_samples=convincing_claude+convincing_bard,
                            dataset=args.dataset)

        all_results = bard_debate(test_samples,
                            all_results,
                            rounds=r,
                            convincing_samples=convincing_claude+convincing_gpt,
                            dataset=args.dataset)

        all_results = clean_output(all_results, r, dataset=args.dataset)
        all_results = parse_output(all_results, r)
        print(f"Round {r} Performance: {evaluate_all(all_results, r)}")

    with open(f'{args.dataset}_round_{args.round}.pkl', 'wb') as f:
        pickle.dump(all_results, f)
run.py
dinobby/ReConcile
import os
import re
import ast
import json
import time

import random
import numpy as np
import pandas as pd
from tqdm import tqdm
from collections import Counter

random.seed(1234)
datasets = ["SQA", "GSM8k", "ECQA", "Aqua"]

def prepare_context(test_sample, convincing_samples=None, intervene=False, dataset="SQA"):
    assert dataset in datasets
    context = []
    if convincing_samples:
        for cs in convincing_samples:
            context.append(f"Q: {cs['train_sample']['question']}\nA:" + str({"reasoning": cs['train_sample']['gold_explanation'], "answer": cs['train_sample']['answer']}))
     
    if intervene:
        context.append("Q: " + test_sample['question'] + "\nAnswer the question given the fact that " + test_sample['gold_explanation'])
    else:
        context.append("Q: " + test_sample['question'])

    if dataset in ["ECQA", "Aqua", "ScienceQA"]:
        context.append("The options are: {}. Please select an option as your answer.".format(" ".join(test_sample["options"])))
        
    context.append("Please answer the question with step-by-step reasoning.")
    context.append("\nAlso, evaluate your confidence level (between 0.0 and 1.0) to indicate the possibility of your answer being right.")
    context.append("Output your answer in json format, with the format as follows: {\"reasoning\": \"\", \"answer\": \"\", \"confidence_level\": \"\"}. Please strictly output in JSON format.")
    if dataset == "SQA":
        context.append("Only answer yes or no in the \"answer\" field.")
    elif dataset == "GSM8k":
        context.append("Only place a single numeric value in the \"answer\" field.")
    elif dataset == "ECQA":
        context.append("Only place 1,2,3,4,5 representing your choice in the \"answer\" field.")
    elif dataset == "Aqua":
        context.append("Only place A,B,C,D,E representing your choice in the \"answer\" field.")  
    context.append("Do not output irrelevant content.")
    return "\n".join(context)
    
def prepare_context_for_chat_assistant(sample, convincing_samples=None, intervene=False, dataset="SQA"):
    assert dataset in datasets
    contexts = []
    if convincing_samples:
        for cs in convincing_samples:
            contexts.append({"role": "user", "content": f"Q: {cs['train_sample']['question']}"})
            contexts.append({"role": "assistant", "content": str({"reasoning": cs['train_sample']['gold_explanation'], "answer": cs['train_sample']['answer']})})

    if intervene:
        contexts.append({"role": "user", "content": f"Q: {sample['question']}" + "\nAnswer the question given the fact that " + sample['gold_explanation']})  
    else:
        contexts.append({"role": "user", "content": f"Q: {sample['question']}"})
        
    if dataset in ["ECQA", "Aqua"]:
        contexts[-1]["content"] += "The options are: {}. Please select an option as your answer.".format(" ".join(sample["options"]))
        
    contexts[-1]["content"] += " Please answer the question with step-by-step reasoning."
    contexts[-1]["content"] += " Also, evaluate your confidence level (between 0.0 and 1.0) to indicate the possibility of your answer being right."
    contexts[-1]["content"] += " Output your answer in json format, with the format as follows: {\"reasoning\": \"\", \"answer\": \"\", \"confidence_level\": \"\"}. Please strictly output in JSON format."
    
    if dataset == "SQA":
        contexts[-1]["content"] += " Only answer yes or no in the \"answer\" field."
    elif dataset == "GSM8k":
        contexts[-1]["content"] += " Only place a single numeric value in the \"answer\" field."
    elif dataset == "ECQA":
        contexts[-1]["content"] += " Only place 1,2,3,4,5 representing your choice in the \"answer\" field."
    elif dataset == "Aqua":
        contexts[-1]["content"] += " Only place A,B,C,D,E representing your choice in the \"answer\" field."    
    contexts[-1]["content"] += " Do not output irrelevant content."
    return contexts

def prepare_context_for_bard(test_sample, convincing_samples=None, intervene=False, dataset="SQA"):
    assert dataset in datasets
    context, convincing_icx, unhelpful_icx = [], [], []
    if convincing_samples:
        for cs in convincing_samples:
            convincing_icx.append((f"Q: {cs['train_sample']['question']}", str({"reasoning": cs['train_sample']['gold_explanation'], "answer": cs['train_sample']['answer']})))
    
    if intervene:
        context.append("Q: " + test_sample['question'] + "\nAnswer the question given the fact that " + test_sample['gold_explanation'])
    else:
        context.append("Q: " + test_sample['question'])
        
    if dataset in ["ECQA", "Aqua"]:
        context.append("The options are: {}. Please select an option as your answer.".format(" ".join(test_sample["options"])))  
        
    context.append("Please answer the question with step-by-step reasoning.")
    context.append("Also, evaluate your confidence level (between 0.0 and 1.0) to indicate the possibility of your answer being right.")
    context.append("Output your answer in json format, with the format as follows: {\"reasoning\": \"\", \"answer\": \"\", \"confidence_level\": \"\"}. Please strictly output in JSON format.")
    
    if dataset == "SQA":
        context.append("Only answer yes or no in the \"answer\" field. Do not output irrelevant content.")
    elif dataset == "GSM8k":
        context.append("Only place a single numeric value in the \"answer\" field. Do not output irrelevant content.")
    elif dataset == "ECQA":
        context.append("Only place 1,2,3,4,5 representing your choice in the \"answer\" field.")
    elif dataset == "Aqua":
        context.append("Only place A,B,C,D,E representing your choice in the \"answer\" field.")
    context.append("Do not output irrelevant content.")
    return "\n".join(context), convincing_icx, unhelpful_icx

def invalid_result(dataset):
    if dataset == "SQA":
        result = {"reasoning": "",
          "answer": np.random.choice(['yes', 'no']),
          "confidence_level": 0.0}
    elif dataset=="GSM8k":
        result = {"reasoning": "",
          "answer": "0",
          "confidence_level": 0.0}
    elif dataset=="ECQA":
        result = {"reasoning": "",
          "answer": np.random.choice(['1', '2', '3', '4', '5']),
          "confidence_level": 0.0}   
    elif dataset=="Aqua":
        result = {"reasoning": "",
          "answer": np.random.choice(['A', 'B', 'C', 'D', 'E']),
          "confidence_level": 0.0}
    return result

def parse_json(model_output):
    if type(model_output) is dict:
        return model_output
    elif type(model_output) is not str:
        model_output = str(model_output)
    try:
        model_output = model_output.replace("\n", " ")
        model_output = re.search('({.+})', model_output).group(0)
        model_output = re.sub(r"(\w)'(\w|\s)", r"\1\\'\2", model_output)
        result = ast.literal_eval(model_output)
    except (SyntaxError, NameError, AttributeError):
        return "ERR_SYNTAX"
    return result

def find_idx_by_element(input_list, element):
    return [i for i, a in enumerate(input_list) if a == element]

def find_element_by_indices(input_list, index_list):
    return [b for i, b in enumerate(input_list) for k in index_list if i == k]

def trans_confidence(x):
    x = float(x)
    if x <= 0.6: return 0.1
    if 0.8 > x > 0.6: return 0.3
    if 0.9 > x >= 0.8: return 0.5
    if 1 > x >= 0.9: return 0.8
    if x == 1: return 1

def parse_output(all_results, rounds):
    c, g, b = "claude", "gpt3", "bard"
    r = "_output_"+str(rounds)
    
    for i in all_results:
        certainty_vote = {}
            
        for o in [c, g, b]:
            if o+r in i:
                i[o+"_pred_"+str(rounds)] = i[o+r]['answer']
                i[o+"_exp_"+str(rounds)] = f"I think the answer is {i[o+r]['answer']} because {i[o+r]['reasoning']} My confidence level is {i[o+r]['confidence_level']}." 
                if i[o+r]['answer'] not in certainty_vote:
                    certainty_vote[i[o+r]['answer']] = trans_confidence(i[o+r]['confidence_level']) + 1e-5
                else:
                    certainty_vote[i[o+r]['answer']] += trans_confidence(i[o+r]['confidence_level'])
        if c+r in i and g+r in i and b+r in i:
            i['vote_'+str(rounds)] = [i['claude_pred_'+str(rounds)], i['gpt3_pred_'+str(rounds)], i['bard_pred_'+str(rounds)]]
            i['exps_'+str(rounds)] = [i['claude_exp_'+str(rounds)], i['gpt3_exp_'+str(rounds)], i['bard_exp_'+str(rounds)]]
            i['weighted_vote_'+str(rounds)] = certainty_vote
            i['weighted_max_'+str(rounds)] = max(certainty_vote, key=certainty_vote.get)

            i['debate_prompt_'+str(rounds)] = ''
            vote = Counter(i['vote_'+str(rounds)]).most_common(2)
            i['majority_ans_'+str(rounds)] = vote[0][0]
            if len(vote) > 1: # not all the agents give the same answer
                for v in vote:
                    i['debate_prompt_'+str(rounds)] += f"There are {v[1]} agents think the answer is {v[0]}. "
                    exp_index = find_idx_by_element(i['vote_'+str(rounds)], v[0])
                    group_exp = find_element_by_indices(i['exps_'+str(rounds)], exp_index)
                    exp = "\n".join(["One agent solution: " + g for g in group_exp])
                    i['debate_prompt_'+str(rounds)] += exp + "\n\n"
                    
    return all_results

def evaluate_single_model(results):
    num_correct = 0
    for i in results:
        if i['gold_answer'] == i['prediction']['answer']:
            num_correct+=1
    return num_correct / len(results)

def clean_output(all_results, rounds, dataset):
    co, go, bo = "claude_output_" + str(rounds), 'gpt3_output_' + str(rounds), 'bard_output_' + str(rounds)
    for i in all_results:
        for o in [co, go, bo]:
            if o in i:
                    
                if 'reasoning' not in i[o]:
                    i[o]['reasoning'] = ""
                elif type(i[o]['reasoning']) is list:
                    i[o]['reasoning'] = " ".join(i[o]['reasoning'])

                if dataset=="SQA":
                    if 'answer' not in i[o] or i[o]['answer'] not in ['yes', 'no']:
                        i[o]['answer'] = np.random.choice(['yes', 'no'])
                elif dataset=="GSM8k":
                    if 'answer' not in i[o]:
                        i[o]['answer'] = "0"
                elif dataset=="ECQA":
                    if 'answer' not in i[o]:
                        i[o]['answer'] = np.random.choice(['1', '2', '3', '4', '5'])
                elif dataset=="Aqua":
                    if 'answer' not in i[o]:
                        i[o]['answer'] = np.random.choice(['A', 'B', 'C', 'D', 'E'])
                        
                if 'confidence_level' not in i[o] or not i[o]['confidence_level']:
                    i[o]['confidence_level'] = 0.0
                else:
                    if type(i[o]['confidence_level']) is str and "%" in i[o]['confidence_level']:
                            i[o]['confidence_level'] = float(i[o]['confidence_level'].replace("%","")) / 100
                    else:
                        try:
                            i[o]['confidence_level'] = float(i[o]['confidence_level'])
                        except:
                            print(i[o]['confidence_level'])
                            i[o]['confidence_level'] = 0.0
                
    return all_results

def evaluate_results(all_results, prefix, rounds):
    num_correct = 0
    for i in all_results:
        r_num = int(rounds)
        while True:
            eval_key = prefix+"_"+str(r_num)
            if eval_key in i:
                if i['gold_answer'] == i[eval_key]:
                    num_correct += 1
                break
            else:
                r_num -= 1
            if r_num<0: break
    return num_correct/len(all_results)

def evaluate_all(all_results, rounds):
    accuracies = {}
    for prefix in ["claude_pred", "gpt3_pred", "bard_pred", "majority_ans", "weighted_max"]:
        accuracies[prefix] = evaluate_results(all_results, prefix, rounds)
    return accuracies


utils.py
dinobby/ReConcile