text: from simpletransformers.classification import ClassificationModel, ClassificationArgs
  import pandas as pd
  import logging
  import json
  import numpy as np
  import sklearn
  import fire
  
  from scipy.optimize import brentq
  from scipy.interpolate import interp1d
  from sklearn.metrics import roc_curve
  
  import data_loading
  import json
  import traceback
  
  logging.basicConfig(level=logging.INFO)
  transformers_logger = logging.getLogger("transformers")
  transformers_logger.setLevel(logging.WARNING)
  
  def eer(y, y_score):
      fpr, tpr, thresholds = roc_curve(y, y_score, pos_label=1)
  
      eer = brentq(lambda x : 1. - x - interp1d(fpr, tpr)(x), 0., 1.)
      thresh = interp1d(fpr, thresholds)(eer)
      print(f'Threshold: {thresh}')
      return eer
  
  source = (
      # "covid_real;"
      # "covid_gen;"
      # "eval1_real;"
      # "eval1_gen;"
      # "sbirs_real;"
      # "sbirs_gen;"
      # "webtext_real;"
      # "webtext_gen;"
      # "covid_real_016;covid_gen_016;eval1_real_016;eval1_gen_016;sbirs_real_016;sbirs_gen_016;webtext_real_016;webtext_gen_016"
      # "covid_real_256;covid_gen_256;eval1_real_256;eval1_gen_256;sbirs_real_256;sbirs_gen_256;webtext_real_256;webtext_gen_256"
  
      # "gptneo_collect_eval2"
      # "generator=mega~dataset=p0.90;"
      # "generator=mega~dataset=p0.92;"
      # "generator=mega~dataset=p0.94;"
      # "generator=mega~dataset=p0.96;"
      # "generator=mega~dataset=p0.98;"
      # "generator=mega~dataset=p1.00;"
      # "xl-1542M;"
      # "xl-1542M-k40;"
      "xl-1542M-nucleus;"
  )
  
  def main (
          source=source,
          data_dir='data',
          load_model_dir="outputs/eval2/best_model_openai_finetune_1",
          checkpoint_dir="outputs/eval2/test_xl-1542M-nucleus_eval2_analytic",
          n_train=250000,
          n_valid=10000,
          n_test=np.inf,
          reprocess_input=True,
      ):
  
      transformers_logger.info(f'source: {source}, checkpoint_dir: {checkpoint_dir}')
  
      test_df = data_loading.load_split(data_dir, source, 'test', n=n_test)
  
      # Optional model configuration
      model_args = ClassificationArgs(
          output_dir=checkpoint_dir,
          num_train_epochs=2,
          evaluate_during_training=True,
          save_steps=25000,
          evaluate_during_training_steps=25000,
          manual_seed=0,
          train_batch_size=256,
          eval_batch_size=256,
          overwrite_output_dir=True,
          reprocess_input_data=reprocess_input,
          n_gpu=2,
          no_cache=True,
      )
  
      # Create a ClassificationModel
      model = ClassificationModel(
          "roberta",
          # model_name="roberta-large-openai-detector",
          load_model_dir,
          args=model_args,
          use_cuda=True
      )
  
      # Evaluate the model
      result, model_outputs, wrong_predictions = model.eval_model(
          test_df,
          f1=sklearn.metrics.f1_score,
          acc=sklearn.metrics.accuracy_score,
          eer=eer
      )
  
  if __name__ == '__main__':
      fire.Fire(main)
  
  """
  All hackathon long domains.
  """