File size: 621 Bytes
d2a7b67 917144d d2a7b67 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 |
{
"architectures": [
"GPTNeoXCodebookModel"
],
"codebook_at": [
"attn_preproj",
"mlp"
],
"codebook_kwargs": {},
"codebook_type": [
"group",
"vanilla"
],
"k_codebook": [
1,
1
],
"kmeans_init": false,
"kmeans_init_examples": 1000,
"kmeans_kwargs": null,
"kmeans_path": null,
"layers_to_snap": [
0,
1,
2,
3
],
"loss": "aeloss",
"model_type": "codebook",
"num_codebooks": [
4,
1
],
"num_codes": 10000,
"replace_codes": false,
"similarity_metric": "inner_product",
"torch_dtype": "float32",
"transformers_version": "4.28.1"
}
|