## llama-3.1-8b-instruct/arithmetic/f_colloq
explicit_acc = 0.8896
parse_fail_rate = 0.0033
n = 300
## llama-3.1-8b-instruct/arithmetic/f_prose
explicit_acc = 0.8833
parse_fail_rate = 0.0000
n = 300
## llama-3.1-8b-instruct/arithmetic/f_symbolic
explicit_acc = 0.8533
parse_fail_rate = 0.0000
n = 300
## llama-3.1-8b-instruct/arithmetic/f_verbal
explicit_acc = 0.8716
parse_fail_rate = 0.0133
n = 300
## llama-3.1-8b-instruct/capital/f_declare
explicit_acc = 0.9933
parse_fail_rate = 0.0000
n = 300
## llama-3.1-8b-instruct/capital/f_equation
explicit_acc = 0.9900
parse_fail_rate = 0.0000
n = 300
## llama-3.1-8b-instruct/capital/f_invert
explicit_acc = 0.9833
parse_fail_rate = 0.0000
n = 300
## llama-3.1-8b-instruct/capital/f_prose
explicit_acc = 0.9800
parse_fail_rate = 0.0000
n = 300
## llama-3.1-8b-instruct/arithmetic
explicit_acc = 0.8745
n_judged = 1195
## llama-3.1-8b-instruct/capital
explicit_acc = 0.9867
n_judged = 1200
## qwen2.5-7b-instruct/arithmetic/f_colloq
explicit_acc = 0.8867
parse_fail_rate = 0.0000
n = 300
## qwen2.5-7b-instruct/arithmetic/f_prose
explicit_acc = 0.9767
parse_fail_rate = 0.0000
n = 300
## qwen2.5-7b-instruct/arithmetic/f_symbolic
explicit_acc = 0.7933
parse_fail_rate = 0.0000
n = 300
## qwen2.5-7b-instruct/arithmetic/f_verbal
explicit_acc = 0.8767
parse_fail_rate = 0.0000
n = 300
## qwen2.5-7b-instruct/capital/f_declare
explicit_acc = 0.9367
parse_fail_rate = 0.0000
n = 300
## qwen2.5-7b-instruct/capital/f_equation
explicit_acc = 0.9500
parse_fail_rate = 0.0000
n = 300
## qwen2.5-7b-instruct/capital/f_invert
explicit_acc = 0.9333
parse_fail_rate = 0.0000
n = 300
## qwen2.5-7b-instruct/capital/f_prose
explicit_acc = 0.9600
parse_fail_rate = 0.0000
n = 300
## qwen2.5-7b-instruct/arithmetic
explicit_acc = 0.8833
n_judged = 1200
## qwen2.5-7b-instruct/capital
explicit_acc = 0.9450
n_judged = 1200
## mistral-7b-instruct-v0.3/arithmetic/f_colloq
explicit_acc = 0.5000
parse_fail_rate = 0.0000
n = 300
## mistral-7b-instruct-v0.3/arithmetic/f_prose
explicit_acc = 0.5200
parse_fail_rate = 0.0000
n = 300
## mistral-7b-instruct-v0.3/arithmetic/f_symbolic
explicit_acc = 0.5167
parse_fail_rate = 0.0000
n = 300
## mistral-7b-instruct-v0.3/arithmetic/f_verbal
explicit_acc = 0.5167
parse_fail_rate = 0.0000
n = 300
## mistral-7b-instruct-v0.3/capital/f_declare
explicit_acc = 0.9167
parse_fail_rate = 0.0000
n = 300
## mistral-7b-instruct-v0.3/capital/f_equation
explicit_acc = 0.9333
parse_fail_rate = 0.0000
n = 300
## mistral-7b-instruct-v0.3/capital/f_invert
explicit_acc = 0.8733
parse_fail_rate = 0.0000
n = 300
## mistral-7b-instruct-v0.3/capital/f_prose
explicit_acc = 0.9800
parse_fail_rate = 0.0000
n = 300
## mistral-7b-instruct-v0.3/arithmetic
explicit_acc = 0.5133
n_judged = 1200
## mistral-7b-instruct-v0.3/capital
explicit_acc = 0.9258
n_judged = 1200
## stage2/llama-3.1-8b-instruct/arithmetic/f_colloq
verbalized_auc = 0.8893
commit_auc = 0.5874
probe_infmt_auc = 0.9847
probe_lofo_auc = 0.9998
chosen_layer = 32
chosen_layer_lofo = 32
flagged = 0
## stage2/llama-3.1-8b-instruct/arithmetic/f_prose
verbalized_auc = 0.8899
commit_auc = 0.6834
probe_infmt_auc = 0.9900
probe_lofo_auc = 1.0000
chosen_layer = 32
chosen_layer_lofo = 32
flagged = 0
## stage2/llama-3.1-8b-instruct/arithmetic/f_symbolic
verbalized_auc = 0.8557
commit_auc = 0.5194
probe_infmt_auc = 0.9867
probe_lofo_auc = 0.9981
chosen_layer = 32
chosen_layer_lofo = 32
flagged = 0
## stage2/llama-3.1-8b-instruct/arithmetic/f_verbal
verbalized_auc = 0.8974
commit_auc = 0.7885
probe_infmt_auc = 0.8943
probe_lofo_auc = 0.8266
chosen_layer = 32
chosen_layer_lofo = 28
flagged = 0
## stage2/llama-3.1-8b-instruct/capital/f_declare
verbalized_auc = 0.9960
commit_auc = 0.9065
probe_infmt_auc = 0.9994
probe_lofo_auc = 1.0000
chosen_layer = 16
chosen_layer_lofo = 16
flagged = 1
## stage2/llama-3.1-8b-instruct/capital/f_equation
verbalized_auc = 0.9962
commit_auc = 0.9724
probe_infmt_auc = 0.9996
probe_lofo_auc = 1.0000
chosen_layer = 16
chosen_layer_lofo = 12
flagged = 1
## stage2/llama-3.1-8b-instruct/capital/f_invert
verbalized_auc = 0.9898
commit_auc = 0.8064
probe_infmt_auc = 0.9934
probe_lofo_auc = 0.9922
chosen_layer = 16
chosen_layer_lofo = 16
flagged = 1
## stage2/llama-3.1-8b-instruct/capital/f_prose
verbalized_auc = 0.9890
commit_auc = 0.9068
probe_infmt_auc = 0.9987
probe_lofo_auc = 0.9996
chosen_layer = 16
chosen_layer_lofo = 12
flagged = 1
## stage2/qwen2.5-7b-instruct/arithmetic/f_colloq
verbalized_auc = 0.8867
commit_auc = 0.8827
probe_infmt_auc = 0.9956
probe_lofo_auc = 0.9927
chosen_layer = 28
chosen_layer_lofo = 28
flagged = 0
## stage2/qwen2.5-7b-instruct/arithmetic/f_prose
verbalized_auc = 0.9767
commit_auc = 0.8678
probe_infmt_auc = 0.9995
probe_lofo_auc = 0.9649
chosen_layer = 28
chosen_layer_lofo = 18
flagged = 1
## stage2/qwen2.5-7b-instruct/arithmetic/f_symbolic
verbalized_auc = 0.7933
commit_auc = 0.7452
probe_infmt_auc = 0.9929
probe_lofo_auc = 0.9955
chosen_layer = 28
chosen_layer_lofo = 28
flagged = 0
## stage2/qwen2.5-7b-instruct/arithmetic/f_verbal
verbalized_auc = 0.8767
commit_auc = 0.7232
probe_infmt_auc = 0.9048
probe_lofo_auc = 0.7291
chosen_layer = 28
chosen_layer_lofo = 7
flagged = 0
## stage2/qwen2.5-7b-instruct/capital/f_declare
verbalized_auc = 0.9494
commit_auc = 0.4651
probe_infmt_auc = 0.9976
probe_lofo_auc = 1.0000
chosen_layer = 21
chosen_layer_lofo = 18
flagged = 0
## stage2/qwen2.5-7b-instruct/capital/f_equation
verbalized_auc = 0.9664
commit_auc = 0.7676
probe_infmt_auc = 0.9972
probe_lofo_auc = 1.0000
chosen_layer = 21
chosen_layer_lofo = 21
flagged = 0
## stage2/qwen2.5-7b-instruct/capital/f_invert
verbalized_auc = 0.9665
commit_auc = 0.1887
probe_infmt_auc = 0.9920
probe_lofo_auc = 0.9980
chosen_layer = 21
chosen_layer_lofo = 21
flagged = 0
## stage2/qwen2.5-7b-instruct/capital/f_prose
verbalized_auc = 0.9760
commit_auc = 0.4611
probe_infmt_auc = 0.9970
probe_lofo_auc = 0.9995
chosen_layer = 21
chosen_layer_lofo = 14
flagged = 0
## stage2/mistral-7b-instruct-v0.3/arithmetic/f_colloq
verbalized_auc = 0.5000
commit_auc = 0.4593
probe_infmt_auc = 0.9812
probe_lofo_auc = 0.9981
chosen_layer = 32
chosen_layer_lofo = 32
flagged = 0
## stage2/mistral-7b-instruct-v0.3/arithmetic/f_prose
verbalized_auc = 0.5200
commit_auc = 0.5538
probe_infmt_auc = 0.9840
probe_lofo_auc = 0.9989
chosen_layer = 32
chosen_layer_lofo = 32
flagged = 0
## stage2/mistral-7b-instruct-v0.3/arithmetic/f_symbolic
verbalized_auc = 0.5167
commit_auc = 0.5671
probe_infmt_auc = 0.9844
probe_lofo_auc = 0.9959
chosen_layer = 32
chosen_layer_lofo = 32
flagged = 0
## stage2/mistral-7b-instruct-v0.3/arithmetic/f_verbal
verbalized_auc = 0.5167
commit_auc = 0.5032
probe_infmt_auc = 0.8548
probe_lofo_auc = 0.8676
chosen_layer = 32
chosen_layer_lofo = 32
flagged = 0
## stage2/mistral-7b-instruct-v0.3/capital/f_declare
verbalized_auc = 0.9167
commit_auc = 0.7896
probe_infmt_auc = 0.9961
probe_lofo_auc = 1.0000
chosen_layer = 12
chosen_layer_lofo = 12
flagged = 0
## stage2/mistral-7b-instruct-v0.3/capital/f_equation
verbalized_auc = 0.9333
commit_auc = 0.4404
probe_infmt_auc = 0.9964
probe_lofo_auc = 1.0000
chosen_layer = 12
chosen_layer_lofo = 12
flagged = 0
## stage2/mistral-7b-instruct-v0.3/capital/f_invert
verbalized_auc = 0.8733
commit_auc = 0.8946
probe_infmt_auc = 0.9952
probe_lofo_auc = 0.9985
chosen_layer = 12
chosen_layer_lofo = 12
flagged = 0
## stage2/mistral-7b-instruct-v0.3/capital/f_prose
verbalized_auc = 0.9800
commit_auc = 0.6326
probe_infmt_auc = 0.9959
probe_lofo_auc = 0.9997
chosen_layer = 12
chosen_layer_lofo = 12
flagged = 1
## stage2/commit/orientation
commit_sign = 1.0000
## stage2/correlation/all_cells
pearson_r = 0.1227
pearson_ci_lo = -0.1347
pearson_ci_hi = 0.5874
spearman_r = 0.3875
spearman_p = 0.061346
## stage2/correlation/nonflagged
pearson_r = 0.0504
pearson_ci_lo = -0.2267
pearson_ci_hi = 0.6266
spearman_r = 0.4875
spearman_p = 0.040156
## stage2/gap/model/llama-3.1-8b-instruct
mean_probe_lofo = 0.9561
mean_verbalized = 0.8831
mean_commit = 0.6447
mean_gap = 0.0730
## stage2/gap/model/mistral-7b-instruct-v0.3
mean_probe_lofo = 0.9799
mean_verbalized = 0.6824
mean_commit = 0.6011
mean_gap = 0.2975
## stage2/gap/model/qwen2.5-7b-instruct
mean_probe_lofo = 0.9593
mean_verbalized = 0.9164
mean_commit = 0.6048
mean_gap = 0.0428
## stage2/gap/domain/arithmetic
mean_probe_lofo = 0.9457
mean_verbalized = 0.7402
mean_commit = 0.6376
mean_gap = 0.2054
## stage2/gap/domain/capital
mean_probe_lofo = 0.9994
mean_verbalized = 0.9402
mean_commit = 0.5724
mean_gap = 0.0592
## stage2/itemlevel/arithmetic_pooled
both_correct_rate = 0.6697
knows_but_says_wrong_rate = 0.2292
kbsw_ci_lo = 0.2150
kbsw_ci_hi = 0.2442
verb_only_rate = 0.0864
vo_ci_lo = 0.0778
vo_ci_hi = 0.0961
both_wrong_rate = 0.0147
phi = -0.0768
## stage2/itemlevel/capital_pooled
both_correct_rate = 0.9447
knows_but_says_wrong_rate = 0.0456
kbsw_ci_lo = 0.0389
kbsw_ci_hi = 0.0528
verb_only_rate = 0.0078
vo_ci_lo = 0.0050
vo_ci_hi = 0.0106
both_wrong_rate = 0.0019
phi = 0.0710
## stage2/itemlevel/llama-3.1-8b-instruct/arithmetic
both_correct_rate = 0.7917
knows_but_says_wrong_rate = 0.1158
kbsw_ci_lo = 0.0983
kbsw_ci_hi = 0.1333
verb_only_rate = 0.0800
vo_ci_lo = 0.0650
vo_ci_hi = 0.0950
both_wrong_rate = 0.0125
phi = 0.0065
## stage2/itemlevel/llama-3.1-8b-instruct/capital
both_correct_rate = 0.9767
knows_but_says_wrong_rate = 0.0117
kbsw_ci_lo = 0.0058
kbsw_ci_hi = 0.0183
verb_only_rate = 0.0100
vo_ci_lo = 0.0050
vo_ci_hi = 0.0158
both_wrong_rate = 0.0017
phi = 0.1227
## stage2/itemlevel/mistral-7b-instruct-v0.3/arithmetic
both_correct_rate = 0.4625
knows_but_says_wrong_rate = 0.4667
kbsw_ci_lo = 0.4400
kbsw_ci_hi = 0.4942
verb_only_rate = 0.0508
vo_ci_lo = 0.0383
vo_ci_hi = 0.0634
both_wrong_rate = 0.0200
phi = -0.1129
## stage2/itemlevel/mistral-7b-instruct-v0.3/capital
both_correct_rate = 0.9192
knows_but_says_wrong_rate = 0.0725
kbsw_ci_lo = 0.0583
kbsw_ci_hi = 0.0867
verb_only_rate = 0.0067
vo_ci_lo = 0.0025
vo_ci_hi = 0.0117
both_wrong_rate = 0.0017
phi = 0.0440
## stage2/itemlevel/qwen2.5-7b-instruct/arithmetic
both_correct_rate = 0.7550
knows_but_says_wrong_rate = 0.1050
kbsw_ci_lo = 0.0883
kbsw_ci_hi = 0.1217
verb_only_rate = 0.1283
vo_ci_lo = 0.1100
vo_ci_hi = 0.1467
both_wrong_rate = 0.0117
phi = -0.0419
## stage2/itemlevel/qwen2.5-7b-instruct/capital
both_correct_rate = 0.9383
knows_but_says_wrong_rate = 0.0525
kbsw_ci_lo = 0.0400
kbsw_ci_hi = 0.0650
verb_only_rate = 0.0067
vo_ci_lo = 0.0025
vo_ci_hi = 0.0117
both_wrong_rate = 0.0025
phi = 0.0919
## stage4/cot_diagnostic/mistral-7b-instruct-v0.3
kbsw_strict_n = 518
strict_cot_correct = 178
strict_cot_wrong = 94
strict_cot_none = 246
strict_correct_share_shown = 0.6544
strict_correct_share_full = 0.3436
strict_noworking_share_full = 0.4749
loose_correct_share_shown = 0.6477
## stage4/cot_diagnostic/llama-3.1-8b-instruct
kbsw_strict_n = 136
strict_cot_correct = 16
strict_cot_wrong = 119
strict_cot_none = 1
strict_correct_share_shown = 0.1185
strict_correct_share_full = 0.1176
strict_noworking_share_full = 0.0074
loose_correct_share_shown = 0.1241
## stage4/cot_diagnostic/qwen2.5-7b-instruct
kbsw_strict_n = 119
strict_cot_correct = 61
strict_cot_wrong = 31
strict_cot_none = 27
strict_correct_share_shown = 0.6630
strict_correct_share_full = 0.5126
strict_noworking_share_full = 0.2269
loose_correct_share_shown = 0.6990
## stage4/cot_diagnostic/pooled
kbsw_strict_n = 773
strict_cot_correct = 255
strict_cot_wrong = 244
strict_cot_none = 274
strict_correct_share_shown = 0.5110
strict_correct_share_full = 0.3299
strict_noworking_share_full = 0.3545
loose_correct_share_shown = 0.5142
## stage4/cot_artifact/mistral-7b-instruct-v0.3/correct
n = 215
probe_acc = 0.9814
verb_acc = 0.1535
## stage4/cot_artifact/mistral-7b-instruct-v0.3/wrong
n = 101
probe_acc = 0.9505
verb_acc = 0.0198
## stage4/cot_artifact/llama-3.1-8b-instruct/correct
n = 384
probe_acc = 0.9453
verb_acc = 0.9531
## stage4/cot_artifact/llama-3.1-8b-instruct/wrong
n = 175
probe_acc = 0.9429
verb_acc = 0.2743
## stage4/cot_artifact/qwen2.5-7b-instruct/correct
n = 431
probe_acc = 0.9281
verb_acc = 0.8329
## stage4/cot_artifact/qwen2.5-7b-instruct/wrong
n = 31
probe_acc = 1.0000
verb_acc = 0.0000
## stage4/ensemble/pooled
verbalized_auc = 0.8598
probe_auc = 0.9849
ensemble_auc = 0.9889
probe_kbsw_recovery = 1.0000
ensemble_kbsw_recovery = 0.9695
ensemble_minus_best_single_auc = 0.0040
## stage4/ensemble/arithmetic
verbalized_auc = 0.7595
probe_auc = 0.9567
ensemble_auc = 0.9690
probe_kbsw_recovery = 1.0000
ensemble_kbsw_recovery = 0.9677
ensemble_minus_best_single_auc = 0.0123
## stage4/ensemble/capital
verbalized_auc = 0.9612
probe_auc = 0.9989
ensemble_auc = 0.9984
probe_kbsw_recovery = 1.0000
ensemble_kbsw_recovery = 0.9863
ensemble_minus_best_single_auc = -0.0005
## stage4/range_restriction/all_cells
sd_probe_lofo = 0.0676
sd_verbalized = 0.1669
pearson_r = 0.1227
spearman_r = 0.3875
spearman_p = 0.061346
logit_pearson_r = 0.2691
## stage4/range_restriction/nonflagged
sd_probe_lofo = 0.0771
sd_verbalized = 0.1733
pearson_r = 0.0504
spearman_r = 0.4875
spearman_p = 0.040156
logit_pearson_r = 0.2699
## stage4/range_restriction/probe_off_ceiling
sd_probe_lofo = 0.0977
sd_verbalized = 0.2047
pearson_r = 0.0661
spearman_r = 0.4000
spearman_p = 0.600000
logit_pearson_r = 0.5543
## stage5/suppression/llama-3.1-8b-instruct
p_false_given_true = 0.2400
p_false_given_false = 0.9883
suppression_gap = 0.7483
## stage5/suppression/qwen2.5-7b-instruct
p_false_given_true = 0.2183
p_false_given_false = 0.9850
suppression_gap = 0.7667
## stage5/suppression/mistral-7b-instruct-v0.3
p_false_given_true = 0.8983
p_false_given_false = 0.9250
suppression_gap = 0.0267
## stage5/cot_artifact_cont/mistral-7b-instruct-v0.3
probe_sep_auc = 0.6325
probe_mean_diff = 0.0416
probe_cohens_d = 0.2782
## stage5/cot_artifact_cont/llama-3.1-8b-instruct
probe_sep_auc = 0.5430
probe_mean_diff = 0.0000
probe_cohens_d = 0.0002
## stage5/cot_artifact_cont/qwen2.5-7b-instruct
probe_sep_auc = 0.4447
probe_mean_diff = -0.0600
probe_cohens_d = -0.2953