run_id_prefix: full_arc_challenge_w2s_rationales
n_rows: 65
loss_curves: not recorded
parse_fail_rate.labels_only: 0.000455
raw_single_letter_rate.labels_only: 1.000000
pred_equals_raw_rate.labels_only: 0.999545
parse_fail_rate.labels_rationales: 0.000000
raw_single_letter_rate.labels_rationales: 1.000000
pred_equals_raw_rate.labels_rationales: 1.000000
parse_fail_rate.clean_ceiling: 0.000000
raw_single_letter_rate.clean_ceiling: 0.999659
pred_equals_raw_rate.clean_ceiling: 0.999659
parse_fail_rate.zeroshot_floor: 0.000000
raw_single_letter_rate.zeroshot_floor: 0.998294
pred_equals_raw_rate.zeroshot_floor: 0.998294
parse_fail_rate.rationale_only: 0.001991
raw_single_letter_rate.rationale_only: 0.261320
pred_equals_raw_rate.rationale_only: 0.261320
parse_fail_rate.shuffled_rationale: 0.000000
raw_single_letter_rate.shuffled_rationale: 1.000000
pred_equals_raw_rate.shuffled_rationale: 1.000000
clean_acc_mean.labels_only.eps0.15: 0.713993
clean_acc_std.labels_only.eps0.15: 0.091749
n_seeds.labels_only.eps0.15: 5
clean_acc_mean.labels_only.eps0.30: 0.737713
clean_acc_std.labels_only.eps0.30: 0.047128
n_seeds.labels_only.eps0.30: 5
clean_acc_mean.labels_only.eps0.45: 0.480375
clean_acc_std.labels_only.eps0.45: 0.245788
n_seeds.labels_only.eps0.45: 5
clean_acc_mean.labels_rationales.eps0.15: 0.684130
clean_acc_std.labels_rationales.eps0.15: 0.048434
n_seeds.labels_rationales.eps0.15: 5
clean_acc_mean.labels_rationales.eps0.30: 0.614505
clean_acc_std.labels_rationales.eps0.30: 0.056059
n_seeds.labels_rationales.eps0.30: 5
clean_acc_mean.labels_rationales.eps0.45: 0.611945
clean_acc_std.labels_rationales.eps0.45: 0.038308
n_seeds.labels_rationales.eps0.45: 5
clean_acc_mean.clean_ceiling.eps0.30: 0.737457
clean_acc_std.clean_ceiling.eps0.30: 0.181847
n_seeds.clean_ceiling.eps0.30: 10
clean_acc_mean.zeroshot_floor.eps0.30: 0.801195
clean_acc_std.zeroshot_floor.eps0.30: 0.000000
n_seeds.zeroshot_floor.eps0.30: 5
clean_acc_mean.rationale_only.eps0.15: 0.490102
clean_acc_std.rationale_only.eps0.15: 0.112255
n_seeds.rationale_only.eps0.15: 5
clean_acc_mean.rationale_only.eps0.30: 0.449829
clean_acc_std.rationale_only.eps0.30: 0.100833
n_seeds.rationale_only.eps0.30: 5
clean_acc_mean.rationale_only.eps0.45: 0.438225
clean_acc_std.rationale_only.eps0.45: 0.089948
n_seeds.rationale_only.eps0.45: 5
clean_acc_mean.shuffled_rationale.eps0.30: 0.769113
clean_acc_std.shuffled_rationale.eps0.30: 0.031406
n_seeds.shuffled_rationale.eps0.30: 5
effect_B_minus_A.eps0.15: -0.029863
effect_B_minus_A.eps0.15.ci_lo: -0.042150
effect_B_minus_A.eps0.15.ci_hi: -0.017918
effect_B_minus_A.eps0.15.p: 0.000100
effect_B_minus_A.eps0.15.n_pairs: 5860
effect_B_minus_A.eps0.15.n_seeds: 5
effect_B_minus_A.eps0.30: -0.123208
effect_B_minus_A.eps0.30.ci_lo: -0.135324
effect_B_minus_A.eps0.30.ci_hi: -0.111092
effect_B_minus_A.eps0.30.p: 0.000100
effect_B_minus_A.eps0.30.n_pairs: 5860
effect_B_minus_A.eps0.30.n_seeds: 5
effect_B_minus_A.eps0.45: 0.131570
effect_B_minus_A.eps0.45.ci_lo: 0.115358
effect_B_minus_A.eps0.45.ci_hi: 0.147952
effect_B_minus_A.eps0.45.p: 0.000100
effect_B_minus_A.eps0.45.n_pairs: 5860
effect_B_minus_A.eps0.45.n_seeds: 5
primary_effect_eps0.30: -0.123208
primary_ci_excludes_zero: 0
primary_point_ge_min_effect: 0
primary_confirmed: 0
manip_rationale_only_minus_zeroshot.eps0.30: -0.351365
manip_rationale_only_minus_zeroshot.eps0.30.ci_lo: -0.365358
manip_rationale_only_minus_zeroshot.eps0.30.ci_hi: -0.337372
manip_rationale_only_minus_zeroshot.eps0.30.p: 0.000100
manip_check_passed: 0
degradation_gap.eps0.15: 0.023464
degradation_gap.eps0.30: -0.000256
degradation_gap.eps0.45: 0.257082
degradation_gap_paired.eps0.30: -0.062628
degradation_gap_paired.eps0.30.ci_lo: -0.075939
degradation_gap_paired.eps0.30.ci_hi: -0.049659
degradation_gap_paired.eps0.30.p: 0.000100
real_minus_shuffled_rationale.eps0.30: -0.154608
real_minus_shuffled_rationale.eps0.30.ci_lo: -0.166041
real_minus_shuffled_rationale.eps0.30.ci_hi: -0.143174
real_minus_shuffled_rationale.eps0.30.p: 0.000100
holm_family_non_primary_tests: effect_B_minus_A.eps0.15,effect_B_minus_A.eps0.45,manip_rationale_only_minus_zeroshot.eps0.30,degradation_gap_paired.eps0.30,real_minus_shuffled_rationale.eps0.30
effect_B_minus_A.eps0.15.p_holm: 0.000100
effect_B_minus_A.eps0.45.p_holm: 0.000100
manip_rationale_only_minus_zeroshot.eps0.30.p_holm: 0.000100
degradation_gap_paired.eps0.30.p_holm: 0.000100
real_minus_shuffled_rationale.eps0.30.p_holm: 0.000100