{
  "composition_baseline": {
    "auc": 0.7361638907870743,
    "auc_std": 0.09840516114010511,
    "folds": 4
  },
  "retrieval_baseline": {
    "auc": 0.9305779227932498,
    "n": 1032,
    "note": "cosine similarity to the nearest measured positive outside the enzyme's own 30% cluster"
  },
  "cluster_30": {
    "all-annotated": {
      "auc": 0.9673680105356348,
      "auc_std": 0.01967744225054828,
      "ap": 0.98343275587132,
      "brier": 0.0398776266080585,
      "n_pos": 791,
      "n_neg": 241,
      "n_pos_clusters": 60,
      "folds": 4
    },
    "measured-only": {
      "auc": 0.9214275459526514,
      "auc_std": 0.07158805107619742,
      "ap": 0.9598145815030918,
      "brier": 0.06040969347497282,
      "n_pos": 304,
      "n_neg": 241,
      "n_pos_clusters": 46,
      "folds": 4
    },
    "pu": {
      "auc": 0.7820336641265146,
      "auc_std": 0.15914023583640732,
      "ap": 0.8815251082852031,
      "brier": 0.16956357194657354,
      "c_estimate": 0.6446542590856552,
      "n_pos": 304,
      "n_neg": 241,
      "folds": 4
    },
    "_n_clusters": 115
  },
  "cluster_50": {
    "all-annotated": {
      "auc": 0.9747499623334854,
      "auc_std": 0.011525711567500334,
      "ap": 0.9922684025011426,
      "brier": 0.03601269521506266,
      "n_pos": 791,
      "n_neg": 241,
      "n_pos_clusters": 97,
      "folds": 4
    },
    "measured-only": {
      "auc": 0.9417417647459794,
      "auc_std": 0.04818900245482182,
      "ap": 0.9734910776444796,
      "brier": 0.0635170719352694,
      "n_pos": 304,
      "n_neg": 241,
      "n_pos_clusters": 61,
      "folds": 4
    },
    "pu": {
      "auc": 0.8596647240745687,
      "auc_std": 0.09008551393146179,
      "ap": 0.9020424102206778,
      "brier": 0.12500178698702963,
      "c_estimate": 0.7317991256713867,
      "n_pos": 304,
      "n_neg": 241,
      "folds": 4
    },
    "_n_clusters": 205
  },
  "leave_one_family_out": {
    "n_families": 13,
    "n_mixed_families": 0,
    "families": {
      "Polyesterase-lipase-cutinase": {
        "pos": 77,
        "neg": 0
      },
      "Epoxide_hydrolase": {
        "pos": 0,
        "neg": 3
      },
      "Bacterial_lip_FamI.1": {
        "pos": 0,
        "neg": 2
      },
      "Dienelactone_hydrolase": {
        "pos": 0,
        "neg": 7
      },
      "A85-Mycolyl-transferase": {
        "pos": 0,
        "neg": 44
      },
      "6_AlphaBeta_hydrolase": {
        "pos": 0,
        "neg": 25
      },
      "A85-EsteraseD-FGH": {
        "pos": 0,
        "neg": 8
      },
      "AlphaBeta_hydrolase": {
        "pos": 0,
        "neg": 6
      },
      "OHBut_olig_hydro_put": {
        "pos": 0,
        "neg": 6
      },
      "Proline_iminopeptidase": {
        "pos": 0,
        "neg": 1
      },
      "Haloperoxidase": {
        "pos": 0,
        "neg": 1
      },
      "Hormone-sensitive_lipase_like": {
        "pos": 0,
        "neg": 5
      },
      "Cutinase": {
        "pos": 0,
        "neg": 110
      }
    },
    "evaluable": false,
    "note": "every family is wholly positive or wholly negative, so holding one out removes a class and AUC is undefined: in this catalogue the label is family membership"
  },
  "prospective_holdout": {
    "cutoff": "2020-01-01",
    "n_dated": 822,
    "n_train": 449,
    "n_test": 372,
    "test_pos": 366,
    "test_neg": 6,
    "auc": 0.9080145719489983,
    "note": "UniProt first-public dates, not PDB release dates, which are absent for the whole catalogue; UNDERPOWERED -- the test side holds only 6 of the smaller class, so this AUC is indicative at best",
    "underpowered": true
  },
  "reliability": {
    "mean_predicted": [
      5.9138630181883986e-05,
      0.14518472748490902,
      0.9641826106596363,
      0.998973201411639,
      0.999743575316209,
      0.9999332129955292,
      0.9999766571577205,
      0.9999958369158959
    ],
    "observed_fraction": [
      0.031007751937984496,
      0.24031007751937986,
      0.9224806201550387,
      0.9612403100775194,
      1.0,
      1.0,
      0.9922480620155039,
      0.9844961240310077
    ],
    "n_scored": 1032,
    "note": "a well-calibrated head lies on the diagonal; above it is under-confident, below it over-confident"
  },
  "verdict_vs_baselines": {
    "all-annotated": {
      "auc": 0.9673680105356348,
      "minus_composition": 0.23120411974856048,
      "minus_retrieval": 0.036790087742385014
    },
    "measured-only": {
      "auc": 0.9214275459526514,
      "minus_composition": 0.18526365516557708,
      "minus_retrieval": -0.009150376840598384
    }
  },
  "pu_prior_sweep": {
    "estimated": {
      "c": 0.6446542590856552,
      "auc": 0.7820336641265146,
      "auc_std": 0.15914023583640732
    },
    "n_scored_out_of_fold": 1033,
    "assumed": {
      "1%": {
        "n_called_positive": 805,
        "fraction_called_positive": 0.7792836398838335
      },
      "3%": {
        "n_called_positive": 758,
        "fraction_called_positive": 0.7337850919651501
      },
      "5%": {
        "n_called_positive": 718,
        "fraction_called_positive": 0.6950629235237173
      },
      "10%": {
        "n_called_positive": 675,
        "fraction_called_positive": 0.6534365924491772
      }
    },
    "note": "AUC is omitted on purpose: it is invariant to the prior by construction. The prior sets how many sequences get CALLED positive, which is the decision it actually governs."
  }
}