Add PCA NRE

README.md

@@ -246,6 +246,7 @@ distance: f4d5b3 -->
 > - [x] [![she](https://img.shields.io/badge/ICLR'23-SHE-fdd7e6?style=for-the-badge)](https://github.com/zjs975584714/SHE)    ![postprocess]
 > - [x] [![gen](https://img.shields.io/badge/CVPR'23-GEN-fdd7e6?style=for-the-badge)](https://openaccess.thecvf.com/content/CVPR2023/papers/Liu_GEN_Pushing_the_Limits_of_Softmax-Based_Out-of-Distribution_Detection_CVPR_2023_paper.pdf)    ![postprocess]
 > - [x] [![nnguide](https://img.shields.io/badge/ICCV'23-NNGuide-fdd7e6?style=for-the-badge)](https://arxiv.org/abs/2309.14888)    ![postprocess]
+> - [x] [![pca-nre](https://img.shields.io/badge/ICCV'23-PCA\_NRE-fdd7e6?style=for-the-badge)](https://github.com/SYSU-MIA-GROUP/pca-based-out-of-distribution-detection)    ![postprocess]
 > - [x] [![relation](https://img.shields.io/badge/NEURIPS'23-Relation-fdd7e6?style=for-the-badge)](https://arxiv.org/abs/2301.12321)    ![postprocess]
 > - [x] [![vra](https://img.shields.io/badge/NeurIPS'23-VRA-fdd7e6?style=for-the-badge)](https://github.com/zeroQiaoba/VRA)    ![postprocess]
 > - [x] [![scale](https://img.shields.io/badge/ICLR'24-Scale-fdd7e6?style=for-the-badge)](https://github.com/kai422/SCALE)    ![postprocess]

configs/postprocessors/pca_nre.yml

@@ -0,0 +1,6 @@
+postprocessor:
+  name: pca_nre
+  APS_mode: False
+  postprocessor_args:
+    k: 256
+    percentile: 92

openood/evaluation_api/postprocessor.py

@@ -16,8 +16,9 @@
     RMDSPostprocessor, SHEPostprocessor, CIDERPostprocessor, NPOSPostprocessor,
     GENPostprocessor, NNGuidePostprocessor, RelationPostprocessor,
     T2FNormPostprocessor, ReweightOODPostprocessor, fDBDPostprocessor,
-    AdaScalePostprocessor, IODINPostprocessor, NCIPostprocessor,CFOODPostprocessor,
-    VRAPostprocessor, GrOODPostprocessor)
+    AdaScalePostprocessor, IODINPostprocessor, NCIPostprocessor,
+    PCANREPostprocessor, CFOODPostprocessor, VRAPostprocessor,
+    GrOODPostprocessor)
 from openood.utils.config import Config, merge_configs
 
 postprocessors = {
@@ -70,6 +71,7 @@
     'reweightood': ReweightOODPostprocessor,
     'adascale_a': AdaScalePostprocessor,
     'adascale_l': AdaScalePostprocessor,
+    'pca_nre': PCANREPostprocessor,
     'grood': GrOODPostprocessor,
     'vra': VRAPostprocessor,
     'cfood': CFOODPostprocessor,

openood/postprocessors/__init__.py

@@ -47,7 +47,7 @@
 from .t2fnorm_postprocessor import T2FNormPostprocessor
 from .reweightood_postprocessor import ReweightOODPostprocessor
 from .adascale_postprocessor import AdaScalePostprocessor
+from .pca_nre_postprocessor import PCANREPostprocessor
 from .grood import GrOODPostprocessor
 from .vra_postprocessor import VRAPostprocessor
 from .cfood_postprocessor import CFOODPostprocessor
-

openood/postprocessors/pca_nre_postprocessor.py

@@ -0,0 +1,71 @@
+from typing import Any
+
+import numpy as np
+import torch
+import torch.nn as nn
+from scipy.special import logsumexp
+from tqdm import tqdm
+
+from .base_postprocessor import BasePostprocessor
+
+
+class PCANREPostprocessor(BasePostprocessor):
+    def __init__(self, config):
+        super().__init__(config)
+        self.args = self.config.postprocessor.postprocessor_args
+        self.k = self.args.k
+        self.percentile = self.args.percentile
+        self.setup_flag = False
+
+    def setup(self, net: nn.Module, id_loader_dict, ood_loader_dict):
+        if not self.setup_flag:
+            activation_log = []
+            net.eval()
+            with torch.no_grad():
+                self.w, self.b = net.get_fc()
+                for batch in tqdm(id_loader_dict['train'],
+                                  desc='Setup: ',
+                                  position=0,
+                                  leave=True):
+                    data = batch['data'].cuda()
+                    data = data.float()
+
+                    _, feature = net(data, return_feature=True)
+                    activation_log.append(feature.data.cpu().numpy())
+
+            self.activation_log = np.concatenate(activation_log, axis=0)
+            self.setup_flag = True
+        else:
+            pass
+
+        self.threshold = np.percentile(self.activation_log.flatten(),
+                                       self.percentile)
+
+        self.activation_log_mean = np.mean(self.activation_log, axis=0)
+
+        cov = np.cov(self.activation_log.T)
+        u, s, v = np.linalg.svd(cov)
+
+        self.M = u[:, :self.k] @ u[:, :self.k].T
+        self.dim = self.M.shape[0]
+
+    @torch.no_grad()
+    def postprocess(self, net: nn.Module, data: Any):
+        _, feature_ood = net.forward(data, return_feature=True)
+        feature_ood = feature_ood.cpu()
+        feature_ood = feature_ood.clip(max=self.threshold)
+        logit_ood = feature_ood @ self.w.T + self.b
+        _, pred = torch.max(logit_ood, dim=1)
+        rec_ood = np.linalg.norm((feature_ood - self.activation_log_mean)
+                                 @ (np.identity(self.dim) - self.M),
+                                 axis=-1)
+        r_ood = rec_ood / np.linalg.norm(feature_ood, axis=-1)
+        score_ood = logsumexp(logit_ood, axis=-1) * (1.0 - r_ood)
+        return pred, torch.from_numpy(score_ood)
+
+    def set_hyperparam(self, hyperparam: list):
+        self.k = hyperparam[0]
+        self.percentile = hyperparam[1]
+
+    def get_hyperparam(self):
+        return [self.k, self.percentile]

openood/postprocessors/utils.py

@@ -43,6 +43,7 @@
 from .rts_postprocessor import RTSPostprocessor
 from .gen_postprocessor import GENPostprocessor
 from .relation_postprocessor import RelationPostprocessor
+from .pca_nre_postprocessor import PCANREPostprocessor
 from .grood import GrOODPostprocessor
 from .vra_postprocessor import VRAPostprocessor
 
@@ -92,6 +93,7 @@ def get_postprocessor(config: Config):
         'gen': GENPostprocessor,
         'relation': RelationPostprocessor,
         't2fnorm': T2FNormPostprocessor,
+        'pca_nre': PCANREPostprocessor,
         'grood': GrOODPostprocessor,
         'vra': VRAPostprocessor,
     }

scripts/ood/pca_nre/cifar100_test_ood_pca_nre.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+# sh scripts/ood/ash/cifar100_test_ood_ash.sh
+
+# GPU=1
+# CPU=1
+# node=73
+# jobname=openood
+
+PYTHONPATH='.':$PYTHONPATH \
+# srun -p dsta --mpi=pmi2 --gres=gpu:${GPU} -n1 \
+# --cpus-per-task=${CPU} --ntasks-per-node=${GPU} \
+# --kill-on-bad-exit=1 --job-name=${jobname} -w SG-IDC1-10-51-2-${node} \
+
+python main.py \
+    --config configs/datasets/cifar100/cifar100.yml \
+    configs/datasets/cifar100/cifar100_ood.yml \
+    configs/networks/resnet18_32x32.yml \
+    configs/pipelines/test/test_ood.yml \
+    configs/preprocessors/base_preprocessor.yml \
+    configs/postprocessors/pca_nre.yml \
+    --network.checkpoint 'results/cifar100_resnet18_32x32_base_e100_lr0.1_default/s0/best.ckpt'
+
+############################################
+# alternatively, we recommend using the
+# new unified, easy-to-use evaluator with
+# the example script scripts/eval_ood.py
+# especially if you want to get results from
+# multiple runs
+python scripts/eval_ood.py \
+    --id-data cifar100 \
+    --root ./results/cifar100_resnet18_32x32_base_e100_lr0.1_default \
+    --postprocessor pca_nre \
+    --save-score --save-csv

scripts/ood/pca_nre/cifar10_test_ood_pca_nre.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+# sh scripts/ood/she/cifar10_test_ood_she.sh
+
+# GPU=1
+# CPU=1
+# node=73
+# jobname=openood
+
+PYTHONPATH='.':$PYTHONPATH \
+# srun -p dsta --mpi=pmi2 --gres=gpu:${GPU} -n1 \
+# --cpus-per-task=${CPU} --ntasks-per-node=${GPU} \
+# --kill-on-bad-exit=1 --job-name=${jobname} -w SG-IDC1-10-51-2-${node} \
+
+python main.py \
+    --config configs/datasets/cifar10/cifar10.yml \
+    configs/datasets/cifar10/cifar10_ood.yml \
+    configs/networks/resnet18_32x32.yml \
+    configs/pipelines/test/test_ood.yml \
+    configs/preprocessors/base_preprocessor.yml \
+    configs/postprocessors/pca_nre.yml \
+    --num_workers 8 \
+    --network.checkpoint 'results/cifar10_resnet18_32x32_base_e100_lr0.1_default/s0/best.ckpt' \
+    --mark 1
+
+############################################
+# alternatively, we recommend using the
+# new unified, easy-to-use evaluator with
+# the example script scripts/eval_ood.py
+# especially if you want to get results from
+# multiple runs
+python scripts/eval_ood.py \
+    --id-data cifar10 \
+    --root ./results/cifar10_resnet18_32x32_base_e100_lr0.1_default \
+    --postprocessor pca_nre \
+    --save-score --save-csv

scripts/ood/pca_nre/imagenet200_test_ood_pca_nre.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+# sh scripts/ood/ash/imagenet200_test_ood_ash.sh
+
+############################################
+# alternatively, we recommend using the
+# new unified, easy-to-use evaluator with
+# the example script scripts/eval_ood.py
+# especially if you want to get results from
+# multiple runs
+
+# ood
+python scripts/eval_ood.py \
+    --id-data imagenet200 \
+    --root ./results/imagenet200_resnet18_224x224_base_e90_lr0.1_default \
+    --postprocessor pca_nre \
+    --save-score --save-csv #--fsood
+
+# full-spectrum ood
+python scripts/eval_ood.py \
+    --id-data imagenet200 \
+    --root ./results/imagenet200_resnet18_224x224_base_e90_lr0.1_default \
+    --postprocessor pca_nre \
+    --save-score --save-csv --fsood

scripts/ood/pca_nre/imagenet_test_ood_pca_nre.sh

@@ -0,0 +1,47 @@
+#!/bin/bash
+# sh scripts/ood/ash/imagenet_test_ood_ash.sh
+
+GPU=1
+CPU=1
+node=63
+jobname=openood
+
+PYTHONPATH='.':$PYTHONPATH \
+# srun -p dsta --mpi=pmi2 --gres=gpu:${GPU} -n1 \
+# --cpus-per-task=${CPU} --ntasks-per-node=${GPU} \
+# --kill-on-bad-exit=1 --job-name=${jobname} -w SG-IDC1-10-51-2-${node} \
+# python main.py \
+#     --config configs/datasets/imagenet/imagenet.yml \
+#     configs/datasets/imagenet/imagenet_ood.yml \
+#     configs/networks/resnet50.yml \
+#     configs/pipelines/test/test_ood.yml \
+#     configs/preprocessors/base_preprocessor.yml \
+#     configs/postprocessors/gen.yml \
+#     --num_workers 4 \
+#     --ood_dataset.image_size 256 \
+#     --dataset.test.batch_size 256 \
+#     --dataset.val.batch_size 256 \
+#     --network.pretrained True \
+#     --network.checkpoint 'results/pretrained_weights/resnet50_imagenet1k_v1.pth' \
+#     --merge_option merge
+
+############################################
+# we recommend using the
+# new unified, easy-to-use evaluator with
+# the example script scripts/eval_ood_imagenet.py
+
+# available architectures:
+# resnet50, swin-t, vit-b-16
+# ood
+python scripts/eval_ood_imagenet.py \
+    --tvs-pretrained \
+    --arch resnet50 \
+    --postprocessor pca_nre \
+    --save-score --save-csv #--fsood
+
+# full-spectrum ood
+python scripts/eval_ood_imagenet.py \
+    --tvs-pretrained \
+    --arch resnet50 \
+    --postprocessor pca_nre \
+    --save-score --save-csv --fsood