diff --git a/README.md b/README.md index 4d7ab53c..0996b623 100644 --- a/README.md +++ b/README.md @@ -246,6 +246,7 @@ distance: f4d5b3 --> > - [x] [![she](https://img.shields.io/badge/ICLR'23-SHE-fdd7e6?style=for-the-badge)](https://github.com/zjs975584714/SHE)    ![postprocess] > - [x] [![gen](https://img.shields.io/badge/CVPR'23-GEN-fdd7e6?style=for-the-badge)](https://openaccess.thecvf.com/content/CVPR2023/papers/Liu_GEN_Pushing_the_Limits_of_Softmax-Based_Out-of-Distribution_Detection_CVPR_2023_paper.pdf)    ![postprocess] > - [x] [![nnguide](https://img.shields.io/badge/ICCV'23-NNGuide-fdd7e6?style=for-the-badge)](https://arxiv.org/abs/2309.14888)    ![postprocess] +> - [x] [![pca-nre](https://img.shields.io/badge/ICCV'23-PCA\_NRE-fdd7e6?style=for-the-badge)](https://github.com/SYSU-MIA-GROUP/pca-based-out-of-distribution-detection)    ![postprocess] > - [x] [![relation](https://img.shields.io/badge/NEURIPS'23-Relation-fdd7e6?style=for-the-badge)](https://arxiv.org/abs/2301.12321)    ![postprocess] > - [x] [![vra](https://img.shields.io/badge/NeurIPS'23-VRA-fdd7e6?style=for-the-badge)](https://github.com/zeroQiaoba/VRA)    ![postprocess] > - [x] [![scale](https://img.shields.io/badge/ICLR'24-Scale-fdd7e6?style=for-the-badge)](https://github.com/kai422/SCALE)    ![postprocess] diff --git a/configs/postprocessors/pca_nre.yml b/configs/postprocessors/pca_nre.yml new file mode 100644 index 00000000..f69dfd4f --- /dev/null +++ b/configs/postprocessors/pca_nre.yml @@ -0,0 +1,6 @@ +postprocessor: + name: pca_nre + APS_mode: False + postprocessor_args: + k: 256 + percentile: 92 diff --git a/openood/evaluation_api/postprocessor.py b/openood/evaluation_api/postprocessor.py index e7c33b61..ad285cbd 100644 --- a/openood/evaluation_api/postprocessor.py +++ b/openood/evaluation_api/postprocessor.py @@ -16,8 +16,9 @@ RMDSPostprocessor, SHEPostprocessor, CIDERPostprocessor, NPOSPostprocessor, GENPostprocessor, NNGuidePostprocessor, RelationPostprocessor, T2FNormPostprocessor, ReweightOODPostprocessor, fDBDPostprocessor, - AdaScalePostprocessor, IODINPostprocessor, NCIPostprocessor,CFOODPostprocessor, - VRAPostprocessor, GrOODPostprocessor) + AdaScalePostprocessor, IODINPostprocessor, NCIPostprocessor, + PCANREPostprocessor, CFOODPostprocessor, VRAPostprocessor, + GrOODPostprocessor) from openood.utils.config import Config, merge_configs postprocessors = { @@ -70,6 +71,7 @@ 'reweightood': ReweightOODPostprocessor, 'adascale_a': AdaScalePostprocessor, 'adascale_l': AdaScalePostprocessor, + 'pca_nre': PCANREPostprocessor, 'grood': GrOODPostprocessor, 'vra': VRAPostprocessor, 'cfood': CFOODPostprocessor, diff --git a/openood/postprocessors/__init__.py b/openood/postprocessors/__init__.py index 13454fb1..50e06299 100644 --- a/openood/postprocessors/__init__.py +++ b/openood/postprocessors/__init__.py @@ -47,7 +47,7 @@ from .t2fnorm_postprocessor import T2FNormPostprocessor from .reweightood_postprocessor import ReweightOODPostprocessor from .adascale_postprocessor import AdaScalePostprocessor +from .pca_nre_postprocessor import PCANREPostprocessor from .grood import GrOODPostprocessor from .vra_postprocessor import VRAPostprocessor from .cfood_postprocessor import CFOODPostprocessor - diff --git a/openood/postprocessors/pca_nre_postprocessor.py b/openood/postprocessors/pca_nre_postprocessor.py new file mode 100644 index 00000000..214bdb2f --- /dev/null +++ b/openood/postprocessors/pca_nre_postprocessor.py @@ -0,0 +1,71 @@ +from typing import Any + +import numpy as np +import torch +import torch.nn as nn +from scipy.special import logsumexp +from tqdm import tqdm + +from .base_postprocessor import BasePostprocessor + + +class PCANREPostprocessor(BasePostprocessor): + def __init__(self, config): + super().__init__(config) + self.args = self.config.postprocessor.postprocessor_args + self.k = self.args.k + self.percentile = self.args.percentile + self.setup_flag = False + + def setup(self, net: nn.Module, id_loader_dict, ood_loader_dict): + if not self.setup_flag: + activation_log = [] + net.eval() + with torch.no_grad(): + self.w, self.b = net.get_fc() + for batch in tqdm(id_loader_dict['train'], + desc='Setup: ', + position=0, + leave=True): + data = batch['data'].cuda() + data = data.float() + + _, feature = net(data, return_feature=True) + activation_log.append(feature.data.cpu().numpy()) + + self.activation_log = np.concatenate(activation_log, axis=0) + self.setup_flag = True + else: + pass + + self.threshold = np.percentile(self.activation_log.flatten(), + self.percentile) + + self.activation_log_mean = np.mean(self.activation_log, axis=0) + + cov = np.cov(self.activation_log.T) + u, s, v = np.linalg.svd(cov) + + self.M = u[:, :self.k] @ u[:, :self.k].T + self.dim = self.M.shape[0] + + @torch.no_grad() + def postprocess(self, net: nn.Module, data: Any): + _, feature_ood = net.forward(data, return_feature=True) + feature_ood = feature_ood.cpu() + feature_ood = feature_ood.clip(max=self.threshold) + logit_ood = feature_ood @ self.w.T + self.b + _, pred = torch.max(logit_ood, dim=1) + rec_ood = np.linalg.norm((feature_ood - self.activation_log_mean) + @ (np.identity(self.dim) - self.M), + axis=-1) + r_ood = rec_ood / np.linalg.norm(feature_ood, axis=-1) + score_ood = logsumexp(logit_ood, axis=-1) * (1.0 - r_ood) + return pred, torch.from_numpy(score_ood) + + def set_hyperparam(self, hyperparam: list): + self.k = hyperparam[0] + self.percentile = hyperparam[1] + + def get_hyperparam(self): + return [self.k, self.percentile] diff --git a/openood/postprocessors/utils.py b/openood/postprocessors/utils.py index 133e0aa4..0cdda9f9 100644 --- a/openood/postprocessors/utils.py +++ b/openood/postprocessors/utils.py @@ -43,6 +43,7 @@ from .rts_postprocessor import RTSPostprocessor from .gen_postprocessor import GENPostprocessor from .relation_postprocessor import RelationPostprocessor +from .pca_nre_postprocessor import PCANREPostprocessor from .grood import GrOODPostprocessor from .vra_postprocessor import VRAPostprocessor @@ -92,6 +93,7 @@ def get_postprocessor(config: Config): 'gen': GENPostprocessor, 'relation': RelationPostprocessor, 't2fnorm': T2FNormPostprocessor, + 'pca_nre': PCANREPostprocessor, 'grood': GrOODPostprocessor, 'vra': VRAPostprocessor, } diff --git a/scripts/ood/pca_nre/cifar100_test_ood_pca_nre.sh b/scripts/ood/pca_nre/cifar100_test_ood_pca_nre.sh new file mode 100755 index 00000000..c804251b --- /dev/null +++ b/scripts/ood/pca_nre/cifar100_test_ood_pca_nre.sh @@ -0,0 +1,33 @@ +#!/bin/bash +# sh scripts/ood/ash/cifar100_test_ood_ash.sh + +# GPU=1 +# CPU=1 +# node=73 +# jobname=openood + +PYTHONPATH='.':$PYTHONPATH \ +# srun -p dsta --mpi=pmi2 --gres=gpu:${GPU} -n1 \ +# --cpus-per-task=${CPU} --ntasks-per-node=${GPU} \ +# --kill-on-bad-exit=1 --job-name=${jobname} -w SG-IDC1-10-51-2-${node} \ + +python main.py \ + --config configs/datasets/cifar100/cifar100.yml \ + configs/datasets/cifar100/cifar100_ood.yml \ + configs/networks/resnet18_32x32.yml \ + configs/pipelines/test/test_ood.yml \ + configs/preprocessors/base_preprocessor.yml \ + configs/postprocessors/pca_nre.yml \ + --network.checkpoint 'results/cifar100_resnet18_32x32_base_e100_lr0.1_default/s0/best.ckpt' + +############################################ +# alternatively, we recommend using the +# new unified, easy-to-use evaluator with +# the example script scripts/eval_ood.py +# especially if you want to get results from +# multiple runs +python scripts/eval_ood.py \ + --id-data cifar100 \ + --root ./results/cifar100_resnet18_32x32_base_e100_lr0.1_default \ + --postprocessor pca_nre \ + --save-score --save-csv diff --git a/scripts/ood/pca_nre/cifar10_test_ood_pca_nre.sh b/scripts/ood/pca_nre/cifar10_test_ood_pca_nre.sh new file mode 100755 index 00000000..e04bc62e --- /dev/null +++ b/scripts/ood/pca_nre/cifar10_test_ood_pca_nre.sh @@ -0,0 +1,35 @@ +#!/bin/bash +# sh scripts/ood/she/cifar10_test_ood_she.sh + +# GPU=1 +# CPU=1 +# node=73 +# jobname=openood + +PYTHONPATH='.':$PYTHONPATH \ +# srun -p dsta --mpi=pmi2 --gres=gpu:${GPU} -n1 \ +# --cpus-per-task=${CPU} --ntasks-per-node=${GPU} \ +# --kill-on-bad-exit=1 --job-name=${jobname} -w SG-IDC1-10-51-2-${node} \ + +python main.py \ + --config configs/datasets/cifar10/cifar10.yml \ + configs/datasets/cifar10/cifar10_ood.yml \ + configs/networks/resnet18_32x32.yml \ + configs/pipelines/test/test_ood.yml \ + configs/preprocessors/base_preprocessor.yml \ + configs/postprocessors/pca_nre.yml \ + --num_workers 8 \ + --network.checkpoint 'results/cifar10_resnet18_32x32_base_e100_lr0.1_default/s0/best.ckpt' \ + --mark 1 + +############################################ +# alternatively, we recommend using the +# new unified, easy-to-use evaluator with +# the example script scripts/eval_ood.py +# especially if you want to get results from +# multiple runs +python scripts/eval_ood.py \ + --id-data cifar10 \ + --root ./results/cifar10_resnet18_32x32_base_e100_lr0.1_default \ + --postprocessor pca_nre \ + --save-score --save-csv diff --git a/scripts/ood/pca_nre/imagenet200_test_ood_pca_nre.sh b/scripts/ood/pca_nre/imagenet200_test_ood_pca_nre.sh new file mode 100755 index 00000000..b3694d9b --- /dev/null +++ b/scripts/ood/pca_nre/imagenet200_test_ood_pca_nre.sh @@ -0,0 +1,23 @@ +#!/bin/bash +# sh scripts/ood/ash/imagenet200_test_ood_ash.sh + +############################################ +# alternatively, we recommend using the +# new unified, easy-to-use evaluator with +# the example script scripts/eval_ood.py +# especially if you want to get results from +# multiple runs + +# ood +python scripts/eval_ood.py \ + --id-data imagenet200 \ + --root ./results/imagenet200_resnet18_224x224_base_e90_lr0.1_default \ + --postprocessor pca_nre \ + --save-score --save-csv #--fsood + +# full-spectrum ood +python scripts/eval_ood.py \ + --id-data imagenet200 \ + --root ./results/imagenet200_resnet18_224x224_base_e90_lr0.1_default \ + --postprocessor pca_nre \ + --save-score --save-csv --fsood diff --git a/scripts/ood/pca_nre/imagenet_test_ood_pca_nre.sh b/scripts/ood/pca_nre/imagenet_test_ood_pca_nre.sh new file mode 100755 index 00000000..a25cdc7d --- /dev/null +++ b/scripts/ood/pca_nre/imagenet_test_ood_pca_nre.sh @@ -0,0 +1,47 @@ +#!/bin/bash +# sh scripts/ood/ash/imagenet_test_ood_ash.sh + +GPU=1 +CPU=1 +node=63 +jobname=openood + +PYTHONPATH='.':$PYTHONPATH \ +# srun -p dsta --mpi=pmi2 --gres=gpu:${GPU} -n1 \ +# --cpus-per-task=${CPU} --ntasks-per-node=${GPU} \ +# --kill-on-bad-exit=1 --job-name=${jobname} -w SG-IDC1-10-51-2-${node} \ +# python main.py \ +# --config configs/datasets/imagenet/imagenet.yml \ +# configs/datasets/imagenet/imagenet_ood.yml \ +# configs/networks/resnet50.yml \ +# configs/pipelines/test/test_ood.yml \ +# configs/preprocessors/base_preprocessor.yml \ +# configs/postprocessors/gen.yml \ +# --num_workers 4 \ +# --ood_dataset.image_size 256 \ +# --dataset.test.batch_size 256 \ +# --dataset.val.batch_size 256 \ +# --network.pretrained True \ +# --network.checkpoint 'results/pretrained_weights/resnet50_imagenet1k_v1.pth' \ +# --merge_option merge + +############################################ +# we recommend using the +# new unified, easy-to-use evaluator with +# the example script scripts/eval_ood_imagenet.py + +# available architectures: +# resnet50, swin-t, vit-b-16 +# ood +python scripts/eval_ood_imagenet.py \ + --tvs-pretrained \ + --arch resnet50 \ + --postprocessor pca_nre \ + --save-score --save-csv #--fsood + +# full-spectrum ood +python scripts/eval_ood_imagenet.py \ + --tvs-pretrained \ + --arch resnet50 \ + --postprocessor pca_nre \ + --save-score --save-csv --fsood