[wip] flux eval

Summary:

Test Plan:

Reviewers:

Subscribers:

Tasks:

Tags:
ghstack-source-id: 70a7b71
ghstack-comment-id: 3667066648
Pull-Request: #3502

Author: vkuzo

.gitignore

@@ -381,3 +381,4 @@ torchao/experimental/deps
 
 # local claude code files
 CLAUDE.local.md
+.claude

benchmarks/quantization/create_quantized_model.py

@@ -9,36 +9,7 @@
 
 import torch
 from transformers import AutoModelForCausalLM, AutoTokenizer, TorchAoConfig
-
-from torchao.quantization import (
-    Float8DynamicActivationFloat8WeightConfig,
-    Float8DynamicActivationInt4WeightConfig,
-    Int4WeightOnlyConfig,
-    Int8DynamicActivationInt8WeightConfig,
-    Int8WeightOnlyConfig,
-    PerRow,
-)
-
-
-def string_to_config(s):
-    if s == "None":
-        return None
-    elif s == "float8_rowwise":
-        return Float8DynamicActivationFloat8WeightConfig(granularity=PerRow())
-    elif s == "int4_groupwise_weight_float8_rowwise_activation":
-        return Float8DynamicActivationInt4WeightConfig()
-    elif s == "int4_groupwise_hqq_weight_only":
-        return Int4WeightOnlyConfig(
-            group_size=32,
-            int4_packing_format="tile_packed_to_4d",
-            int4_choose_qparams_algorithm="hqq",
-        )
-    elif s == "int8_rowwise_weight_only":
-        return Int8WeightOnlyConfig()
-    elif s == "int8_rowwise":
-        return Int8DynamicActivationInt8WeightConfig()
-    else:
-        raise AssertionError(f"unsupported {s}")
+from utils import string_to_config
 
 
 def quantize_model_and_save(model_id, quant_config, output_dir):