remove old timm models from benchmark (#164805)

Summary:
Prune models from TorchInductor dashboard to reduce ci cost. This PR prunes for timm models according to the [doc](https://docs.google.com/document/d/1nLPNNAU-_M9Clx9FMrJ1ycdPxe-xRA54olPnsFzdpoU/edit?tab=t.0), which reduces from 60 to 14 models.

X-link: pytorch/pytorch#164805
Approved by: https://github.com/anijain2305, https://github.com/seemethere, https://github.com/huydhn, https://github.com/malfet

Reviewed By: izaitsevfb

Differential Revision: D84181820

fbshipit-source-id: 13034ebbe8511b819c895467a04415c999b6a173

Author: BoyuanFeng

userbenchmark/dynamo/dynamobench/common.py

@@ -117,11 +117,7 @@ class CI(NamedTuple):
 
 
 CI_SKIP_OPTIMIZER = {
-    # TIMM
-    "convmixer_768_32",  # accuracy
-    "hrnet_w18",  # Stack issue in fx
     # HF
-    "pnasnet5large",  # Stack issue in fx
     "MobileBertForMaskedLM",  # Stack issue in fx
 }
 
@@ -200,25 +196,8 @@ class CI(NamedTuple):
     "XGLMForCausalLM",
     # TIMM
     "adv_inception_v3",
-    "botnet26t_256",
-    "cait_m36_384",  # OOM
-    "coat_lite_mini",
-    "convit_base",
-    "dpn107",
-    "fbnetv3_b",
-    "gernet_l",
-    "lcnet_050",
-    "mixnet_l",
-    "res2net101_26w_4s",
-    "res2net50_14w_8s",
-    "res2next50",
-    "resnest101e",
-    "sebotnet33ts_256",
-    "swsl_resnext101_32x16d",
     "tf_efficientnet_b0",
     "ghostnet_100",
-    "gmixer_24_224",
-    "tinynet_a",
 }
 
 # These models OOM in CI
@@ -243,25 +222,17 @@ class CI(NamedTuple):
     "mobilenet_v2_quantized_qat",
     "phi_1_5 resnet50_quantized_qat",
     "BlenderbotForCausalLM",
-    "cait_m36_384",
     "DALLE2_pytorch",
     "moco",
     "timm_efficientdet",
     "ghostnet_100",
-    "regnety_002",
-    "poolformer_m36",
     "inception_v3",
-    "tinynet_a",
-    "selecsls42b",
     "mobilevit_s",
     "pytorch_CycleGAN_and_pix2pix",
     "vision_maskrcnn",
-    "resmlp_12_224",
     "dlrm",
     "resnet50",
     "dm_nfnet_f0",
-    "pit_b_224",
-    "tf_mixnet_l",
 }
 
 
@@ -3782,7 +3753,6 @@ def run(runner, args, original_dir=None):
         os.environ["CUBLAS_WORKSPACE_CONFIG"] = ":4096:8"
         if args.only is not None and args.only in {
             "nvidia_deeprecommender",
-            "crossvit_9_240",
         }:
             # These seem unhappy with numerics of larger cuBLASLt workspace
             torch.backends.cuda.matmul.allow_bf16_reduced_precision_reduction = False
@@ -3810,7 +3780,6 @@ def run(runner, args, original_dir=None):
             runner.skip_models.update(
                 {
                     # xfail: https://github.com/pytorch/pytorch/issues/145773
-                    "convit_base",
                     "llama",
                     "cm3leon_generate",
                 }

userbenchmark/dynamo/dynamobench/timm_models.py

@@ -60,84 +60,41 @@ def pip_install(package):
 
 BATCH_SIZE_DIVISORS = {
     "beit_base_patch16_224": 2,
-    "convit_base": 2,
-    "convmixer_768_32": 2,
-    "convnext_base": 2,
-    "cspdarknet53": 2,
     "deit_base_distilled_patch16_224": 2,
     "gluon_xception65": 2,
     "mobilevit_s": 2,
-    "pnasnet5large": 2,
-    "poolformer_m36": 2,
-    "resnest101e": 2,
     "swin_base_patch4_window7_224": 2,
-    "swsl_resnext101_32x16d": 2,
-    "vit_base_patch16_224": 2,
-    "volo_d1_224": 2,
-    "jx_nest_base": 4,
 }
 
 REQUIRE_HIGHER_TOLERANCE = {
-    "crossvit_9_240",
-    "fbnetv3_b",
-    "gmixer_24_224",
-    "hrnet_w18",
     "inception_v3",
-    "mixer_b16_224",
     "mobilenetv3_large_100",
-    "sebotnet33ts_256",
-    "selecsls42b",
-    "convnext_base",
-    "cait_m36_384",
 }
 
-REQUIRE_HIGHER_TOLERANCE_AMP = {
-    "poolformer_m36",
-}
+REQUIRE_HIGHER_TOLERANCE_AMP = {}
 
 REQUIRE_EVEN_HIGHER_TOLERANCE = {
-    "levit_128",
-    "sebotnet33ts_256",
     "beit_base_patch16_224",
-    "cspdarknet53",
 }
 
 # These models need higher tolerance in MaxAutotune mode
-REQUIRE_EVEN_HIGHER_TOLERANCE_MAX_AUTOTUNE = {
-    "gluon_inception_v3",
-}
+REQUIRE_EVEN_HIGHER_TOLERANCE_MAX_AUTOTUNE = {}
 
 REQUIRE_HIGHER_TOLERANCE_FOR_FREEZING = {
     "adv_inception_v3",
-    "botnet26t_256",
-    "gluon_inception_v3",
-    "selecsls42b",
-    "swsl_resnext101_32x16d",
 }
 
 SCALED_COMPUTE_LOSS = {
-    "ese_vovnet19b_dw",
-    "fbnetc_100",
-    "mnasnet_100",
     "mobilevit_s",
-    "sebotnet33ts_256",
 }
 
-FORCE_AMP_FOR_FP16_BF16_MODELS = {
-    "convit_base",
-    "xcit_large_24_p8_224",
-}
+FORCE_AMP_FOR_FP16_BF16_MODELS = {}
 
-SKIP_ACCURACY_CHECK_AS_EAGER_NON_DETERMINISTIC_MODELS = {
-    "xcit_large_24_p8_224",
-}
+SKIP_ACCURACY_CHECK_AS_EAGER_NON_DETERMINISTIC_MODELS = {}
 
 REQUIRE_LARGER_MULTIPLIER_FOR_SMALLER_TENSOR = {
     "inception_v3",
     "mobilenetv3_large_100",
-    "cspdarknet53",
-    "gluon_inception_v3",
-    "cait_m36_384",
 }
 
 
@@ -269,15 +226,11 @@ def skip_accuracy_check_as_eager_non_deterministic(self):
 
     @property
     def guard_on_nn_module_models(self):
-        return {
-            "convit_base",
-        }
+        return {}
 
     @property
     def inline_inbuilt_nn_modules_models(self):
-        return {
-            "lcnet_050",
-        }
+        return {}
 
     @download_retry_decorator
     def _download_model(self, model_name):

userbenchmark/dynamo/dynamobench/timm_models_list.txt

@@ -1,61 +1,14 @@
 adv_inception_v3 128
 beit_base_patch16_224 128
-botnet26t_256 128
-cait_m36_384 4
-coat_lite_mini 128
-convit_base 128
-convmixer_768_32 64
-convnext_base 128
-crossvit_9_240 256
-cspdarknet53 128
 deit_base_distilled_patch16_224 128
-dla102 128
 dm_nfnet_f0 128
-dpn107 64
-eca_botnext26ts_256 128
-eca_halonext26ts 128
-ese_vovnet19b_dw 256
-fbnetc_100 512
-fbnetv3_b 256
-gernet_l 128
 ghostnet_100 512
-gluon_inception_v3 256
-gmixer_24_224 128
-gmlp_s16_224 128
-hrnet_w18 128
 inception_v3 128
-jx_nest_base 128
-lcnet_050 256
-levit_128 1024
-mixer_b16_224 128
-mixnet_l 128
-mnasnet_100 512
 mobilenetv2_100 128
 mobilenetv3_large_100 512
 mobilevit_s 128
 nfnet_l0 128
-pit_b_224 64
-pnasnet5large 32
-poolformer_m36 128
-regnety_002 1024
 repvgg_a2 128
-res2net101_26w_4s 128
-res2net50_14w_8s 128
-res2next50 128
-resmlp_12_224 128
-resnest101e 128
-rexnet_100 256
-sebotnet33ts_256 64
-selecsls42b 128
-spnasnet_100 128
 swin_base_patch4_window7_224 128
-swsl_resnext101_32x16d 64
 tf_efficientnet_b0 128
-tf_mixnet_l 128
-tinynet_a 128
-tnt_s_patch16_224 128
-twins_pcpvt_base 128
 visformer_small 128
-vit_base_patch16_224 128
-volo_d1_224 128
-xcit_large_24_p8_224 16

userbenchmark/dynamo/dynamobench/timm_models_list_cpu.txt

@@ -1,59 +1,14 @@
 adv_inception_v3,128
 beit_base_patch16_224,64
-botnet26t_256,128
-cait_m36_384,4
-coat_lite_mini,32
-convit_base,64
-convmixer_768_32,2
-convnext_base,64
-crossvit_9_240,32
-cspdarknet53,64
 deit_base_distilled_patch16_224,64
 dm_nfnet_f0,128
-dpn107,32
-eca_botnext26ts_256,128
-eca_halonext26ts,128
-ese_vovnet19b_dw,128
-fbnetc_100,32
-fbnetv3_b,32
-gernet_l,128
 ghostnet_100,128
-gluon_inception_v3,128
-gmixer_24_224,16
-gmlp_s16_224,128
-hrnet_w18,128
 inception_v3,128
-jx_nest_base,32
-lcnet_050,64
-mixer_b16_224,128
-mixnet_l,128
-mnasnet_100,32
 mobilenetv2_100,32
 mobilenetv3_large_100,32
 mobilevit_s,256
 nfnet_l0,128
-pit_b_224,64
-pnasnet5large,16
-poolformer_m36,64
-regnety_002,128
 repvgg_a2,128
-res2net101_26w_4s,64
-res2net50_14w_8s,128
-res2next50,128
-resmlp_12_224,128
-resnest101e,64
-rexnet_100,128
-sebotnet33ts_256,64
-selecsls42b,128
-spnasnet_100,32
 swin_base_patch4_window7_224,64
-swsl_resnext101_32x16d,32
 tf_efficientnet_b0,128
-tf_mixnet_l,32
-tinynet_a,128
-tnt_s_patch16_224,32
-twins_pcpvt_base,64
 visformer_small,128
-vit_base_patch16_224,64
-volo_d1_224,64
-xcit_large_24_p8_224,5