Nine rules for tests/models

[rules.TRF060]
description = "A test that loads real pretrained weights must be marked @slow."
default_enabled = true
allowlist_models = ["auto"]

[rules.TRF060.explanation]
what_it_does = "In tests/models/*/test_modeling_*.py, flags a test method that calls from_pretrained() on a hub id -- an `org/name` string literal, or a class-level constant holding one -- unless the method or its class carries @slow. Ids that name random weights (anything matching `tiny-`, `-tiny`, `tiny-random` or `random-`) are not real weights and are left to the fast suite. `auto` is allowlisted: test_modeling_auto.py resolves remote-code repositories by design, and those fixtures are the subject under test rather than model weights."
why_bad = "The pull-request CI runs every test that is not marked @slow, on every pull request in the repository. A test that pulls real weights charges its download and its inference to contributors touching unrelated files, and turns a hub hiccup into a red CI on their branch. @slow is what moves it to the nightly run that is provisioned for it."
diff = '''
 class AcmeIntegrationTest(unittest.TestCase):
+    @slow
     def test_inference_logits(self):
         model = AcmeForCausalLM.from_pretrained("acme-ai/acme-7b").to(torch_device)
'''

[rules.TRF062]
description = "Tests must take the device from torch_device, never a hardcoded \"cuda\" string."
default_enabled = true
allowlist_models = []

[rules.TRF062.explanation]
what_it_does = "Flags a hardcoded accelerator string in a test file: `.to(\"cuda\")`, a `device=` or `device_map=` keyword whose value is a `cuda` literal, and direct `torch.cuda.*` calls that have a device-agnostic helper in testing_utils (`empty_cache` -> `backend_empty_cache`, `device_count` -> `backend_device_count`, `manual_seed_all` -> `backend_manual_seed_all`). Comparisons such as `torch_device == \"cuda\"` and Expectations SM keys such as `(\"cuda\", (8, 6))` are not flagged: naming the hardware is the point in both."
why_bad = "The slow suite runs on XPU, NPU and ROCm as well as CUDA. A hardcoded device string either errors on those runners or, worse, quietly leaves the tensor somewhere the model is not, so the test measures nothing and reports green. torch_device and the backend_* helpers are already resolved for the runner in hand."
diff = '''
-inputs = tokenizer("Hello", return_tensors="pt").to("cuda")
-torch.cuda.empty_cache()
+inputs = tokenizer("Hello", return_tensors="pt").to(torch_device)
+backend_empty_cache(torch_device)
'''

[rules.TRF063]
description = "A skip that defers a problem must link the issue tracking it."
default_enabled = true
allowlist_models = []

[rules.TRF063.explanation]
what_it_does = "In any tests/models test file, flags `@unittest.skip` whose reason is missing, is a bare verdict (`TODO`, `Not implemented`, `Very flaky`), or defers the problem without naming where it is tracked. Deferral is read from the wording -- `for now`, `temporarily`, `to investigate`, `needs a fix`, `undefined behaviour`, `no idea why` -- and a reason carrying an issue reference (`#12345`) or a URL satisfies the rule however it is phrased. A reason that states a permanent architectural fact (`Acme has no inputs_embeds`, `this module does not support standalone training`) is not a deferral and needs no link."
why_bad = "Requiring any reason at all would be theatre: 2590 of 2617 skips already carry one. What is missing is the 4% that say a test was turned off and nothing else -- 23 copies of `Skip for now`, `broken now, no idea why`, `keep hitting edge cases`. Those are the ones that never get switched back on, because there is nothing to close and nobody owns them; only 34 reasons in the whole suite reference an issue. A link turns a silent hole in coverage into tracked work."
diff = '''
-@unittest.skip(reason="Some undefined behavior with test versions of this model. Skip for now.")
+@unittest.skip(reason="Tiny checkpoint emits NaN logits past 2 layers, see #43210")
 def test_batched_generation(self):
'''

[rules.TRF065]
description = "A collected model test class must be decorated with @require_torch."
default_enabled = true
allowlist_models = []

[rules.TRF065.explanation]
what_it_does = "In test_modeling_*.py, flags a collected test class that mixes in the shared suite (ModelTesterMixin, or one of CausalLMModelTest / VLMModelTest / ALMModelTest) and carries no `require_torch` decorator. The narrower gates count: require_torch_gpu, require_torch_accelerator, require_torch_multi_gpu and friends all imply torch."
why_bad = "Model test files import their torch symbols under `if is_torch_available():`, so without the decorator the class is still collected in an environment without torch and every one of its tests errors on a NameError instead of skipping. The decorator is what turns that into an honest skip."
diff = '''
+@require_torch
 class AcmeModelTest(ModelTesterMixin, unittest.TestCase):
     all_model_classes = (AcmeModel, AcmeForCausalLM)
'''

[rules.TRF068]
description = "A model test file must contain at least one @slow integration test."
default_enabled = true
allowlist_models = []

[rules.TRF068.explanation]
what_it_does = "Flags a tests/models/*/test_modeling_*.py file in which no test method or test class carries @slow. Reported once per file, on the first collected test class. A model whose checkpoint is not published yet records that on the class -- `has_public_checkpoint = False` once the integration mixin exists, or `# trf-ignore: TRF068` above the class until it does, naming the checkpoint it is waiting on -- the directive is the record that the integration test is still owed, and grep finds every one of them when a release lands. Files covering a backbone or sub-component that the parent model's file already exercises end-to-end belong in allowlist_models instead, since that exemption is permanent."
why_bad = "The mixin suite runs tiny random configs and checks shapes, gradients and save/load round-trips. It passes on a model whose attention scaling, rope layout or mask is wrong, because wrong numbers have the right shape. A file with no integration test has never once compared this architecture against a checkpoint, which is the only check that would have caught that -- and CONTRIBUTING asks contributors to prioritize exactly that test."
diff = '''
+@slow
+@require_torch
+class AcmeIntegrationTest(unittest.TestCase):
+    def test_inference_logits(self):
+        model = AcmeForCausalLM.from_pretrained("acme-ai/acme-7b").to(torch_device)
+        expected = torch.tensor([[-1.2345, 0.6789, 2.3456]])
+        torch.testing.assert_close(out.logits[0, :1, :3], expected, atol=1e-4, rtol=1e-4)
'''

[rules.TRF061]
description = "An integration test class must clean the accelerator up in setUp and tearDown."
default_enabled = true
allowlist_models = []
cutoff_date = "2026-09-01"

[rules.TRF061.explanation]
what_it_does = "In test_modeling_*.py, for a collected test class whose @slow tests load pretrained weights, checks that the class cleans the device up at both ends of every test. Inheriting a base that does it -- the proposed IntegrationTesterMixin -- satisfies the rule outright; so does calling `cleanup(torch_device, ...)`, or backend_empty_cache plus gc.collect, in the class's own setUp and tearDown. Bases are resolved in the same file and one hop into another model's test class, as TRF042 does, so a class that inherits its lifecycle is never reported for not repeating it. Reported once per class, on the class definition."
why_bad = "The slow suite runs model after model in one process on one accelerator. A class that returns without freeing its weights leaves them resident, and the next model in the run hits an out-of-memory error -- so the failure surfaces in someone else's file, and the model that actually leaked looks green. Cleanup at both ends is what makes a class survive a predecessor that skipped its own. The rule accepts an inherited lifecycle deliberately: 492 integration classes currently hand-roll 370 setUp/tearDown methods to do the same two calls, and one shared mixin is both less to write and the only way the next change to what cleanup means -- another cache to drop, another reset to call -- lands in one place instead of 370."
diff = '''
-class AcmeIntegrationTest(unittest.TestCase):
+class AcmeIntegrationTest(IntegrationTesterMixin, unittest.TestCase):
+    model_id = "acme-ai/acme-7b"
+
     @slow
     def test_inference_logits(self):
'''

[rules.TRF066]
description = "A test fixture -- generated checkpoint or fixture dataset -- should live under hf-internal-testing."
default_enabled = true
allowlist_models = []
cutoff_date = "2026-09-01"

[rules.TRF066.explanation]
what_it_does = "Flags two kinds of fixture reference in a tests/models test file whose owner is not hf-internal-testing. First, a from_pretrained() hub id that names generated weights (`tiny-random-*`, `*-random-tiny`, `random-*`, `dummy-*`); a published checkpoint that merely has `tiny` or `small` in its name -- facebook/convnext-tiny-224, openai/whisper-tiny, asapp/sew-d-tiny-100k -- is real released weights and is never flagged. Second, a load_dataset(), hf_hub_download() or snapshot_download() repo id whose name reads as a test fixture: `dummy`, `fixtures`, `example`, `sample`, `test`, `testing`, `demo` or `ci` as a whole token. A published corpus keeps its own name and is left alone -- facebook/xnli, mozilla-foundation/common_voice_11_0 and google/code_x_glue_ct_code_to_text are data the test consumes, not fixtures the CI owns. Whether the test is @slow does not matter: a fixture the CI depends on has to be hosted where the team can restore it either way."
why_bad = "Generated checkpoints and hand-made fixture assets are both legitimate -- the smallest published checkpoint often does not fit the runner, and a deterministic logits check catches a numerical regression even when the output is garbage. What matters is who can delete the fixture. 17 checkpoint references and 43 dataset references point at personal accounts: patrickvonplaten/longformer-random-tiny, sgugger/funnel-random-tiny, nielsr/example-pdf, raushan-testing-hf/videos-test, anton-l/superb_dummy. Any of them takes a model's whole suite down the day that person tidies their profile, and nobody else can restore it. hf-internal-testing is the org the team owns, utils/create_dummy_models.py already uploads checkpoints there, and fixture assets belong beside them in an hf-internal-testing/fixtures_* dataset repo."
diff = '''
 @slow
 def test_inference_logits(self):
-    model = AcmeForCausalLM.from_pretrained("jdoe/tiny-random-acme")
+    model = AcmeForCausalLM.from_pretrained("hf-internal-testing/tiny-random-acme")
-    path = hf_hub_download(repo_id="jdoe/acme-test-images", filename="cats.jpg", repo_type="dataset")
+    path = hf_hub_download(
+        repo_id="hf-internal-testing/fixtures_acme", filename="cats.jpg", repo_type="dataset"
+    )
'''

[rules.TRF067]
description = "A causal-LM model test should build on CausalLMModelTest, not hand-rolled mixins."
default_enabled = true
allowlist_models = []
cutoff_date = "2026-09-01"

[rules.TRF067.explanation]
what_it_does = "When the model directory defines a <Model>ForCausalLM, flags a collected test class in test_modeling_*.py that mixes ModelTesterMixin in directly instead of inheriting CausalLMModelTest. Reads the companion modeling_*.py to decide whether the model is a causal LM; a test class whose bases cannot be resolved is inconclusive and is skipped. Encoder-decoder and standalone-decoder test classes in a file that also has a causal-LM class are not the target -- the rule reports the class that stands in for the model's main suite."
why_bad = "CausalLMModelTest is the recommended base in the testing guide, and it carries TrainingTesterMixin and TensorParallelTesterMixin on top of the three shared mixins. A hand-assembled class silently ships without training and tensor-parallel coverage, and it stops inheriting whatever the shared base gains next -- which is how the same model ends up with a hundred fewer tests than the model merged beside it."
diff = '''
-class AcmeModelTest(ModelTesterMixin, GenerationTesterMixin, PipelineTesterMixin, unittest.TestCase):
-    all_model_classes = (AcmeModel, AcmeForCausalLM)
-
-    def setUp(self):
-        self.model_tester = AcmeModelTester(self)
+class AcmeModelTester(CausalLMModelTester):
+    if is_torch_available():
+        base_model_class = AcmeModel
+
+
+class AcmeModelTest(CausalLMModelTest, unittest.TestCase):
+    model_tester_class = AcmeModelTester
'''

[rules.TRF070]
description = "A remote test fixture must be fetched from the Hub, not a third-party URL."
default_enabled = true
allowlist_models = []
cutoff_date = "2026-09-01"

[rules.TRF070.explanation]
what_it_does = "Flags an `http(s)://` string literal in a tests/models test file whose host is not huggingface.co or hf.co -- the images, videos, audio clips and PDFs that tests download as inputs. Hub repo ids passed to from_pretrained(), load_dataset() or hf_hub_download() are not URLs and are unaffected, so the fix is always to upload the asset to a dataset repo (`hf-internal-testing/fixtures_*`, as test_modeling_pix2struct.py already does with fixtures_docvqa) and reference it by id."
why_bad = "202 fixture URLs across 90 test files point at hosts nobody here controls: images.cocodataset.org, cdn.britannica.com, thumbs.dreamstime.com, a personal file in a GitHub repo, and picsum.photos, which returns a different random image on every request. Each one is a way for a model's integration test to break, or to quietly start testing a different input, because of a change made by a stranger -- and the expected logits in the test were recorded against the old asset. A fixture on the Hub is as permanent and as reviewable as the test that uses it."
diff = '''
-url = "http://images.cocodataset.org/val2017/000000039769.jpg"
-image = Image.open(requests.get(url, stream=True).raw)
+path = hf_hub_download(
+    repo_id="hf-internal-testing/fixtures_acme", filename="cats.jpg", repo_type="dataset"
+)
+image = Image.open(path)
'''