[Styling] stylify using ruff (#27144)

* try to stylify using ruff

* might need to remove these changes?

* use ruf format andruff check

* use isinstance instead of type comparision

* use # fmt: skip

* use # fmt: skip

* nits

* soem styling changes

* update ci job

* nits isinstance

* more files update

* nits

* more nits

* small nits

* check and format

* revert wrong changes

* actually use formatter instead of checker

* nits

* well docbuilder is overwriting this commit

* revert notebook changes

* try to nuke docbuilder

* style

* fix feature exrtaction test

* remve `indent-width = 4`

* fixup

* more nits

* update the ruff version that we use

* style

* nuke docbuilder styling

* leve the print for detected changes

* nits

* Remove file I/O

Co-authored-by: charliermarsh
 <[email protected]>

* style

* nits

* revert notebook changes

* Add # fmt skip when possible

* Add # fmt skip when possible

* Fix

* More `  # fmt: skip` usage

* More `  # fmt: skip` usage

* More `  # fmt: skip` usage

* NIts

* more fixes

* fix tapas

* Another way to skip

* Recommended way

* Fix two more fiels

* Remove asynch
Remove asynch

---------

Co-authored-by: charliermarsh <[email protected]>

Author: ArthurZucker

.circleci/config.yml

@@ -157,11 +157,10 @@ jobs:
                 command: pip freeze | tee installed.txt
             - store_artifacts:
                   path: ~/transformers/installed.txt
-            - run: black --check examples tests src utils
-            - run: ruff examples tests src utils
+            - run: ruff check examples tests src utils
+            - run: ruff format tests src utils --check
             - run: python utils/custom_init_isort.py --check_only
             - run: python utils/sort_auto_mappings.py --check_only
-            - run: doc-builder style src/transformers docs/source --max_len 119 --check_only --path_to_docs docs/source
             - run: python utils/check_doc_toc.py
 
     check_repository_consistency:

.circleci/create_circleci_config.py

@@ -15,7 +15,6 @@
 
 import argparse
 import copy
-import glob
 import os
 import random
 from dataclasses import dataclass
@@ -239,7 +238,7 @@ def to_dict(self):
 
         py_command = f'import os; fp = open("reports/{self.job_name}/summary_short.txt"); failed = os.linesep.join([x for x in fp.read().split(os.linesep) if x.startswith("ERROR ")]); fp.close(); fp = open("summary_short.txt", "w"); fp.write(failed); fp.close()'
         check_test_command += f"$(python3 -c '{py_command}'); "
-        check_test_command += f'cat summary_short.txt; echo ""; exit -1; '
+        check_test_command += 'cat summary_short.txt; echo ""; exit -1; '
 
         # Deeal with failed tests
         check_test_command += f'elif [ -s reports/{self.job_name}/failures_short.txt ]; '
@@ -249,7 +248,7 @@ def to_dict(self):
 
         py_command = f'import os; fp = open("reports/{self.job_name}/summary_short.txt"); failed = os.linesep.join([x for x in fp.read().split(os.linesep) if x.startswith("FAILED ")]); fp.close(); fp = open("summary_short.txt", "w"); fp.write(failed); fp.close()'
         check_test_command += f"$(python3 -c '{py_command}'); "
-        check_test_command += f'cat summary_short.txt; echo ""; exit -1; '
+        check_test_command += 'cat summary_short.txt; echo ""; exit -1; '
 
         check_test_command += f'elif [ -s reports/{self.job_name}/stats.txt ]; then echo "All tests pass!"; '
 

Makefile

@@ -9,8 +9,8 @@ modified_only_fixup:
 	$(eval modified_py_files := $(shell python utils/get_modified_files.py $(check_dirs)))
 	@if test -n "$(modified_py_files)"; then \
 		echo "Checking/fixing $(modified_py_files)"; \
-		black $(modified_py_files); \
-		ruff $(modified_py_files) --fix; \
+		ruff check $(modified_py_files) --fix; \
+		ruff format $(modified_py_files);\
 	else \
 		echo "No library .py files were modified"; \
 	fi
@@ -48,26 +48,24 @@ repo-consistency:
 # this target runs checks on all files
 
 quality:
-	black --check $(check_dirs) setup.py conftest.py
+	ruff check $(check_dirs) setup.py conftest.py
+	ruff format --check $(check_dirs) setup.py conftest.py
 	python utils/custom_init_isort.py --check_only
 	python utils/sort_auto_mappings.py --check_only
-	ruff $(check_dirs) setup.py conftest.py
-	doc-builder style src/transformers docs/source --max_len 119 --check_only --path_to_docs docs/source
 	python utils/check_doc_toc.py
 
 # Format source code automatically and check is there are any problems left that need manual fixing
 
 extra_style_checks:
 	python utils/custom_init_isort.py
 	python utils/sort_auto_mappings.py
-	doc-builder style src/transformers docs/source --max_len 119 --path_to_docs docs/source
 	python utils/check_doc_toc.py --fix_and_overwrite
 
 # this target runs checks on all files and potentially modifies some of them
 
 style:
-	black $(check_dirs) setup.py conftest.py
-	ruff $(check_dirs) setup.py conftest.py --fix
+	ruff check $(check_dirs) setup.py conftest.py --fix
+	ruff format $(check_dirs) setup.py conftest.py
 	${MAKE} autogenerate_code
 	${MAKE} extra_style_checks
 

docs/source/_config.py

@@ -10,5 +10,5 @@
 black_avoid_patterns = {
     "{processor_class}": "FakeProcessorClass",
     "{model_class}": "FakeModelClass",
-    "{object_class}": "FakeObjectClass",    
+    "{object_class}": "FakeObjectClass",
 }

docs/source/en/_config.py

@@ -10,5 +10,5 @@
 black_avoid_patterns = {
     "{processor_class}": "FakeProcessorClass",
     "{model_class}": "FakeModelClass",
-    "{object_class}": "FakeObjectClass",    
+    "{object_class}": "FakeObjectClass",
 }

docs/source/en/tasks/semantic_segmentation.md

@@ -245,7 +245,7 @@ logits first, and then reshaped to match the size of the labels before you can c
 ...             reduce_labels=False,
 ...         )
 ...         for key, value in metrics.items():
-...             if type(value) is np.ndarray:
+...             if isinstance(value, np.ndarray):
 ...                 metrics[key] = value.tolist()
 ...         return metrics
 ```

docs/source/ko/_config.py

@@ -10,5 +10,5 @@
 black_avoid_patterns = {
     "{processor_class}": "FakeProcessorClass",
     "{model_class}": "FakeModelClass",
-    "{object_class}": "FakeObjectClass",    
+    "{object_class}": "FakeObjectClass",
 }

docs/source/ko/tasks/semantic_segmentation.md

@@ -242,7 +242,7 @@ pip install -q datasets transformers evaluate
 ...             reduce_labels=False,
 ...         )
 ...         for key, value in metrics.items():
-...             if type(value) is np.ndarray:
+...             if isinstance(value, np.ndarray):
 ...                 metrics[key] = value.tolist()
 ...         return metrics
 ```

docs/source/pt/_config.py

@@ -10,5 +10,5 @@
 black_avoid_patterns = {
     "{processor_class}": "FakeProcessorClass",
     "{model_class}": "FakeModelClass",
-    "{object_class}": "FakeObjectClass",    
+    "{object_class}": "FakeObjectClass",
 }

examples/flax/text-classification/run_flax_glue.py

@@ -212,7 +212,7 @@ def __post_init__(self):
             if self.validation_file is not None:
                 extension = self.validation_file.split(".")[-1]
                 assert extension in ["csv", "json"], "`validation_file` should be a csv or a json file."
-        self.task_name = self.task_name.lower() if type(self.task_name) == str else self.task_name
+        self.task_name = self.task_name.lower() if isinstance(self.task_name, str) else self.task_name
 
 
 def create_train_state(

examples/legacy/pytorch-lightning/run_glue.py

@@ -23,7 +23,7 @@ class GLUETransformer(BaseTransformer):
     mode = "sequence-classification"
 
     def __init__(self, hparams):
-        if type(hparams) == dict:
+        if isinstance(hparams, dict):
             hparams = Namespace(**hparams)
         hparams.glue_output_mode = glue_output_modes[hparams.task]
         num_labels = glue_tasks_num_labels[hparams.task]

examples/legacy/pytorch-lightning/run_ner.py

@@ -25,7 +25,7 @@ class NERTransformer(BaseTransformer):
     mode = "token-classification"
 
     def __init__(self, hparams):
-        if type(hparams) == dict:
+        if isinstance(hparams, dict):
             hparams = Namespace(**hparams)
         module = import_module("tasks")
         try:

examples/research_projects/deebert/src/modeling_highway_bert.py

@@ -32,7 +32,7 @@ def __init__(self, config):
         self.early_exit_entropy = [-1 for _ in range(config.num_hidden_layers)]
 
     def set_early_exit_entropy(self, x):
-        if (type(x) is float) or (type(x) is int):
+        if isinstance(x, (float, int)):
             for i in range(len(self.early_exit_entropy)):
                 self.early_exit_entropy[i] = x
         else:
@@ -232,9 +232,7 @@ def forward(
         outputs = (
             sequence_output,
             pooled_output,
-        ) + encoder_outputs[
-            1:
-        ]  # add hidden_states and attentions if they are here
+        ) + encoder_outputs[1:]  # add hidden_states and attentions if they are here
         return outputs  # sequence_output, pooled_output, (hidden_states), (attentions), highway exits
 
 

examples/research_projects/longform-qa/eli5_app.py

@@ -158,9 +158,7 @@ def answer_question(
     </span>
   </body>
 </html>
-""" % (
-    header_html,
-)
+""" % (header_html,)
 st.sidebar.markdown(
     header_full,
     unsafe_allow_html=True,

examples/research_projects/lxmert/modeling_frcnn.py

@@ -1706,9 +1706,7 @@ def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs):
             elif os.path.isfile(pretrained_model_name_or_path) or is_remote_url(pretrained_model_name_or_path):
                 archive_file = pretrained_model_name_or_path
             elif os.path.isfile(pretrained_model_name_or_path + ".index"):
-                assert (
-                    from_tf
-                ), "We found a TensorFlow checkpoint at {}, please set from_tf to True to load from this checkpoint".format(
+                assert from_tf, "We found a TensorFlow checkpoint at {}, please set from_tf to True to load from this checkpoint".format(
                     pretrained_model_name_or_path + ".index"
                 )
                 archive_file = pretrained_model_name_or_path + ".index"

examples/research_projects/movement-pruning/emmental/modeling_bert_masked.py

@@ -652,9 +652,7 @@ def forward(
         outputs = (
             sequence_output,
             pooled_output,
-        ) + encoder_outputs[
-            1:
-        ]  # add hidden_states and attentions if they are here
+        ) + encoder_outputs[1:]  # add hidden_states and attentions if they are here
         return outputs  # sequence_output, pooled_output, (hidden_states), (attentions)
 
 

examples/research_projects/movement-pruning/masked_run_glue.py

@@ -311,8 +311,7 @@ def train(args, train_dataset, model, tokenizer, teacher=None):
             tr_loss += loss.item()
             if (step + 1) % args.gradient_accumulation_steps == 0 or (
                 # last step in epoch but step is always smaller than gradient_accumulation_steps
-                len(epoch_iterator) <= args.gradient_accumulation_steps
-                and (step + 1) == len(epoch_iterator)
+                len(epoch_iterator) <= args.gradient_accumulation_steps and (step + 1) == len(epoch_iterator)
             ):
                 if args.fp16:
                     nn.utils.clip_grad_norm_(amp.master_params(optimizer), args.max_grad_norm)

examples/research_projects/quantization-qdqbert/quant_trainer.py

@@ -239,7 +239,7 @@ def print_model_summary(model, name_width=25, line_width=180, ignore=None):
             continue
         if type(mod) in ignore:
             continue
-        if [True for s in ignore if type(s) is str and s in name]:
+        if [True for s in ignore if isinstance(s, str) and s in name]:
             continue
         act_str = f"Act:{input_q.extra_repr()}"
         wgt_str = f"Wgt:{weight_q.extra_repr()}"

examples/research_projects/visual_bert/modeling_frcnn.py

@@ -1706,9 +1706,7 @@ def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs):
             elif os.path.isfile(pretrained_model_name_or_path) or is_remote_url(pretrained_model_name_or_path):
                 archive_file = pretrained_model_name_or_path
             elif os.path.isfile(pretrained_model_name_or_path + ".index"):
-                assert (
-                    from_tf
-                ), "We found a TensorFlow checkpoint at {}, please set from_tf to True to load from this checkpoint".format(
+                assert from_tf, "We found a TensorFlow checkpoint at {}, please set from_tf to True to load from this checkpoint".format(
                     pretrained_model_name_or_path + ".index"
                 )
                 archive_file = pretrained_model_name_or_path + ".index"

hubconf.py

@@ -15,6 +15,7 @@
 import os
 import sys
 
+
 SRC_DIR = os.path.join(os.path.dirname(__file__), "src")
 sys.path.append(SRC_DIR)
 

pyproject.toml

@@ -1,10 +1,6 @@
-[tool.black]
-line-length = 119
-target-version = ['py37']
-
 [tool.ruff]
 # Never enforce `E501` (line length violations).
-ignore = ["C901", "E501", "E741"]
+ignore = ["C901", "E501", "E741", "F402", "F823" ]
 select = ["C", "E", "F", "I", "W"]
 line-length = 119
 
@@ -18,6 +14,19 @@ line-length = 119
 lines-after-imports = 2
 known-first-party = ["transformers"]
 
+[tool.ruff.format]
+# Like Black, use double quotes for strings.
+quote-style = "double"
+
+# Like Black, indent with spaces, rather than tabs.
+indent-style = "space"
+
+# Like Black, respect magic trailing commas.
+skip-magic-trailing-comma = false
+
+# Like Black, automatically detect the appropriate line ending.
+line-ending = "auto"
+
 [tool.pytest.ini_options]
 doctest_optionflags="NUMBER NORMALIZE_WHITESPACE ELLIPSIS"
 doctest_glob="**/*.md"

scripts/check_tokenizers.py

@@ -1,10 +1,12 @@
 from collections import Counter
+
 import datasets
+
 import transformers
 from transformers.convert_slow_tokenizer import SLOW_TO_FAST_CONVERTERS
-
 from transformers.utils import logging
 
+
 logging.set_verbosity_info()
 
 TOKENIZER_CLASSES = {
@@ -101,8 +103,8 @@ def check_details(line, spm_ids, tok_ids, slow, fast):
     except Exception:
         pass
 
-    ok_start = fast.decode(spm_ids[:first])
-    ok_end = fast.decode(spm_ids[last:])
+    fast.decode(spm_ids[:first])
+    fast.decode(spm_ids[last:])
     wrong = fast.decode(spm_ids[first:last])
     print()
     print(wrong)

scripts/fsmt/fsmt-make-super-tiny-model.py

@@ -24,18 +24,19 @@
 #
 # It will be used then as "stas/tiny-wmt19-en-ru"
 
-from pathlib import Path
 import json
 import tempfile
+from pathlib import Path
 
-from transformers import FSMTTokenizer, FSMTConfig, FSMTForConditionalGeneration
+from transformers import FSMTConfig, FSMTForConditionalGeneration, FSMTTokenizer
 from transformers.models.fsmt.tokenization_fsmt import VOCAB_FILES_NAMES
 
+
 mname_tiny = "tiny-wmt19-en-ru"
 
 # Build
 
-# borrowed from a test 
+# borrowed from a test
 vocab = [ "l", "o", "w", "e", "r", "s", "t", "i", "d", "n", "w</w>", "r</w>", "t</w>", "lo", "low", "er</w>", "low</w>", "lowest</w>", "newer</w>", "wider</w>", "<unk>", ]
 vocab_tokens = dict(zip(vocab, range(len(vocab))))
 merges = ["l o 123", "lo w 1456", "e r</w> 1789", ""]
@@ -57,7 +58,7 @@
         tgt_vocab_file=tgt_vocab_file,
         merges_file=merges_file,
     )
-    
+
 config = FSMTConfig(
     langs=['ru', 'en'],
     src_vocab_size=1000, tgt_vocab_size=1000,

scripts/fsmt/fsmt-make-tiny-model.py

@@ -27,16 +27,18 @@
 # It will be used then as "stas/tiny-wmt19-en-de"
 
 # Build
-from transformers import FSMTTokenizer, FSMTConfig, FSMTForConditionalGeneration
+from transformers import FSMTConfig, FSMTForConditionalGeneration, FSMTTokenizer
+
+
 mname = "facebook/wmt19-en-de"
 tokenizer = FSMTTokenizer.from_pretrained(mname)
 # get the correct vocab sizes, etc. from the master model
 config = FSMTConfig.from_pretrained(mname)
-config.update(dict(
-    d_model=4,
-    encoder_layers=1, decoder_layers=1,
-    encoder_ffn_dim=4, decoder_ffn_dim=4,
-    encoder_attention_heads=1, decoder_attention_heads=1))
+config.update({
+    "d_model": 4,
+    "encoder_layers": 1, "decoder_layers": 1,
+    "encoder_ffn_dim": 4, "decoder_ffn_dim": 4,
+    "encoder_attention_heads": 1, "decoder_attention_heads": 1})
 
 tiny_model = FSMTForConditionalGeneration(config)
 print(f"num of params {tiny_model.num_parameters()}")

scripts/fsmt/gen-card-allenai-wmt16.py

@@ -19,6 +19,7 @@
 import os
 from pathlib import Path
 
+
 def write_model_card(model_card_dir, src_lang, tgt_lang, model_name):
 
     texts = {

scripts/fsmt/gen-card-allenai-wmt19.py

@@ -19,6 +19,7 @@
 import os
 from pathlib import Path
 
+
 def write_model_card(model_card_dir, src_lang, tgt_lang, model_name):
 
     texts = {

scripts/fsmt/gen-card-facebook-wmt19.py

@@ -19,6 +19,7 @@
 import os
 from pathlib import Path
 
+
 def write_model_card(model_card_dir, src_lang, tgt_lang):
 
     texts = {
@@ -39,7 +40,7 @@ def write_model_card(model_card_dir, src_lang, tgt_lang):
 
     readme = f"""
 ---
-language: 
+language:
 - {src_lang}
 - {tgt_lang}
 thumbnail: