Skip to content

Commit 70c85a7

Browse files
clee2000clee2000
authored
[CI][lint] Add lintrunner + spaces, tabs, newlines linters (#3467)
* Add lint job for lintrunner * Add lintrunner.toml * Add newlines, spaces, tabs lines * These download the linter from pytorch/pytorch. The sha is main branch as of now. Not sure if I should do this or just copy the linters over * Big exclusion lists because I don't want to make a ton of changes in 1 PR at once new lines checks that files end with just 1 `\n` spaces checks that there are no trailing whitespace on lines tabs checks that spaces are used instead of tabs
1 parent 755434d commit 70c85a7

File tree

4 files changed

+368
-0
lines changed

4 files changed

+368
-0
lines changed

‎.github/workflows/lintrunner.yml‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
name: Lintrunner
2+
3+
on:
4+
push:
5+
branches:
6+
- main
7+
pull_request:
8+
9+
concurrency:
10+
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}-${{ github.event_name == 'workflow_dispatch' }}
11+
cancel-in-progress: true
12+
13+
jobs:
14+
lintrunner:
15+
name: lintrunner
16+
runs-on: ubuntu-latest
17+
steps:
18+
- name: Checkout tutorials
19+
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
20+
21+
- name: Setup Python
22+
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
23+
with:
24+
python-version: '3.12'
25+
26+
- name: Install Lintrunner
27+
run: |
28+
pip install lintrunner==0.12.5
29+
lintrunner init
30+
31+
- name: Run lintrunner on all files - Linux
32+
run: |
33+
set +e
34+
if ! lintrunner -v --force-color --all-files --tee-json=lint.json; then
35+
echo ""
36+
echo -e "\e[1m\e[36mYou can reproduce these results locally by using \`lintrunner -m main\`.\e[0m"
37+
exit 1
38+
fi

‎.gitignore‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -127,3 +127,6 @@ cleanup.sh
127127

128128
# pyspelling
129129
dictionary.dic
130+
131+
# linters
132+
/.lintbin

‎.lintrunner.toml‎

Lines changed: 246 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,246 @@
1+
merge_base_with = "origin/main"
2+
3+
# 4805a6ead6f1e7f32351056e2602be4e908f69b7 is from pytorch/pytorch main branch 2025-07-16
4+
5+
[[linter]]
6+
code = 'SPACES'
7+
include_patterns = ['**']
8+
exclude_patterns = [
9+
"_static/**/*", # Contains some files that should usually not be linted
10+
# All files below this should be checked and either removed from the
11+
# exclusion list by fixing them or have a reason to be excluded.
12+
".github/ISSUE_TEMPLATE/bug-report.yml",
13+
".github/ISSUE_TEMPLATE/feature-request.yml",
14+
".github/scripts/docathon-label-sync.py",
15+
".github/workflows/MonthlyLinkCheck.yml",
16+
".github/workflows/StalePRs.yml",
17+
".github/workflows/link_checkPR.yml",
18+
".github/workflows/spelling.yml",
19+
".jenkins/post_process_notebooks.py",
20+
".lycheeignore",
21+
"CONTRIBUTING.md",
22+
"advanced_source/coding_ddpg.py",
23+
"advanced_source/cpp_autograd.rst",
24+
"advanced_source/cpp_custom_ops.rst",
25+
"advanced_source/generic_join.rst",
26+
"advanced_source/neural_style_tutorial.py",
27+
"advanced_source/pendulum.py",
28+
"advanced_source/privateuseone.rst",
29+
"advanced_source/semi_structured_sparse.py",
30+
"advanced_source/sharding.rst",
31+
"advanced_source/static_quantization_tutorial.rst",
32+
"advanced_source/torch_script_custom_classes/custom_class_project/custom_test.py",
33+
"advanced_source/transformer__timeseries_cpp_tutorial/transformer_timeseries.cpp",
34+
"advanced_source/usb_semisup_learn.py",
35+
"beginner_source/blitz/README.txt",
36+
"beginner_source/blitz/neural_networks_tutorial.py",
37+
"beginner_source/dcgan_faces_tutorial.py",
38+
"beginner_source/ddp_series_fault_tolerance.rst",
39+
"beginner_source/ddp_series_theory.rst",
40+
"beginner_source/examples_nn/polynomial_module.py",
41+
"beginner_source/examples_nn/polynomial_nn.py",
42+
"beginner_source/hta_intro_tutorial.rst",
43+
"beginner_source/hta_trace_diff_tutorial.rst",
44+
"beginner_source/hybrid_frontend/README.txt",
45+
"beginner_source/hybrid_frontend_tutorial.rst",
46+
"beginner_source/hyperparameter_tuning_tutorial.py",
47+
"beginner_source/introyt/README.txt",
48+
"beginner_source/introyt/autogradyt_tutorial.py",
49+
"beginner_source/introyt/captumyt.py",
50+
"beginner_source/introyt/introyt1_tutorial.py",
51+
"beginner_source/introyt/modelsyt_tutorial.py",
52+
"beginner_source/introyt/tensorboardyt_tutorial.py",
53+
"beginner_source/introyt/tensors_deeper_tutorial.py",
54+
"beginner_source/introyt/trainingyt.py",
55+
"beginner_source/knowledge_distillation_tutorial.py",
56+
"beginner_source/nlp/sequence_models_tutorial.py",
57+
"beginner_source/onnx/export_control_flow_model_to_onnx_tutorial.py",
58+
"beginner_source/onnx/onnx_registry_tutorial.py",
59+
"beginner_source/pytorch_with_examples.rst",
60+
"beginner_source/saving_loading_models.py",
61+
"beginner_source/template_tutorial.py",
62+
"beginner_source/transfer_learning_tutorial.py",
63+
"docathon-leaderboard.md",
64+
"intermediate_source/TCPStore_libuv_backend.rst",
65+
"intermediate_source/ax_multiobjective_nas_tutorial.py",
66+
"intermediate_source/compiled_autograd_tutorial.rst",
67+
"intermediate_source/ddp_series_multinode.rst",
68+
"intermediate_source/dqn_with_rnn_tutorial.py",
69+
"intermediate_source/fx_profiling_tutorial.py",
70+
"intermediate_source/inductor_debug_cpu.py",
71+
"intermediate_source/jacobians_hessians.py",
72+
"intermediate_source/optimizer_step_in_backward_tutorial.py",
73+
"intermediate_source/per_sample_grads.py",
74+
"intermediate_source/pruning_tutorial.py",
75+
"intermediate_source/reinforcement_q_learning.py",
76+
"intermediate_source/tensorboard_profiler_tutorial.py",
77+
"intermediate_source/tiatoolbox_tutorial.rst",
78+
"intermediate_source/torch_compile_tutorial.py",
79+
"intermediate_source/transformer_building_blocks.py",
80+
"prototype_source/README.md",
81+
"prototype_source/README.txt",
82+
"prototype_source/backend_config_tutorial.rst",
83+
"prototype_source/gpu_direct_storage.py",
84+
"prototype_source/inductor_cpp_wrapper_tutorial.rst",
85+
"prototype_source/inductor_windows.rst",
86+
"prototype_source/maskedtensor_advanced_semantics.py",
87+
"prototype_source/max_autotune_on_CPU_tutorial.rst",
88+
"prototype_source/vmap_recipe.py",
89+
"recipes_source/README.txt",
90+
"recipes_source/amx.rst",
91+
"recipes_source/compiling_optimizer.rst",
92+
"recipes_source/compiling_optimizer_lr_scheduler.py",
93+
"recipes_source/distributed_optim_torchscript.rst",
94+
"recipes_source/foreach_map.py",
95+
"recipes_source/profile_with_itt.rst",
96+
"recipes_source/recipes/Captum_Recipe.py",
97+
"recipes_source/recipes/benchmark.py",
98+
"recipes_source/recipes/changing_default_device.py",
99+
"recipes_source/recipes/defining_a_neural_network.py",
100+
"recipes_source/recipes/tensorboard_with_pytorch.py",
101+
"recipes_source/recipes/timer_quick_start.py",
102+
"recipes_source/recipes/tuning_guide.py",
103+
"recipes_source/recipes/warmstarting_model_using_parameters_from_a_different_model.py",
104+
"recipes_source/recipes/what_is_state_dict.py",
105+
"recipes_source/torch_compile_caching_tutorial.rst",
106+
"recipes_source/torch_compile_torch_function_modes.py",
107+
"recipes_source/torch_compile_user_defined_triton_kernel_tutorial.py",
108+
"recipes_source/torch_compiler_set_stance_tutorial.py",
109+
"recipes_source/torch_export_aoti_python.py",
110+
"recipes_source/xeon_run_cpu.rst",
111+
"redirects.py",
112+
"tutorial_submission_policy.md",
113+
".jenkins/build.sh",
114+
"advanced_source/cpp_export.rst",
115+
"advanced_source/torch-script-parallelism.rst",
116+
"advanced_source/torch_script_custom_classes.rst",
117+
"advanced_source/torch_script_custom_ops.rst",
118+
"recipes_source/torchscript_inference.rst",
119+
]
120+
init_command = [
121+
'python3',
122+
'tools/linter/adapters/run_from_link.py',
123+
'--lint-name=grep_linter.py',
124+
'--lint-link=https://raw.githubusercontent.com/pytorch/pytorch/4805a6ead6f1e7f32351056e2602be4e908f69b7/tools/linter/adapters/grep_linter.py',
125+
'--',
126+
'--dry-run={{DRYRUN}}',
127+
]
128+
command = [
129+
'python3',
130+
'tools/linter/adapters/run_from_link.py',
131+
'--run-lint',
132+
'--lint-name=grep_linter.py',
133+
'--',
134+
'--pattern=[[:blank:]]$',
135+
'--linter-name=SPACES',
136+
'--error-name=trailing spaces',
137+
'--replace-pattern=s/[[:blank:]]+$//',
138+
"""--error-description=\
139+
This line has trailing spaces; please remove them.\
140+
""",
141+
'--',
142+
'@{{PATHSFILE}}'
143+
]
144+
145+
[[linter]]
146+
code = 'TABS'
147+
include_patterns = ['**']
148+
exclude_patterns = [
149+
"_static/**/*", # Contains some files that should usually not be linted
150+
".lintrunner.toml", # Ironically needs to contain the tab character to find in other files
151+
# All files below this should be checked and either removed from the
152+
# exclusion list by fixing them or have a reason to be excluded.
153+
"Makefile",
154+
"advanced_source/README.txt",
155+
"advanced_source/cpp_frontend.rst",
156+
"advanced_source/torch_script_custom_ops.rst",
157+
"beginner_source/README.txt",
158+
"beginner_source/basics/tensorqs_tutorial.py",
159+
"beginner_source/blitz/README.txt",
160+
"beginner_source/blitz/tensor_tutorial.py",
161+
"beginner_source/hybrid_frontend/README.txt",
162+
"beginner_source/nlp/README.txt",
163+
"beginner_source/nlp/pytorch_tutorial.py",
164+
"intermediate_source/README.txt",
165+
"intermediate_source/TP_tutorial.rst",
166+
"intermediate_source/inductor_debug_cpu.py",
167+
"prototype_source/README.txt",
168+
"recipes_source/README.txt",
169+
"recipes_source/recipes/README.txt",
170+
"recipes_source/xeon_run_cpu.rst",
171+
]
172+
init_command = [
173+
'python3',
174+
'tools/linter/adapters/run_from_link.py',
175+
'--lint-name=grep_linter.py',
176+
'--lint-link=https://raw.githubusercontent.com/pytorch/pytorch/4805a6ead6f1e7f32351056e2602be4e908f69b7/tools/linter/adapters/grep_linter.py',
177+
'--',
178+
'--dry-run={{DRYRUN}}',
179+
]
180+
command = [
181+
'python3',
182+
'tools/linter/adapters/run_from_link.py',
183+
'--run-lint',
184+
'--lint-name=grep_linter.py',
185+
'--',
186+
# @lint-ignore TXT2
187+
'--pattern=',
188+
'--linter-name=TABS',
189+
'--error-name=saw some tabs',
190+
'--replace-pattern=s/\t/ /',
191+
"""--error-description=\
192+
This line has tabs; please replace them with spaces.\
193+
""",
194+
'--',
195+
'@{{PATHSFILE}}'
196+
]
197+
198+
[[linter]]
199+
code = 'NEWLINE'
200+
include_patterns=['**']
201+
exclude_patterns=[
202+
"_static/**/*", # Contains some files that should usually not be linted
203+
# All files below this should be checked and either removed from the
204+
# exclusion list by fixing them or have a reason to be excluded.
205+
".github/workflows/StalePRs.yml",
206+
"CONTRIBUTING.md",
207+
"advanced_source/extend_dispatcher.rst",
208+
"advanced_source/neural_style_tutorial.py",
209+
"advanced_source/sharding.rst",
210+
"advanced_source/torch_script_custom_classes/custom_class_project/custom_test.py",
211+
"advanced_source/transformer__timeseries_cpp_tutorial/transformer_timeseries.cpp",
212+
"beginner_source/blitz/README.txt",
213+
"beginner_source/dcgan_faces_tutorial.py",
214+
"beginner_source/hta_trace_diff_tutorial.rst",
215+
"beginner_source/hybrid_frontend/README.txt",
216+
"beginner_source/nlp/pytorch_tutorial.py",
217+
"beginner_source/template_tutorial.py",
218+
"beginner_source/transfer_learning_tutorial.py",
219+
"intermediate_source/custom_function_conv_bn_tutorial.py",
220+
"intermediate_source/custom_function_double_backward_tutorial.rst",
221+
"intermediate_source/forced_alignment_with_torchaudio_tutorial.rst",
222+
"intermediate_source/nlp_from_scratch_index.rst",
223+
"intermediate_source/pipeline_tutorial.rst",
224+
"intermediate_source/tiatoolbox_tutorial.rst",
225+
"recipes_source/README.txt",
226+
"recipes_source/script_optimized.rst",
227+
"recipes_source/torch_compile_caching_configuration_tutorial.rst",
228+
"recipes_source/torch_compile_caching_tutorial.rst",
229+
]
230+
init_command = [
231+
'python3',
232+
'tools/linter/adapters/run_from_link.py',
233+
'--lint-name=newlines_linter.py',
234+
'--lint-link=https://raw.githubusercontent.com/pytorch/pytorch/4805a6ead6f1e7f32351056e2602be4e908f69b7/tools/linter/adapters/newlines_linter.py',
235+
'--',
236+
'--dry-run={{DRYRUN}}',
237+
]
238+
command = [
239+
'python3',
240+
'tools/linter/adapters/run_from_link.py',
241+
'--run-lint',
242+
'--lint-name=newlines_linter.py',
243+
'--',
244+
'@{{PATHSFILE}}',
245+
]
246+
is_formatter = true

‎tools/linter/adapters/run_from_link.py‎

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
import argparse
2+
import subprocess
3+
import urllib.request
4+
from pathlib import Path
5+
6+
7+
REPO_ROOT = Path(__file__).absolute().parents[3]
8+
9+
10+
def parse_args() -> argparse.Namespace:
11+
parser = argparse.ArgumentParser(
12+
description="Use a formatter in a different repository.",
13+
)
14+
parser.add_argument(
15+
"--run-init",
16+
action="store_true",
17+
help="Run the initialization script specified by --init-name.",
18+
)
19+
parser.add_argument(
20+
"--run-lint",
21+
action="store_true",
22+
help="Run the linting script specified by --lint-name.",
23+
)
24+
parser.add_argument(
25+
"--init-name",
26+
help="Name of the initialization script. This also serves as the filename.",
27+
)
28+
parser.add_argument(
29+
"--init-link",
30+
help="URL to download the initialization script from.",
31+
)
32+
parser.add_argument(
33+
"--lint-name",
34+
help="Name of the linting script. This also serves as the filename.",
35+
)
36+
parser.add_argument(
37+
"--lint-link",
38+
help="URL to download the linting script from.",
39+
)
40+
41+
parser.add_argument("args_for_file", nargs=argparse.REMAINDER)
42+
args = parser.parse_args()
43+
# Skip the first -- if present
44+
if args.args_for_file and args.args_for_file[0] == "--":
45+
args.args_for_file = args.args_for_file[1:]
46+
return args
47+
48+
49+
def download_file(url: str, location: Path) -> bytes:
50+
response = urllib.request.urlopen(url)
51+
content = response.read()
52+
location.write_bytes(content)
53+
return content
54+
55+
56+
def main() -> None:
57+
args = parse_args()
58+
59+
location = REPO_ROOT / ".lintbin" / "from_link" / "adapters"
60+
location.mkdir(parents=True, exist_ok=True)
61+
62+
if args.lint_link:
63+
download_file(args.lint_link, location / args.lint_name)
64+
65+
if args.init_link:
66+
download_file(args.init_link, location / args.init_name)
67+
68+
if args.run_init:
69+
# Save the content to a file named after the name argument
70+
subprocess_args = ["python3", location / args.init_name] + args.args_for_file
71+
subprocess.run(subprocess_args, check=True)
72+
if args.run_lint:
73+
subprocess_args = ["python3", location / args.lint_name] + args.args_for_file
74+
subprocess.run(
75+
subprocess_args,
76+
check=True,
77+
)
78+
79+
80+
if __name__ == "__main__":
81+
main()

0 commit comments

Comments
 (0)