From 5f75e7bd689b7de0c7b283694ce6cec3f01a9c6b Mon Sep 17 00:00:00 2001 From: Catherine Lee Date: Wed, 16 Jul 2025 10:05:47 -0700 Subject: [PATCH 1/3] tc --- .github/workflows/lintrunner.yml | 38 ++++ .gitignore | 3 + .lintrunner.toml | 239 +++++++++++++++++++++++++ tools/linter/adapters/run_from_link.py | 81 +++++++++ 4 files changed, 361 insertions(+) create mode 100644 .github/workflows/lintrunner.yml create mode 100644 .lintrunner.toml create mode 100644 tools/linter/adapters/run_from_link.py diff --git a/.github/workflows/lintrunner.yml b/.github/workflows/lintrunner.yml new file mode 100644 index 00000000000..e1a6889eb28 --- /dev/null +++ b/.github/workflows/lintrunner.yml @@ -0,0 +1,38 @@ +name: Lintrunner + +on: + push: + branches: + - main + pull_request: + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}-${{ github.event_name == 'workflow_dispatch' }} + cancel-in-progress: true + +jobs: + lintrunner: + name: lintrunner + runs-on: ubuntu-latest + steps: + - name: Checkout tutorials + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + + - name: Setup Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: '3.12' + + - name: Install Lintrunner + run: | + pip install lintrunner==0.12.5 + lintrunner init + + - name: Run lintrunner on all files - Linux + run: | + set +e + if ! lintrunner -v --force-color --all-files --tee-json=lint.json; then + echo "" + echo -e "\e[1m\e[36mYou can reproduce these results locally by using \`lintrunner -m main\`.\e[0m" + exit 1 + fi diff --git a/.gitignore b/.gitignore index 1d9d572e565..b84ca258cad 100644 --- a/.gitignore +++ b/.gitignore @@ -127,3 +127,6 @@ cleanup.sh # pyspelling dictionary.dic + +# linters +/.lintbin diff --git a/.lintrunner.toml b/.lintrunner.toml new file mode 100644 index 00000000000..110c1c1b702 --- /dev/null +++ b/.lintrunner.toml @@ -0,0 +1,239 @@ +merge_base_with = "origin/main" + + +[[linter]] +code = 'SPACES' +include_patterns = ['**'] +exclude_patterns = [ + "_static/**/*", # Contains some files that should usually not be linted + # All files below this should be checked and either removed from the + # exclusion list by fixing them or have a reason to be excluded. + ".github/ISSUE_TEMPLATE/bug-report.yml", + ".github/ISSUE_TEMPLATE/feature-request.yml", + ".github/scripts/docathon-label-sync.py", + ".github/workflows/MonthlyLinkCheck.yml", + ".github/workflows/StalePRs.yml", + ".github/workflows/link_checkPR.yml", + ".github/workflows/spelling.yml", + ".jenkins/post_process_notebooks.py", + ".lycheeignore", + "CONTRIBUTING.md", + "advanced_source/coding_ddpg.py", + "advanced_source/cpp_autograd.rst", + "advanced_source/cpp_custom_ops.rst", + "advanced_source/generic_join.rst", + "advanced_source/neural_style_tutorial.py", + "advanced_source/pendulum.py", + "advanced_source/privateuseone.rst", + "advanced_source/semi_structured_sparse.py", + "advanced_source/sharding.rst", + "advanced_source/static_quantization_tutorial.rst", + "advanced_source/torch_script_custom_classes/custom_class_project/custom_test.py", + "advanced_source/transformer__timeseries_cpp_tutorial/transformer_timeseries.cpp", + "advanced_source/usb_semisup_learn.py", + "beginner_source/blitz/README.txt", + "beginner_source/blitz/neural_networks_tutorial.py", + "beginner_source/dcgan_faces_tutorial.py", + "beginner_source/ddp_series_fault_tolerance.rst", + "beginner_source/ddp_series_theory.rst", + "beginner_source/examples_nn/polynomial_module.py", + "beginner_source/examples_nn/polynomial_nn.py", + "beginner_source/hta_intro_tutorial.rst", + "beginner_source/hta_trace_diff_tutorial.rst", + "beginner_source/hybrid_frontend/README.txt", + "beginner_source/hybrid_frontend_tutorial.rst", + "beginner_source/hyperparameter_tuning_tutorial.py", + "beginner_source/introyt/README.txt", + "beginner_source/introyt/autogradyt_tutorial.py", + "beginner_source/introyt/captumyt.py", + "beginner_source/introyt/introyt1_tutorial.py", + "beginner_source/introyt/modelsyt_tutorial.py", + "beginner_source/introyt/tensorboardyt_tutorial.py", + "beginner_source/introyt/tensors_deeper_tutorial.py", + "beginner_source/introyt/trainingyt.py", + "beginner_source/knowledge_distillation_tutorial.py", + "beginner_source/nlp/sequence_models_tutorial.py", + "beginner_source/onnx/export_control_flow_model_to_onnx_tutorial.py", + "beginner_source/onnx/onnx_registry_tutorial.py", + "beginner_source/pytorch_with_examples.rst", + "beginner_source/saving_loading_models.py", + "beginner_source/template_tutorial.py", + "beginner_source/transfer_learning_tutorial.py", + "docathon-leaderboard.md", + "intermediate_source/TCPStore_libuv_backend.rst", + "intermediate_source/ax_multiobjective_nas_tutorial.py", + "intermediate_source/compiled_autograd_tutorial.rst", + "intermediate_source/ddp_series_multinode.rst", + "intermediate_source/dqn_with_rnn_tutorial.py", + "intermediate_source/fx_profiling_tutorial.py", + "intermediate_source/inductor_debug_cpu.py", + "intermediate_source/jacobians_hessians.py", + "intermediate_source/optimizer_step_in_backward_tutorial.py", + "intermediate_source/per_sample_grads.py", + "intermediate_source/pruning_tutorial.py", + "intermediate_source/reinforcement_q_learning.py", + "intermediate_source/tensorboard_profiler_tutorial.py", + "intermediate_source/tiatoolbox_tutorial.rst", + "intermediate_source/torch_compile_tutorial.py", + "intermediate_source/transformer_building_blocks.py", + "prototype_source/README.md", + "prototype_source/README.txt", + "prototype_source/backend_config_tutorial.rst", + "prototype_source/gpu_direct_storage.py", + "prototype_source/inductor_cpp_wrapper_tutorial.rst", + "prototype_source/inductor_windows.rst", + "prototype_source/maskedtensor_advanced_semantics.py", + "prototype_source/max_autotune_on_CPU_tutorial.rst", + "prototype_source/vmap_recipe.py", + "recipes_source/README.txt", + "recipes_source/amx.rst", + "recipes_source/compiling_optimizer.rst", + "recipes_source/compiling_optimizer_lr_scheduler.py", + "recipes_source/distributed_optim_torchscript.rst", + "recipes_source/foreach_map.py", + "recipes_source/profile_with_itt.rst", + "recipes_source/recipes/Captum_Recipe.py", + "recipes_source/recipes/benchmark.py", + "recipes_source/recipes/changing_default_device.py", + "recipes_source/recipes/defining_a_neural_network.py", + "recipes_source/recipes/tensorboard_with_pytorch.py", + "recipes_source/recipes/timer_quick_start.py", + "recipes_source/recipes/tuning_guide.py", + "recipes_source/recipes/warmstarting_model_using_parameters_from_a_different_model.py", + "recipes_source/recipes/what_is_state_dict.py", + "recipes_source/torch_compile_caching_tutorial.rst", + "recipes_source/torch_compile_torch_function_modes.py", + "recipes_source/torch_compile_user_defined_triton_kernel_tutorial.py", + "recipes_source/torch_compiler_set_stance_tutorial.py", + "recipes_source/torch_export_aoti_python.py", + "recipes_source/xeon_run_cpu.rst", + "redirects.py", + "tutorial_submission_policy.md", +] +init_command = [ + 'python3', + 'tools/linter/adapters/run_from_link.py', + '--lint-name=grep_linter.py', + '--lint-link=https://raw.githubusercontent.com/pytorch/pytorch/4805a6ead6f1e7f32351056e2602be4e908f69b7/tools/linter/adapters/grep_linter.py', + '--', + '--dry-run={{DRYRUN}}', +] +command = [ + 'python3', + 'tools/linter/adapters/run_from_link.py', + '--run-lint', + '--lint-name=grep_linter.py', + '--', + '--pattern=[[:blank:]]$', + '--linter-name=SPACES', + '--error-name=trailing spaces', + '--replace-pattern=s/[[:blank:]]+$//', + """--error-description=\ + This line has trailing spaces; please remove them.\ + """, + '--', + '@{{PATHSFILE}}' +] + +[[linter]] +code = 'TABS' +include_patterns = ['**'] +exclude_patterns = [ + "_static/**/*", # Contains some files that should usually not be linted + ".lintrunner.toml", # Ironically needs to contain the tab character to find in other files + # All files below this should be checked and either removed from the + # exclusion list by fixing them or have a reason to be excluded. + "Makefile", + "advanced_source/README.txt", + "advanced_source/cpp_frontend.rst", + "advanced_source/torch_script_custom_ops.rst", + "beginner_source/README.txt", + "beginner_source/basics/tensorqs_tutorial.py", + "beginner_source/blitz/README.txt", + "beginner_source/blitz/tensor_tutorial.py", + "beginner_source/hybrid_frontend/README.txt", + "beginner_source/nlp/README.txt", + "beginner_source/nlp/pytorch_tutorial.py", + "intermediate_source/README.txt", + "intermediate_source/TP_tutorial.rst", + "intermediate_source/inductor_debug_cpu.py", + "prototype_source/README.txt", + "recipes_source/README.txt", + "recipes_source/recipes/README.txt", + "recipes_source/xeon_run_cpu.rst", +] +init_command = [ + 'python3', + 'tools/linter/adapters/run_from_link.py', + '--lint-name=grep_linter.py', + '--lint-link=https://raw.githubusercontent.com/pytorch/pytorch/4805a6ead6f1e7f32351056e2602be4e908f69b7/tools/linter/adapters/grep_linter.py', + '--', + '--dry-run={{DRYRUN}}', +] +command = [ + 'python3', + 'tools/linter/adapters/run_from_link.py', + '--run-lint', + '--lint-name=grep_linter.py', + '--', + # @lint-ignore TXT2 + '--pattern= ', + '--linter-name=TABS', + '--error-name=saw some tabs', + '--replace-pattern=s/\t/ /', + """--error-description=\ + This line has tabs; please replace them with spaces.\ + """, + '--', + '@{{PATHSFILE}}' +] + +[[linter]] +code = 'NEWLINE' +include_patterns=['**'] +exclude_patterns=[ + "_static/**/*", # Contains some files that should usually not be linted + # All files below this should be checked and either removed from the + # exclusion list by fixing them or have a reason to be excluded. + ".github/workflows/StalePRs.yml", + "CONTRIBUTING.md", + "advanced_source/extend_dispatcher.rst", + "advanced_source/neural_style_tutorial.py", + "advanced_source/sharding.rst", + "advanced_source/torch_script_custom_classes/custom_class_project/custom_test.py", + "advanced_source/transformer__timeseries_cpp_tutorial/transformer_timeseries.cpp", + "beginner_source/blitz/README.txt", + "beginner_source/dcgan_faces_tutorial.py", + "beginner_source/hta_trace_diff_tutorial.rst", + "beginner_source/hybrid_frontend/README.txt", + "beginner_source/nlp/pytorch_tutorial.py", + "beginner_source/template_tutorial.py", + "beginner_source/transfer_learning_tutorial.py", + "intermediate_source/custom_function_conv_bn_tutorial.py", + "intermediate_source/custom_function_double_backward_tutorial.rst", + "intermediate_source/forced_alignment_with_torchaudio_tutorial.rst", + "intermediate_source/nlp_from_scratch_index.rst", + "intermediate_source/pipeline_tutorial.rst", + "intermediate_source/tiatoolbox_tutorial.rst", + "recipes_source/README.txt", + "recipes_source/script_optimized.rst", + "recipes_source/torch_compile_caching_configuration_tutorial.rst", + "recipes_source/torch_compile_caching_tutorial.rst", +] +init_command = [ + 'python3', + 'tools/linter/adapters/run_from_link.py', + '--lint-name=newlines_linter.py', + '--lint-link=https://raw.githubusercontent.com/pytorch/pytorch/4805a6ead6f1e7f32351056e2602be4e908f69b7/tools/linter/adapters/newlines_linter.py', + '--', + '--dry-run={{DRYRUN}}', +] +command = [ + 'python3', + 'tools/linter/adapters/run_from_link.py', + '--run-lint', + '--lint-name=newlines_linter.py', + '--', + '@{{PATHSFILE}}', +] +is_formatter = true diff --git a/tools/linter/adapters/run_from_link.py b/tools/linter/adapters/run_from_link.py new file mode 100644 index 00000000000..57c2f89f9a5 --- /dev/null +++ b/tools/linter/adapters/run_from_link.py @@ -0,0 +1,81 @@ +import argparse +import subprocess +import urllib.request +from pathlib import Path + + +REPO_ROOT = Path(__file__).absolute().parents[3] + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Use a formatter in a different repository.", + ) + parser.add_argument( + "--run-init", + action="store_true", + help="Run the initialization script specified by --init-name.", + ) + parser.add_argument( + "--run-lint", + action="store_true", + help="Run the linting script specified by --lint-name.", + ) + parser.add_argument( + "--init-name", + help="Name of the initialization script. This also serves as the filename.", + ) + parser.add_argument( + "--init-link", + help="URL to download the initialization script from.", + ) + parser.add_argument( + "--lint-name", + help="Name of the linting script. This also serves as the filename.", + ) + parser.add_argument( + "--lint-link", + help="URL to download the linting script from.", + ) + + parser.add_argument("args_for_file", nargs=argparse.REMAINDER) + args = parser.parse_args() + # Skip the first -- if present + if args.args_for_file and args.args_for_file[0] == "--": + args.args_for_file = args.args_for_file[1:] + return args + + +def download_file(url: str, location: Path) -> bytes: + response = urllib.request.urlopen(url) + content = response.read() + location.write_bytes(content) + return content + + +def main() -> None: + args = parse_args() + + location = REPO_ROOT / ".lintbin" / "from_link" / "adapters" + location.mkdir(parents=True, exist_ok=True) + + if args.lint_link: + download_file(args.lint_link, location / args.lint_name) + + if args.init_link: + download_file(args.init_link, location / args.init_name) + + if args.run_init: + # Save the content to a file named after the name argument + subprocess_args = ["python3", location / args.init_name] + args.args_for_file + subprocess.run(subprocess_args, check=True) + if args.run_lint: + subprocess_args = ["python3", location / args.lint_name] + args.args_for_file + subprocess.run( + subprocess_args, + check=True, + ) + + +if __name__ == "__main__": + main() From d767b15e7ea923a25915e12bf6cb89ec21139b3b Mon Sep 17 00:00:00 2001 From: Catherine Lee Date: Wed, 16 Jul 2025 10:07:10 -0700 Subject: [PATCH 2/3] tc --- .lintrunner.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/.lintrunner.toml b/.lintrunner.toml index 110c1c1b702..20cd85cda05 100644 --- a/.lintrunner.toml +++ b/.lintrunner.toml @@ -1,5 +1,6 @@ merge_base_with = "origin/main" +# 4805a6ead6f1e7f32351056e2602be4e908f69b7 is from pytorch/pytorch main branch 2025-07-16 [[linter]] code = 'SPACES' From 1898dac6dc4ef5a77ea240190f5f415323da0ba5 Mon Sep 17 00:00:00 2001 From: Catherine Lee Date: Wed, 16 Jul 2025 10:09:08 -0700 Subject: [PATCH 3/3] tc --- .lintrunner.toml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.lintrunner.toml b/.lintrunner.toml index 20cd85cda05..86a531f340c 100644 --- a/.lintrunner.toml +++ b/.lintrunner.toml @@ -110,6 +110,12 @@ exclude_patterns = [ "recipes_source/xeon_run_cpu.rst", "redirects.py", "tutorial_submission_policy.md", + ".jenkins/build.sh", + "advanced_source/cpp_export.rst", + "advanced_source/torch-script-parallelism.rst", + "advanced_source/torch_script_custom_classes.rst", + "advanced_source/torch_script_custom_ops.rst", + "recipes_source/torchscript_inference.rst", ] init_command = [ 'python3',