Skip to content

📝 Update markdown includes format #1254

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom
Merged
merged 5 commits into from
Dec 22, 2024
Merged

📝 Update markdown includes format #1254

merged 5 commits into from
Dec 22, 2024

Conversation

@tiangolo
Copy link
Member

@tiangolo tiangolo commented Dec 22, 2024 •
edited
Loading

📝 Update markdown includes format

This would solve / related to: #1150

The changes were generated with a script that was developed iteratively, updating it and fixing it to account for all the cases. It's much better and easier to automatize the process and review the results consistently than to review changes made by hand, file by file.

Here's the script:

import re from pathlib import Path import typer from pydantic import BaseModel class InvalidFormatError(Exception): """  Invalid format  """ def get_hl_lines(old_first_line: str) -> list[tuple[int, int]] | None: match = re.search(r"hl_lines=\"(.*)\"", old_first_line) if match is None: return None old_hl = match.group(1) assert isinstance(old_hl, str) hl_lines = [] for line in old_hl.split(): if "-" in line: start, _, end = line.partition("-") hl_lines.append((int(start), int(end))) else: hl_lines.append((int(line), int(line))) return hl_lines class FileReference(BaseModel): file: str lines: tuple[int, int] | None = None def get_file_reference(old_reference_line: str) -> FileReference: file_match = re.search(r"./(.*).py", old_reference_line) if file_match is None: raise InvalidFormatError("Invalid file reference.") file_ref = file_match.group(0) line_match = re.search(r"\[ln:(.*)\]", old_reference_line) if line_match is None: return FileReference(file=file_ref) line_ref = line_match.group(1) if "-" not in line_ref: return FileReference(file=file_ref, lines=(int(line_ref), int(line_ref))) start, end = line_ref.split("-") return FileReference(file=file_ref, lines=(int(start), int(end))) PYTHON_BLOCK = "```Python" END_BLOCK = "```" PYTHON_TAB = ("//// tab | Python", "//// tab | 🐍", "//// tab | 파이썬") TAB_END = "////" FULL_PREVIEW = "/// details | 👀 Full file preview" END_PREVIEW = "///" class FencedPython(BaseModel): first_line: str body: list[str] def __str__(self) -> str: formatted_body_lines = format_fenced_python(self) body = "\n".join(formatted_body_lines) return body def contains_includes(self) -> bool: return any(line.startswith("{!") for line in self.body) class Details(BaseModel): first_line: str body: list[str] def __str__(self) -> str: for line in self.body: if line.startswith("{!"): ref = get_file_reference(line) return "{* " + ref.file + " ln[0] *}" return "" class Tab(BaseModel): first_line: str body: list[str] def __str__(self) -> str: body = "\n".join(self.body).strip() return f"{self.first_line}\n\n{body}\n\n{TAB_END}" class TabGroup(BaseModel): tabs: list[Tab] details: list[Details] = [] def __str__(self) -> str: return format_tab_group(self) def format_fenced_python(block: FencedPython) -> list[str]: if not block.contains_includes(): return [block.first_line] + block.body + [END_BLOCK] old_hl_ines = get_hl_lines(block.first_line) refs: list[tuple[int, FileReference]] = [] offset = 0 for i, line in enumerate(block.body): if line.startswith("{!"): ref = get_file_reference(line) refs.append((offset + i, ref)) if ref.lines is not None: offset += ref.lines[1] - ref.lines[0] first_ref = refs[0][1] assert all(ref[1].file == first_ref.file for ref in refs) new_hl_lines: list[tuple[int, int]] = [] for hl_line in old_hl_ines or []: for o, ref in refs: assert ref.lines is not None ln_start, ln_end = ref.lines ln_start_render = o + 1 ln_end_render = ln_end - ln_start + ln_start_render old_hl_start, old_hl_end = hl_line if old_hl_start >= ln_start_render and old_hl_end <= ln_end_render: new_hl_start = old_hl_start - o + ln_start - 1 new_hl_end = old_hl_end - o + ln_start - 1 new_hl_lines.append((new_hl_start, new_hl_end)) break file_ref = first_ref.file lns = [ref[1].lines for ref in refs if ref[1].lines is not None] new_lns = [] for ln in lns: if ln[0] == ln[1]: new_lns.append(str(ln[0])) continue new_lns.append(f"{ln[0]}:{ln[1]}") new_ln_str = ",".join(new_lns) new_hl = [] for start, end in new_hl_lines: if start == end: new_hl.append(str(start)) continue new_hl.append(f"{start}:{end}") new_hl_str = ",".join(new_hl) new_ln_instruction = f"ln[{new_ln_str}]" if new_ln_str else "" new_hl_instruction = f"hl[{new_hl_str}]" if new_hl_str else "" new_include = "{* " + f"{file_ref}" if new_ln_instruction: new_include += f" {new_ln_instruction}" if new_hl_instruction: new_include += f" {new_hl_instruction}" new_include += " *}" return [new_include] def format_tab_group(group: TabGroup) -> str: if not group.tabs and group.details: return str(group.details[0]) first_tab = group.tabs[0] fenced_block: FencedPython | None = None for line in first_tab.body: if fenced_block: if line == END_BLOCK: if fenced_block.contains_includes(): # Return the first one, omit the rest return str(fenced_block) return "\n\n".join(str(tab) for tab in group.tabs) fenced_block.body.append(line) continue if line and not line.startswith(PYTHON_BLOCK): return "\n\n".join(str(tab) for tab in group.tabs) if line.startswith(PYTHON_BLOCK): fenced_block = FencedPython(first_line=line, body=[]) raise RuntimeError("Invalid tab group") def process_lines(lines: list[str]) -> list[str | FencedPython | Tab | Details]: new_sections: list[str | FencedPython | Tab | Details] = [] fenced_block: FencedPython | None = None tab: Tab | None = None details: Details | None = None for line in lines: if not fenced_block and not tab and not details: if line.startswith(PYTHON_BLOCK): fenced_block = FencedPython(first_line=line, body=[]) continue if line.startswith(PYTHON_TAB): tab = Tab(first_line=line, body=[]) continue if line.startswith(FULL_PREVIEW): details = Details(first_line=line, body=[]) continue new_sections.append(line) continue if fenced_block: if line == END_BLOCK: new_sections.append(fenced_block) fenced_block = None continue fenced_block.body.append(line) continue if tab: if line == TAB_END: new_sections.append(tab) tab = None continue tab.body.append(line) continue if details: if line == END_PREVIEW: new_sections.append(details) details = None continue details.body.append(line) return new_sections def process_fragments( fragments: list[str | FencedPython | Tab | Details], ) -> list[str | FencedPython | TabGroup | Details]: new_fragments: list[str | FencedPython | TabGroup | Details] = [] tab_group: TabGroup | None = None for fragment in fragments: if not tab_group: if isinstance(fragment, Tab): tab_group = TabGroup(tabs=[fragment]) continue if isinstance(fragment, Details): tab_group = TabGroup(tabs=[], details=[fragment]) continue new_fragments.append(fragment) continue if tab_group: if isinstance(fragment, Tab): tab_group.tabs.append(fragment) continue if isinstance(fragment, Details): tab_group.details.append(fragment) continue if fragment == "": continue new_fragments.append(tab_group) new_fragments.append("") tab_group = None new_fragments.append(fragment) continue if tab_group: new_fragments.append(tab_group) return new_fragments def process_file(file_path: Path) -> None: lines = file_path.read_text().splitlines() sections = process_lines(lines) groups = process_fragments(sections) group_str = [str(group) for group in groups] file_path.write_text("\n".join(group_str).strip() + "\n") skip_file_names = ["code-structure.md"] def main(file_path: Path = None) -> None: if file_path: process_file(file_path) return files_with_errors = [] for f in Path("docs/").glob("**/*.md"): if f.name in skip_file_names: continue try: process_file(f) except Exception as e: print(f"An error occurred in file {f}: {e}") files_with_errors.append(f) print("Files with errors:") for f in files_with_errors: print(f) if __name__ == "__main__": typer.run(main) # file_path = Path("demo.md") # process_file(file_path)

Modified Pages to Review
@github-actions github-actions bot added the docs Improvements or additions to documentation label Dec 22, 2024
@github-actions
Copy link
Contributor

📝 Docs preview for commit e962393 at: https://885cf9ae.sqlmodel.pages.dev

Modified Pages
@github-actions
Copy link
Contributor

📝 Docs preview for commit ed2a90c at: https://662a5e84.sqlmodel.pages.dev

Modified Pages
@tiangolo tiangolo marked this pull request as ready for review December 22, 2024 14:24
@tiangolo tiangolo merged commit 5100200 into main Dec 22, 2024
26 checks passed
@tiangolo tiangolo deleted the includes branch December 22, 2024 14:30
@tiangolo tiangolo mentioned this pull request Dec 22, 2024
Closed
1 task
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment

Labels

docs Improvements or additions to documentation

2 participants

@tiangolo