|
| 1 | +"""Cli for rendering markdown docs""" |
| 2 | + |
| 3 | +import argparse |
| 4 | +import textwrap |
| 5 | +import os |
| 6 | +import re |
| 7 | + |
| 8 | +import dlt.cli.echo as fmt |
| 9 | + |
| 10 | +HEADER = """--- |
| 11 | +title: Command Line Interface |
| 12 | +description: Command line interface (CLI) full reference of dlt |
| 13 | +keywords: [command line interface, cli, dlt init] |
| 14 | +--- |
| 15 | +
|
| 16 | +
|
| 17 | +# Command Line Interface Reference |
| 18 | +
|
| 19 | +<!-- this page is fully generated from the argparse object of dlt, run make update-cli-docs to update it --> |
| 20 | +
|
| 21 | +This page contains all commands available in the dlt CLI and is generated |
| 22 | +automatically from the fully populated python argparse object of dlt. |
| 23 | +:::note |
| 24 | +Flags and positional commands are inherited from the parent command. Position within the command string |
| 25 | +is important. For example if you want to enable debug mode on the pipeline command, you need to add the |
| 26 | +debug flag to the base dlt command: |
| 27 | +
|
| 28 | +```sh |
| 29 | +dlt --debug pipeline |
| 30 | +``` |
| 31 | +
|
| 32 | +Adding the flag after the pipeline keyword will not work. |
| 33 | +::: |
| 34 | +
|
| 35 | +""" |
| 36 | + |
| 37 | +# Developer NOTE: This generation is based on parsing the output of the help string in argparse. |
| 38 | +# It works very well at the moment, but there may be cases where it will break due to unanticipated |
| 39 | +# argparse help output. The cleaner solution would be to implement an argparse help formatter, |
| 40 | +# but this would require more work and also some not very nice parsing. |
| 41 | + |
| 42 | + |
| 43 | +class _WidthFormatter(argparse.RawTextHelpFormatter): |
| 44 | + def __init__(self, prog: str) -> None: |
| 45 | + super().__init__(prog, width=99999) |
| 46 | + |
| 47 | + |
| 48 | +def render_argparse_markdown( |
| 49 | + name: str, |
| 50 | + parser: argparse.ArgumentParser, |
| 51 | + header: str = HEADER, |
| 52 | +) -> str: |
| 53 | + def get_parser_help_recursive( |
| 54 | + parser: argparse.ArgumentParser, |
| 55 | + cmd: str = "", |
| 56 | + parent: str = "", |
| 57 | + nesting: int = 0, |
| 58 | + help_string: str = None, |
| 59 | + ) -> str: |
| 60 | + markdown = "" |
| 61 | + |
| 62 | + # Prevent wrapping in help output for better parseability |
| 63 | + parser.formatter_class = _WidthFormatter |
| 64 | + if parser.description: |
| 65 | + # remove markdown from description |
| 66 | + parser.description = parser.description.markup # type: ignore |
| 67 | + help_output = parser.format_help() |
| 68 | + sections = help_output.split("\n\n") |
| 69 | + |
| 70 | + def _text_wrap_line(line: str, indent: int = 4) -> str: |
| 71 | + return "\n".join( |
| 72 | + textwrap.wrap( |
| 73 | + line, |
| 74 | + width=80, |
| 75 | + break_on_hyphens=False, |
| 76 | + subsequent_indent=" " * indent, |
| 77 | + break_long_words=False, |
| 78 | + ) |
| 79 | + ) |
| 80 | + |
| 81 | + # extract usage section denoted by "usage: " |
| 82 | + usage = sections[0] |
| 83 | + usage = usage.replace("usage: ", "") |
| 84 | + usage = _text_wrap_line(usage) |
| 85 | + |
| 86 | + # get description or use help passed down from choices |
| 87 | + help_string = help_string or "" |
| 88 | + help_string = help_string.strip().strip(" .") + "." |
| 89 | + description = parser.description or help_string or "" |
| 90 | + description = description.strip().strip(" .") + "." |
| 91 | + |
| 92 | + if nesting == 0: |
| 93 | + help_string = description |
| 94 | + description = None |
| 95 | + |
| 96 | + if not parser.description: |
| 97 | + fmt.warning(f"No description found for {cmd}, please consider providing one.") |
| 98 | + |
| 99 | + inherits_from = "" |
| 100 | + if parent: |
| 101 | + parent_slug = parent.lower().replace(" ", "-") |
| 102 | + inherits_from = f"Inherits arguments from [`{parent}`](#{parent_slug})." |
| 103 | + |
| 104 | + # extract all other sections |
| 105 | + # here we remove excess information and style the args nicely |
| 106 | + # into markdown lists |
| 107 | + extracted_sections = [] |
| 108 | + for section in sections[1:]: |
| 109 | + section_lines = section.splitlines() |
| 110 | + |
| 111 | + # detect full sections |
| 112 | + if not section_lines[0].endswith(":"): |
| 113 | + continue |
| 114 | + |
| 115 | + if len(section_lines[0]) > 30: |
| 116 | + continue |
| 117 | + |
| 118 | + # remove first line with header and empty lines |
| 119 | + header = section_lines[0].replace(":", "") |
| 120 | + |
| 121 | + if header.lower() not in ["available subcommands", "positional arguments", "options"]: |
| 122 | + fmt.warning(f"Skipping unknown section {header} of {cmd}.") |
| 123 | + continue |
| 124 | + |
| 125 | + section_lines = section_lines[1:] |
| 126 | + section_lines = [line for line in section_lines if line] |
| 127 | + section = textwrap.dedent(os.linesep.join(section_lines)) |
| 128 | + |
| 129 | + # NOTE: this is based on convention to name the subcommands section |
| 130 | + # "available subcommands" |
| 131 | + is_subcommands_list = "subcommands" in header |
| 132 | + |
| 133 | + # split args into array and remove more unneeded lines |
| 134 | + section_lines = re.split(r"\s{2,}|\n+", section) |
| 135 | + section_lines = [line for line in section_lines if not line.startswith("{")] |
| 136 | + assert len(section_lines) % 2 == 0, ( |
| 137 | + f"Expected even number of lines, args and descriptions in section {header} of" |
| 138 | + f" {cmd}. Possible problems are a different help formatter or arguments that are" |
| 139 | + " missing help strings." |
| 140 | + ) |
| 141 | + |
| 142 | + # make markdown list of args |
| 143 | + section = "" |
| 144 | + for x in range(0, len(section_lines), 2): |
| 145 | + arg_title = section_lines[x] |
| 146 | + if is_subcommands_list: |
| 147 | + full_command = f"{cmd} {arg_title}" |
| 148 | + anchor_slug = full_command.lower().replace(" ", "-") |
| 149 | + arg_title = f"[`{arg_title}`](#{anchor_slug})" |
| 150 | + else: |
| 151 | + arg_title = f"`{arg_title}`" |
| 152 | + section += f"* {arg_title} - {section_lines[x+1].capitalize()}\n" |
| 153 | + |
| 154 | + extracted_sections.append({"header": header.capitalize(), "section": section}) |
| 155 | + |
| 156 | + heading = "##" if nesting < 2 else "###" |
| 157 | + markdown += f"{heading} `{cmd}`\n\n" |
| 158 | + if help_string: |
| 159 | + markdown += f"{help_string}\n\n" |
| 160 | + markdown += "**Usage**\n" |
| 161 | + markdown += "```sh\n" |
| 162 | + markdown += f"{usage}\n" |
| 163 | + markdown += "```\n\n" |
| 164 | + |
| 165 | + if description: |
| 166 | + markdown += "**Description**\n\n" |
| 167 | + markdown += f"{description}\n\n" |
| 168 | + |
| 169 | + markdown += "<details>\n\n" |
| 170 | + markdown += "<summary>Show Arguments and Options</summary>\n\n" |
| 171 | + |
| 172 | + if inherits_from: |
| 173 | + markdown += f"{inherits_from}\n\n" |
| 174 | + |
| 175 | + for es in extracted_sections: |
| 176 | + markdown += f"**{es['header']}**\n" |
| 177 | + markdown += f"{es['section']}\n" |
| 178 | + |
| 179 | + markdown += "</details>\n\n" |
| 180 | + |
| 181 | + # traverse the subparsers and forward help strings to the recursive function |
| 182 | + for action in parser._actions: |
| 183 | + if isinstance(action, argparse._SubParsersAction): |
| 184 | + for subaction in action._get_subactions(): |
| 185 | + subparser = action._name_parser_map[subaction.dest] |
| 186 | + assert ( |
| 187 | + subaction.help |
| 188 | + ), f"Subparser help string of {subaction.dest} is empty, please provide one." |
| 189 | + markdown += get_parser_help_recursive( |
| 190 | + subparser, |
| 191 | + f"{cmd} {subaction.dest}", |
| 192 | + parent=cmd, |
| 193 | + nesting=nesting + 1, |
| 194 | + help_string=subaction.help, |
| 195 | + ) |
| 196 | + return markdown |
| 197 | + |
| 198 | + markdown = get_parser_help_recursive(parser, name) |
| 199 | + |
| 200 | + return header + markdown |
0 commit comments