feat: add Markdown formatter

Signed-off-by: Yordis Prieto <yordis.prieto@gmail.com>

Author: yordis

lib/ex_doc/doc_ast.ex

@@ -65,6 +65,76 @@ defmodule ExDoc.DocAST do
  Enum.map(attrs, fn {key, val} -> " #{key}=\"#{ExDoc.Utils.h(val)}\"" end)
  end
 
+ @doc """
+ Transform AST into markdown string.
+
+ The optional `fun` argument allows post-processing each AST node
+ after it's been converted to markdown.
+ """
+ def to_markdown_string(ast, fun \\ fn _ast, string -> string end)
+
+ def to_markdown_string(binary, _fun) when is_binary(binary) do
+ ExDoc.Utils.h(binary)
+ end
+
+ def to_markdown_string(list, fun) when is_list(list) do
+ result = Enum.map_join(list, "", &to_markdown_string(&1, fun))
+ fun.(list, result)
+ end
+
+ def to_markdown_string({:comment, _attrs, inner, _meta} = ast, fun) do
+ fun.(ast, "<!--#{inner}-->")
+ end
+
+ def to_markdown_string({:code, attrs, inner, _meta} = ast, fun) do
+ lang = attrs[:class] || ""
+ result = """
+ ```#{lang}
+ #{inner}
+ ```
+ """
+
+ fun.(ast, result)
+ end
+
+ def to_markdown_string({:a, attrs, inner, _meta} = ast, fun) do
+ result = "[#{to_markdown_string(inner, fun)}](#{attrs[:href]})"
+ fun.(ast, result)
+ end
+
+ def to_markdown_string({:hr, _attrs, _inner, _meta} = ast, fun) do
+ result = "\n\n---\n\n"
+ fun.(ast, result)
+ end
+
+ def to_markdown_string({tag, _attrs, _inner, _meta} = ast, fun) when tag in [:p, :br] do
+ result = "\n\n"
+ fun.(ast, result)
+ end
+
+ def to_markdown_string({:img, attrs, _inner, _meta} = ast, fun) do
+ alt = attrs[:alt] || ""
+ title = attrs[:title] || ""
+ result = "![#{alt}](#{attrs[:src]} \"#{title}\")"
+ fun.(ast, result)
+ end
+
+ # ignoring these: area base col command embed input keygen link meta param source track wbr
+ def to_markdown_string({tag, _attrs, _inner, _meta} = ast, fun) when tag in @void_elements do
+ result = ""
+ fun.(ast, result)
+ end
+
+ def to_markdown_string({_tag, _attrs, inner, %{verbatim: true}} = ast, fun) do
+ result = Enum.join(inner, "")
+ fun.(ast, result)
+ end
+
+ def to_markdown_string({_tag, _attrs, inner, _meta} = ast, fun) do
+ result = to_markdown_string(inner, fun)
+ fun.(ast, result)
+ end
+
  ## parse markdown
 
  defp parse_markdown(markdown, opts) do

lib/ex_doc/formatter.ex

@@ -48,14 +48,14 @@ defmodule ExDoc.Formatter do
 
  specs = Enum.map(child_node.specs, &language.autolink_spec(&1, autolink_opts))
  child_node = %{child_node | specs: specs}
- render_doc(child_node, language, autolink_opts, opts)
+ render_doc(child_node, ext, language, autolink_opts, opts)
  end
 
- %{render_doc(group, language, autolink_opts, opts) | docs: docs}
+ %{render_doc(group, ext, language, autolink_opts, opts) | docs: docs}
  end
 
  %{
- render_doc(node, language, [{:id, node.id} | autolink_opts], opts)
+ render_doc(node, ext, language, [{:id, node.id} | autolink_opts], opts)
  | docs_groups: docs_groups
  }
  end,
@@ -117,11 +117,11 @@ defmodule ExDoc.Formatter do
 
  # Helper functions
 
- defp render_doc(%{doc: nil} = node, _language, _autolink_opts, _opts),
+ defp render_doc(%{doc: nil} = node, _ext, _language, _autolink_opts, _opts),
  do: node
 
- defp render_doc(%{doc: doc} = node, language, autolink_opts, opts) do
- doc = autolink_and_highlight(doc, language, autolink_opts, opts)
+ defp render_doc(%{doc: doc} = node, ext, language, autolink_opts, opts) do
+ doc = autolink_and_render(doc, ext, language, autolink_opts, opts)
  %{node | doc: doc}
  end
 
@@ -137,7 +137,13 @@ defmodule ExDoc.Formatter do
  mod_id <> "." <> id
  end
 
- defp autolink_and_highlight(doc, language, autolink_opts, opts) do
+ defp autolink_and_render(doc, ".md", language, autolink_opts, _opts) do
+ doc
+ |> language.autolink_doc(autolink_opts)
+ |> ExDoc.DocAST.to_markdown_string()
+ end
+
+ defp autolink_and_render(doc, _html_ext, language, autolink_opts, opts) do
  doc
  |> language.autolink_doc(autolink_opts)
  |> ExDoc.DocAST.highlight(language, opts)
@@ -183,6 +189,7 @@ defmodule ExDoc.Formatter do
  id = input_options[:filename] || input |> filename_to_title() |> Utils.text_to_id()
  source_file = input_options[:source] || input
  opts = [file: source_file, line: 1]
+ ext = Keyword.fetch!(autolink_opts, :ext)
 
  {extension, source, ast} =
  case extension_name(input) do
@@ -198,7 +205,7 @@ defmodule ExDoc.Formatter do
  source
  |> Markdown.to_ast(opts)
  |> ExDoc.DocAST.add_ids_to_headers([:h2, :h3])
- |> autolink_and_highlight(language, [file: input] ++ autolink_opts, opts)
+ |> autolink_and_render(ext, language, [file: input] ++ autolink_opts, opts)
 
  {extension, source, ast}
 

lib/ex_doc/formatter/markdown.ex

@@ -0,0 +1,211 @@
+defmodule ExDoc.Formatter.MARKDOWN do
+ @moduledoc false
+
+ alias __MODULE__.{Templates}
+ alias ExDoc.Formatter
+ alias ExDoc.Utils
+
+ @doc """
+ Generates Markdown documentation for the given modules.
+ """
+ @spec run([ExDoc.ModuleNode.t()], [ExDoc.ModuleNode.t()], ExDoc.Config.t()) :: String.t()
+ def run(project_nodes, filtered_modules, config) when is_map(config) do
+ Utils.unset_warned()
+
+ config = normalize_config(config)
+ File.rm_rf!(config.output)
+ File.mkdir_p!(config.output)
+
+ extras = Formatter.build_extras(config, ".md")
+
+ project_nodes =
+ project_nodes
+ |> Formatter.render_all(filtered_modules, ".md", config, highlight_tag: "samp")
+
+ nodes_map = %{
+ modules: Formatter.filter_list(:module, project_nodes),
+ tasks: Formatter.filter_list(:task, project_nodes)
+ }
+
+ config = %{config | extras: extras}
+
+ generate_nav(config, nodes_map)
+ generate_extras(config)
+ generate_list(config, nodes_map.modules)
+ generate_list(config, nodes_map.tasks)
+ generate_llm_index(config, nodes_map)
+
+ config.output |> Path.join("index.md") |> Path.relative_to_cwd()
+ end
+
+ defp normalize_config(config) do
+ output =
+ config.output
+ |> Path.expand()
+ |> Path.join("markdown")
+
+ %{config | output: output}
+ end
+
+ defp normalize_output(output) do
+ output
+ |> String.replace(~r/\r\n|\r|\n/, "\n")
+ |> String.replace(~r/\n{3,}/, "\n\n")
+ end
+
+ defp generate_nav(config, nodes) do
+ nodes =
+ Map.update!(nodes, :modules, fn modules ->
+ modules |> Enum.chunk_by(& &1.group) |> Enum.map(&{hd(&1).group, &1})
+ end)
+
+ content =
+ Templates.nav_template(config, nodes)
+ |> normalize_output()
+
+ File.write("#{config.output}/index.md", content)
+ end
+
+ defp generate_extras(config) do
+ for {_title, extras} <- config.extras do
+ Enum.each(extras, fn %{id: id, source: content} ->
+ output = "#{config.output}/#{id}.md"
+
+ if File.regular?(output) do
+ Utils.warn("file #{Path.relative_to_cwd(output)} already exists", [])
+ end
+
+ File.write!(output, normalize_output(content))
+ end)
+ end
+ end
+
+ defp generate_list(config, nodes) do
+ nodes
+ |> Task.async_stream(&generate_module_page(&1, config), timeout: :infinity)
+ |> Enum.map(&elem(&1, 1))
+ end
+
+ ## Helpers
+
+ defp generate_module_page(module_node, config) do
+ content =
+ Templates.module_page(config, module_node)
+ |> normalize_output()
+
+ File.write("#{config.output}/#{module_node.id}.md", content)
+ end
+
+ defp generate_llm_index(config, nodes_map) do
+ content = generate_llm_index_content(config, nodes_map)
+ File.write("#{config.output}/llms.txt", content)
+ end
+
+ defp generate_llm_index_content(config, nodes_map) do
+ project_info = """
+ # #{config.project} #{config.version}
+
+ #{config.project} documentation index for Large Language Models.
+
+ ## Modules
+
+ """
+
+ modules_info =
+ nodes_map.modules
+ |> Enum.map(fn module_node ->
+ "- **#{module_node.title}** (#{module_node.id}.md): #{module_node.doc |> extract_summary()}"
+ end)
+ |> Enum.join("\n")
+
+ tasks_info =
+ if length(nodes_map.tasks) > 0 do
+ tasks_list =
+ nodes_map.tasks
+ |> Enum.map(fn task_node ->
+ "- **#{task_node.title}** (#{task_node.id}.md): #{task_node.doc |> extract_summary()}"
+ end)
+ |> Enum.join("\n")
+
+ "\n\n## Mix Tasks\n\n" <> tasks_list
+ else
+ ""
+ end
+
+ extras_info =
+ if is_list(config.extras) and length(config.extras) > 0 do
+ extras_list =
+ config.extras
+ |> Enum.flat_map(fn
+ {_group, extras} when is_list(extras) -> extras
+ _ -> []
+ end)
+ |> Enum.map(fn extra ->
+ "- **#{extra.title}** (#{extra.id}.md): #{extra.title}"
+ end)
+ |> Enum.join("\n")
+
+ if extras_list == "" do
+ ""
+ else
+ "\n\n## Guides\n\n" <> extras_list
+ end
+ else
+ ""
+ end
+
+ project_info <> modules_info <> tasks_info <> extras_info
+ end
+
+ defp extract_summary(nil), do: "No documentation available"
+ defp extract_summary(""), do: "No documentation available"
+
+ defp extract_summary(doc) when is_binary(doc) do
+ doc
+ |> String.split("\n")
+ |> Enum.find("", fn line -> String.trim(line) != "" end)
+ |> String.trim()
+ |> case do
+ "" ->
+ "No documentation available"
+
+ summary ->
+ summary
+ |> String.slice(0, 150)
+ |> then(fn s -> if String.length(s) == 150, do: s <> "...", else: s end)
+ end
+ end
+
+ defp extract_summary(doc_ast) when is_list(doc_ast) do
+ # For DocAST (which is a list), extract the first text node
+ extract_first_text_from_ast(doc_ast)
+ end
+
+ defp extract_summary(_), do: "No documentation available"
+
+ defp extract_first_text_from_ast([]), do: "No documentation available"
+
+ defp extract_first_text_from_ast([{:p, _, content} | _rest]) do
+ extract_text_from_content(content)
+ |> String.slice(0, 150)
+ |> then(fn s -> if String.length(s) == 150, do: s <> "...", else: s end)
+ end
+
+ defp extract_first_text_from_ast([_node | rest]) do
+ extract_first_text_from_ast(rest)
+ end
+
+ defp extract_text_from_content([]), do: ""
+ defp extract_text_from_content([text | _rest]) when is_binary(text), do: text
+
+ defp extract_text_from_content([{_tag, _attrs, content} | rest]) do
+ case extract_text_from_content(content) do
+ "" -> extract_text_from_content(rest)
+ text -> text
+ end
+ end
+
+ defp extract_text_from_content([_node | rest]) do
+ extract_text_from_content(rest)
+ end
+end