Add CSV include directive (#1742)

* Add CSV file directive

* Change to Sep and defer rendering

* Use only Sep

* Add streaming and max size

* Update docs/syntax/csv-file.md

Co-authored-by: shainaraskas <[email protected]>

* Refactor and other stuff

* Remove preview mode

* Increase rows limit

* Formatting

* Reinstate old unsupported statement

* Add also to dict

* Change columns default

---------

Co-authored-by: shainaraskas <[email protected]>

Author: theletterf

Directory.Packages.props

@@ -56,6 +56,7 @@
     <PackageVersion Include="Proc" Version="0.9.1" />
     <PackageVersion Include="RazorSlices" Version="0.9.4" />
     <PackageVersion Include="Samboy063.Tomlet" Version="6.0.0" />
+    <PackageVersion Include="Sep" Version="0.11.0" />
     <PackageVersion Include="Slugify.Core" Version="4.0.1" />
     <PackageVersion Include="SoftCircuits.IniFileParser" Version="2.7.0" />
     <PackageVersion Include="System.IO.Abstractions" Version="22.0.15" />

docs/_docset.yml

@@ -96,6 +96,7 @@ toc:
       - file: code.md
       - file: comments.md
       - file: conditionals.md
+      - file: csv-include.md
       - hidden: diagrams.md
       - file: dropdowns.md
       - file: definition-lists.md

docs/_snippets/sample-data.csv

@@ -0,0 +1,6 @@
+Name,Age,City,Occupation
+John Doe,30,New York,Software Engineer
+Jane Smith,25,Los Angeles,Product Manager
+Bob Johnson,35,Chicago,Data Scientist
+Alice Brown,28,San Francisco,UX Designer
+Charlie Wilson,32,Boston,DevOps Engineer

docs/syntax/csv-include.md

@@ -0,0 +1,70 @@
+# CSV files
+
+The `{csv-include}` directive allows you to include and render CSV files as formatted tables in your documentation. The directive automatically parses CSV content and renders it using the standard table styles defined in `table.css`.
+
+## Usage
+
+:::::{tab-set}
+
+::::{tab-item} Output
+
+:::{csv-include} ../_snippets/sample-data.csv
+:caption: Sample user data from the database
+:::
+
+::::
+
+::::{tab-item} Markdown
+
+```markdown
+:::{csv-include} _snippets/sample-data.csv
+:::
+```
+
+::::
+
+:::::
+
+## Options
+
+The CSV file directive supports several options to customize the table rendering:
+
+### Caption
+
+Add a descriptive caption above the table:
+
+```markdown
+:::{csv-include} _snippets/sample-data.csv
+:caption: Sample user data from the database
+:::
+```
+
+### Custom separator
+
+Specify a custom field separator (default is comma):
+
+```markdown
+:::{csv-include} _snippets/sample-data.csv
+:separator: ;
+:::
+```
+
+### Performance limits
+
+The directive includes built-in performance limits to handle large files efficiently:
+
+- **Row limit**: Maximum of 25,000 rows will be displayed
+- **Column limit**: Maximum of 10 columns will be displayed  
+- **File size limit**: Maximum file size of 10MB
+
+## Performance considerations
+
+The CSV directive is optimized for large files:
+
+- Files are processed using streaming to avoid loading everything into memory
+- Built-in size validation prevents processing of files that exceed 10MB
+- Row and column limits protect against accidentally rendering massive tables
+- Warning messages are displayed when limits are exceeded
+
+For optimal performance with large CSV files, consider:
+- Breaking very large files into smaller, more manageable chunks

docs/syntax/index.md

@@ -12,6 +12,23 @@ V3 fully supports [CommonMark](https://commonmark.org/), a strongly defined, sta
 * [Directives](#directives)
 * [GitHub-flavored markdown](#github-flavored-markdown)
 
+## Available directives
+
+The following directives are available in Elastic Docs V3:
+
+* [Admonitions](admonitions.md) - Callouts and warnings
+* [Code blocks](code.md) - Syntax-highlighted code
+* [CSV include](csv-include.md) - Render CSV files as tables
+* [Diagrams](diagrams.md) - Visual diagrams and charts
+* [Dropdowns](dropdowns.md) - Collapsible content
+* [Images](images.md) - Enhanced image handling
+* [Include](file_inclusion.md) - Include content from other files
+* [Settings](automated_settings.md) - Configuration blocks
+* [Stepper](stepper.md) - Step-by-step content
+* [Tabs](tabs.md) - Tabbed content organization
+* [Tables](tables.md) - Data tables
+* [Version blocks](version-variables.md) - API version information
+
 ## Directives
 
 Directives extend CommonMark functionality. Directives have the following syntax:

src/Elastic.Markdown/Elastic.Markdown.csproj

@@ -20,6 +20,7 @@
     <PackageReference Include="Markdig" />
     <PackageReference Include="Microsoft.Extensions.Logging" />
     <PackageReference Include="RazorSlices" />
+    <PackageReference Include="Sep" />
     <PackageReference Include="Slugify.Core" />
     <PackageReference Include="Utf8StreamReader" />
     <PackageReference Include="Vecc.YamlDotNet.Analyzers.StaticGenerator" />

src/Elastic.Markdown/Myst/Directives/CsvInclude/CsvIncludeBlock.cs

@@ -0,0 +1,84 @@
+// Licensed to Elasticsearch B.V under one or more agreements.
+// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information
+
+using System.Globalization;
+using System.IO.Abstractions;
+using Elastic.Markdown.Diagnostics;
+
+namespace Elastic.Markdown.Myst.Directives.CsvInclude;
+
+public class CsvIncludeBlock(DirectiveBlockParser parser, ParserContext context) : DirectiveBlock(parser, context)
+{
+	public override string Directive => "csv-include";
+
+	public string? CsvFilePath { get; private set; }
+	public string? CsvFilePathRelativeToSource { get; private set; }
+	public bool Found { get; private set; }
+	public string? Caption { get; private set; }
+	public string Separator { get; private set; } = ",";
+	public int MaxRows { get; private set; } = 25000;
+	public long MaxFileSizeBytes { get; private set; } = 10 * 1024 * 1024; // 10MB
+	public int MaxColumns { get; private set; } = 10;
+
+	public override void FinalizeAndValidate(ParserContext context)
+	{
+		Caption = Prop("caption");
+
+		var separator = Prop("separator", "delimiter");
+		if (!string.IsNullOrEmpty(separator))
+			Separator = separator;
+
+
+
+		ExtractCsvPath(context);
+	}
+
+	private void ExtractCsvPath(ParserContext context)
+	{
+		var csvPath = Arguments;
+		if (string.IsNullOrWhiteSpace(csvPath))
+		{
+			this.EmitError("csv-include requires an argument specifying the path to the CSV file.");
+			return;
+		}
+
+		var csvFrom = context.MarkdownSourcePath.Directory!.FullName;
+		if (csvPath.StartsWith('/'))
+			csvFrom = Build.DocumentationSourceDirectory.FullName;
+
+		CsvFilePath = Path.Combine(csvFrom, csvPath.TrimStart('/'));
+		CsvFilePathRelativeToSource = Path.GetRelativePath(Build.DocumentationSourceDirectory.FullName, CsvFilePath);
+
+		if (Build.ReadFileSystem.File.Exists(CsvFilePath))
+		{
+			ValidateFileSize();
+			Found = true;
+		}
+		else
+			this.EmitError($"CSV file `{CsvFilePath}` does not exist.");
+	}
+
+	private void ValidateFileSize()
+	{
+		if (CsvFilePath == null)
+			return;
+
+		try
+		{
+			var fileInfo = Build.ReadFileSystem.FileInfo.New(CsvFilePath);
+			if (fileInfo.Length > MaxFileSizeBytes)
+			{
+				var sizeMB = fileInfo.Length / (1024.0 * 1024.0);
+				var maxSizeMB = MaxFileSizeBytes / (1024.0 * 1024.0);
+				this.EmitError($"CSV file `{CsvFilePath}` is {sizeMB:F1}MB, which exceeds the maximum allowed size of {maxSizeMB:F1}MB.");
+				Found = false;
+			}
+		}
+		catch (Exception ex)
+		{
+			this.EmitError($"Could not validate CSV file size: {ex.Message}");
+			Found = false;
+		}
+	}
+}

src/Elastic.Markdown/Myst/Directives/CsvInclude/CsvIncludeView.cshtml

@@ -0,0 +1,46 @@
+@using Elastic.Markdown.Myst.Directives.CsvInclude
+@inherits RazorSlice<CsvIncludeViewModel>
+
+@{
+    var csvBlock = (CsvIncludeBlock)Model.DirectiveBlock;
+    if (!csvBlock.Found)
+    {
+        return;
+    }
+    
+    var csvRows = Model.GetCsvRows().ToList();
+    if (!csvRows.Any())
+    {
+        return;
+    }
+}
+
+<div class="table-wrapper">
+    @if (!string.IsNullOrEmpty(csvBlock.Caption))
+    {
+        <caption>@csvBlock.Caption</caption>
+    }
+    
+    <table>
+        <thead>
+            <tr>
+                @for (var i = 0; i < csvRows[0].Length; i++)
+                {
+                    <th>@csvRows[0][i]</th>
+                }
+            </tr>
+        </thead>
+        
+        <tbody>
+            @for (var rowIndex = 1; rowIndex < csvRows.Count; rowIndex++)
+            {
+                <tr>
+                    @for (var i = 0; i < csvRows[rowIndex].Length; i++)
+                    {
+                        <td>@csvRows[rowIndex][i]</td>
+                    }
+                </tr>
+            }
+        </tbody>
+    </table>
+</div>

src/Elastic.Markdown/Myst/Directives/CsvInclude/CsvIncludeViewModel.cs

@@ -0,0 +1,56 @@
+// Licensed to Elasticsearch B.V under one or more agreements.
+// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information
+
+using Elastic.Markdown.Diagnostics;
+
+namespace Elastic.Markdown.Myst.Directives.CsvInclude;
+
+public class CsvIncludeViewModel : DirectiveViewModel
+{
+	public IEnumerable<string[]> GetCsvRows()
+	{
+		if (DirectiveBlock is not CsvIncludeBlock csvBlock || !csvBlock.Found || string.IsNullOrEmpty(csvBlock.CsvFilePath))
+			return [];
+
+		var csvData = CsvReader.ReadCsvFile(csvBlock.CsvFilePath, csvBlock.Separator, csvBlock.Build.ReadFileSystem);
+		var rowCount = 0;
+		var columnCountExceeded = false;
+
+		return csvData.TakeWhile(row =>
+		{
+			if (rowCount >= csvBlock.MaxRows)
+			{
+				csvBlock.EmitWarning($"CSV file contains more than {csvBlock.MaxRows} rows. Only the first {csvBlock.MaxRows} rows will be displayed.");
+				return false;
+			}
+
+			if (row.Length > csvBlock.MaxColumns)
+			{
+				if (!columnCountExceeded)
+				{
+					csvBlock.EmitWarning($"CSV file contains more than {csvBlock.MaxColumns} columns. Only the first {csvBlock.MaxColumns} columns will be displayed.");
+					columnCountExceeded = true;
+				}
+			}
+
+			rowCount++;
+			return true;
+		}).Select(row =>
+		{
+			if (row.Length > csvBlock.MaxColumns)
+			{
+				var trimmedRow = new string[csvBlock.MaxColumns];
+				Array.Copy(row, trimmedRow, csvBlock.MaxColumns);
+				return trimmedRow;
+			}
+			return row;
+		});
+	}
+
+	public static CsvIncludeViewModel Create(CsvIncludeBlock csvBlock) =>
+		new()
+		{
+			DirectiveBlock = csvBlock
+		};
+}

src/Elastic.Markdown/Myst/Directives/CsvInclude/CsvReader.cs

@@ -0,0 +1,56 @@
+// Licensed to Elasticsearch B.V under one or more agreements.
+// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information
+
+using System.IO.Abstractions;
+using nietras.SeparatedValues;
+
+namespace Elastic.Markdown.Myst.Directives.CsvInclude;
+
+public static class CsvReader
+{
+	public static IEnumerable<string[]> ReadCsvFile(string filePath, string separator, IFileSystem? fileSystem = null)
+	{
+		var fs = fileSystem ?? new FileSystem();
+		return ReadWithSep(filePath, separator, fs);
+	}
+
+	private static IEnumerable<string[]> ReadWithSep(string filePath, string separator, IFileSystem fileSystem)
+	{
+		var separatorChar = separator == "," ? ',' : separator[0];
+		var spec = Sep.New(separatorChar);
+
+		// Sep works with actual file paths, not virtual file systems
+		// For testing with MockFileSystem, we'll read content first
+		if (fileSystem.GetType().Name == "MockFileSystem")
+		{
+			var content = fileSystem.File.ReadAllText(filePath);
+			using var reader = spec.Reader(o => o with { HasHeader = false, Unescape = true }).FromText(content);
+
+			foreach (var row in reader)
+			{
+				var rowData = new string[row.ColCount];
+				for (var i = 0; i < row.ColCount; i++)
+				{
+					rowData[i] = row[i].ToString();
+				}
+				yield return rowData;
+			}
+		}
+		else
+		{
+			using var reader = spec.Reader(o => o with { HasHeader = false, Unescape = true }).FromFile(filePath);
+
+			foreach (var row in reader)
+			{
+				var rowData = new string[row.ColCount];
+				for (var i = 0; i < row.ColCount; i++)
+				{
+					rowData[i] = row[i].ToString();
+				}
+				yield return rowData;
+			}
+		}
+	}
+
+}