Implement table log monitor

Author: tkf

Project.toml

@@ -5,6 +5,7 @@ version = "0.1.0"
 
 [deps]
 Logging = "56ddb016-857b-54e1-b83d-db4d58db5568"
+PrettyTables = "08abe8d2-0d0c-5749-adfa-8a2ac140af0d"
 ProgressMeter = "92933f4c-e287-5a05-a399-4b506db050ca"
 
 [compat]

src/StickyMessages.jl

@@ -42,6 +42,14 @@ function StickyMessages(io::IO; ansi_codes=io isa Base.TTY &&
     sticky
 end
 
+function firstline!(sticky::StickyMessages, label)
+    idx = findfirst(m -> m[1] == label, sticky.messages)
+    idx === nothing && throw(KeyError(label))
+    idx == 1 && return
+    sticky.messages[idx], sticky.messages[1] = sticky.messages[1], sticky.messages[idx]
+    return
+end
+
 # Count newlines in a message or sequence of messages
 _countlines(msg::String) = sum(c->c=='\n', msg)
 _countlines(messages) = length(messages) > 0 ? sum(_countlines, messages) : 0
@@ -93,7 +101,7 @@ function showsticky(io, prev_nlines, messages)
     nothing
 end
 
-function Base.push!(sticky::StickyMessages, message::Pair)
+function Base.push!(sticky::StickyMessages, message::Pair; first=true)
     if !sticky.ansi_codes
         write(sticky.io, message[2])
         return
@@ -103,9 +111,18 @@ function Base.push!(sticky::StickyMessages, message::Pair)
     prev_nlines = _countlines(sticky.messages)
     idx = findfirst(m->m[1] == label, sticky.messages)
     if idx === nothing
-        push!(sticky.messages, label=>text)
+        if first
+            insert!(sticky.messages, 1, label=>text)
+        else
+            push!(sticky.messages, label=>text)
+        end
     else
-        sticky.messages[idx] = label=>text
+        if first
+            sticky.messages[idx] = sticky.messages[1]
+            sticky.messages[1] = label=>text
+        else
+            sticky.messages[idx] = label=>text
+        end
     end
     showsticky(sticky.io, prev_nlines, sticky.messages)
 end

src/TableMonitor.jl

@@ -0,0 +1,108 @@
+struct Row
+    id
+    names::Vector{String}
+    values::Vector{String}
+end
+
+struct TableMonitor
+    sticky_messages::StickyMessages
+    io::IO
+    rows::Vector{Row}
+    flushed::Vector{Bool}
+end
+
+TableMonitor(sticky_messages::StickyMessages, io::IO = sticky_messages.io) =
+    TableMonitor(sticky_messages, io, [], [])
+
+function tablelines(
+    table::TableMonitor,
+    oldrows = nothing,
+    tobeflushed = ();
+    bottom_line = false,
+)
+    data = permutedims(mapreduce(row -> row.values, vcat, table.rows))
+    highlighters = ()
+    if oldrows !== nothing
+        data = vcat(data, permutedims(mapreduce(row -> row.values, vcat, oldrows)))
+        right = cumsum(Int[length(row.values) for row in oldrows])
+        left = insert!(right[1:end-1], 1, 0)
+        # Dim unchanged rows:
+        highlighters = Highlighter(
+            (data, i, j) -> i == 2 && !any(k -> left[k] < j <= right[k], tobeflushed);
+            foreground = :dark_gray,
+        )
+    end
+    text = sprint(context = table.io) do io
+        pretty_table(
+            io,
+            data,
+            mapreduce(row -> row.names, vcat, table.rows),
+            PrettyTableFormat(top_line = false, bottom_line = bottom_line);
+            highlighters = highlighters,
+        )
+    end
+    return split(text, "\n")
+end
+
+function draw!(table::TableMonitor, oldrows = nothing, tobeflushed = ())
+    lines = tablelines(table, oldrows, tobeflushed)
+    formatted = join(reverse!(lines[1:3]), "\n")  # row, hline, header
+    if oldrows !== nothing
+        print(table.io, '\n', lines[4])
+    end
+    push!(table.sticky_messages, _tablelabel => formatted; first = true)
+end
+
+const _tablelabel = gensym("TableMonitor")
+
+Base.push!(table::TableMonitor, (id, namedtuple)::Pair{<:Any,<:NamedTuple}) = push!(
+    table,
+    Row(id, collect(string.(keys(namedtuple))), collect(string.(values(namedtuple)))),
+)
+
+function Base.push!(table::TableMonitor, row::Row)
+    idx = findfirst(x -> x.id == row.id, table.rows)
+    if idx !== nothing
+        if table.flushed[idx]
+            tobeflushed = ()
+            oldrows = nothing
+            table.flushed[idx] = false
+        else
+            tobeflushed = findall(!, table.flushed)
+            oldrows = copy(table.rows)
+            table.flushed .= [i != idx for i in 1:length(table.rows)]
+        end
+        table.rows[idx] = row
+        draw!(table, oldrows, tobeflushed)
+    else
+        push!(table.flushed, false)
+        push!(table.rows, row)
+        draw!(table)
+    end
+    return table
+end
+
+function flush_table!(table::TableMonitor)
+    pop!(table.sticky_messages, _tablelabel)
+
+    lines = tablelines(table; bottom_line = true)[1:end-1]
+    bottom = pop!(lines)
+    for line in reverse!(lines)
+        print(table.io, '\n', line)
+    end
+    print(table.io, '\n', bottom)
+
+    fill!(table.flushed, true)
+end
+
+function Base.pop!(table::TableMonitor, id)
+    idx = findfirst(x -> x.id == id, table.rows)
+    idx === nothing && return
+
+    if !table.flushed[idx]
+        flush_table!(table)
+    end
+
+    deleteat!(table.flushed, idx)
+    return deleteat!(table.rows, idx)
+end

src/TerminalLogger.jl

@@ -29,12 +29,16 @@ struct TerminalLogger <: AbstractLogger
     message_limits::Dict{Any,Int}
     sticky_messages::StickyMessages
     bars::Dict{Any,Progress}
+    was_table::Base.RefValue{Bool}
+    table::TableMonitor
 end
 function TerminalLogger(stream::IO=stderr, min_level=ProgressLevel;
                         meta_formatter=default_metafmt, show_limited=true,
                         right_justify=0)
+    sticky_messages = StickyMessages(stream)
     TerminalLogger(stream, min_level, meta_formatter,
-                   show_limited, right_justify, Dict{Any,Int}(), StickyMessages(stream), Dict{Any,Progress}())
+                   show_limited, right_justify, Dict{Any,Int}(), sticky_messages, Dict{Any,Progress}(),
+                   Ref(false), TableMonitor(sticky_messages))
 end
 
 shouldlog(logger::TerminalLogger, level, _module, group, id) =
@@ -147,6 +151,7 @@ function handle_progress(logger, message, id, progress)
         end
 
         if progress == "done" || progress >= 1
+            flush_table(logger)
             pop!(logger.sticky_messages, id)
             println(logger.stream, msg)
         else
@@ -160,6 +165,36 @@ function handle_progress(logger, message, id, progress)
     end
 end
 
+struct Unspecified end
+
+function maybe_handle_table(logger, id; table = Unspecified(), _...)
+    if table isa NamedTuple
+        push!(logger.table, id => table)
+        logger.was_table[] = true
+        return true
+    elseif table === nothing
+        pop!(logger.table, id)
+        logger.was_table[] = true
+        return true
+    end
+    flush_table(logger)
+    return false
+end
+
+function flush_table(logger)
+    if logger.was_table[]
+        flush_table!(logger.table)
+
+        # To "connect" flushed rows with the current row and header in
+        # the sticky messages, `TableMonitor` do not print newline
+        # after the message.  Thus, we need to print a newline when
+        # switching to non-table message:
+        println(logger.stream)
+
+        logger.was_table[] = false
+    end
+end
+
 function handle_message(logger::TerminalLogger, level, message, _module, group, id,
                         filepath, line; maxlog=nothing, progress=nothing,
                         sticky=nothing, kwargs...)
@@ -174,6 +209,8 @@ function handle_message(logger::TerminalLogger, level, message, _module, group,
         return
     end
 
+    maybe_handle_table(logger, id; kwargs...) && return
+
 	substr(s) = SubString(s, 1, length(s)) # julia 0.6 compat
 
     # Generate a text representation of the message and all key value pairs,

src/TerminalLoggers.jl

@@ -10,11 +10,15 @@ import Logging:
 using ProgressMeter:
     Progress, update!
 
+using PrettyTables:
+    Highlighter, PrettyTableFormat, pretty_table
+
 export TerminalLogger
 
 const ProgressLevel = LogLevel(-1)
 
 include("StickyMessages.jl")
+include("TableMonitor.jl")
 include("TerminalLogger.jl")
 
 end # module