From a0fb7e778bf4f3a2885fb7a0fc429a52252b52e6 Mon Sep 17 00:00:00 2001 From: Nicholas Blumhardt Date: Tue, 1 Sep 2026 15:20:44 +1000 Subject: [PATCH 01/15] Migrate data handling from Serilog's `LogEvent` across to `System.Text.Json.JsonObject`, with the help of Seq.Syntax v2.0. Assisted-by: Claude:claude-fable-5 --- src/SeqCli/Apps/AppLoader.cs | 6 +- src/SeqCli/Apps/Hosting/AppContainer.cs | 3 +- .../Apps/Hosting/SerilogLevelMapping.cs | 41 ++++ src/SeqCli/Cli/Commands/IngestCommand.cs | 22 +- src/SeqCli/Cli/Commands/PrintCommand.cs | 20 +- src/SeqCli/Cli/Commands/TraceCommand.cs | 4 +- src/SeqCli/Csv/CsvWriter.cs | 56 +++-- src/SeqCli/Forwarder/ForwarderModule.cs | 9 +- .../Web/Api/IngestionLogEndpoints.cs | 9 +- src/SeqCli/Ingestion/BatchResult.cs | 10 +- src/SeqCli/Ingestion/EnrichingReader.cs | 19 +- .../{ILogEventReader.cs => IEventReader.cs} | 2 +- src/SeqCli/Ingestion/JsonEventReader.cs | 64 ++++++ src/SeqCli/Ingestion/JsonLogEventReader.cs | 94 --------- src/SeqCli/Ingestion/LogShipper.cs | 40 ++-- src/SeqCli/Ingestion/ReadResult.cs | 15 +- src/SeqCli/Ingestion/SerilogEventJson.cs | 91 ++++++++ .../Ingestion/SerilogTracingConventions.cs | 46 ++++ .../Ingestion/StaticMessageTemplateReader.cs | 32 ++- src/SeqCli/Ingestion/TraceConstants.cs | 8 - src/SeqCli/Mapping/EventEntityJson.cs | 104 +++++++++ src/SeqCli/Mapping/LevelMapping.cs | 126 ++++++----- src/SeqCli/Mapping/MetricsMapping.cs | 8 - src/SeqCli/Mcp/Tools/Search/SearchTools.cs | 17 +- src/SeqCli/Output/FlareTheme.cs | 30 +-- src/SeqCli/Output/OutputFormat.cs | 197 +++--------------- .../Output/StripStructureTypeEnricher.cs | 25 --- src/SeqCli/Output/TextFormatters.cs | 52 ++--- src/SeqCli/Output/TraceFormatter.cs | 49 +++-- src/SeqCli/Output/TracingFunctions.cs | 55 ----- .../PlainText/LogEvents/EventJsonBuilder.cs | 100 +++++++++ .../PlainText/LogEvents/LogEventBuilder.cs | 131 ------------ .../PlainText/LogEvents/TextOnlyException.cs | 32 --- ...EventReader.cs => PlainTextEventReader.cs} | 6 +- .../{ => Sample}/Ingestion/BufferingSink.cs | 30 ++- src/SeqCli/Sample/Ingestion/MetricsMapping.cs | 59 ++++++ src/SeqCli/Sample/Loader/Simulation.cs | 2 +- src/SeqCli/SeqCli.csproj | 6 +- src/SeqCli/Syntax/EventJson.cs | 66 ++++++ .../IEventEnricher.cs} | 27 +-- .../LevelEnricher.cs} | 18 +- .../ScalarPropertyEnricher.cs | 22 +- src/SeqCli/Syntax/SeqCliNameResolver.cs | 20 -- src/SeqCli/Syntax/SeqSyntax.cs | 50 ++++- src/SeqCli/Syntax/V1/TracingFunctions.cs | 58 ++++++ src/SeqCli/Traces/StructuredMessage.cs | 70 +++++-- src/SeqCli/Traces/TraceTreeElement.cs | 6 +- .../Traces/TraceTreeJObjectConverter.cs | 24 +-- src/SeqCli/Util/JsonNetDestructuringPolicy.cs | 91 -------- src/SeqCli/Util/JsonNodes.cs | 45 ++++ src/SeqCli/Util/LogEventPropertyFactory.cs | 33 --- test/SeqCli.Tests/Csv/CsvWriterTests.cs | 6 +- .../Ingestion/SerilogEventJsonTests.cs | 109 ++++++++++ test/SeqCli.Tests/Output/OutputFormatTests.cs | 60 ++++-- .../Output/TextFormattersTests.cs | 86 ++++---- .../Output/TraceFormatterTests.cs | 19 +- .../PlainText/EventJsonBuilderTests.cs | 60 ++++++ .../PlainText/LogEventBuilderTests.cs | 56 ----- .../StaticMessageTemplateReaderTests.cs | 13 +- ...dLogEventReader.cs => FixedEventReader.cs} | 4 +- test/SeqCli.Tests/Support/Some.cs | 22 +- .../Traces/StructuredMessageTests.cs | 55 +++-- test/SeqCli.Tests/Traces/TraceQueryTests.cs | 9 +- .../Traces/TraceTreeBuilderTests.cs | 13 +- .../Traces/TraceTreeJObjectConverterTests.cs | 30 +-- 65 files changed, 1437 insertions(+), 1255 deletions(-) create mode 100644 src/SeqCli/Apps/Hosting/SerilogLevelMapping.cs rename src/SeqCli/Ingestion/{ILogEventReader.cs => IEventReader.cs} (79%) create mode 100644 src/SeqCli/Ingestion/JsonEventReader.cs delete mode 100644 src/SeqCli/Ingestion/JsonLogEventReader.cs create mode 100644 src/SeqCli/Ingestion/SerilogEventJson.cs create mode 100644 src/SeqCli/Ingestion/SerilogTracingConventions.cs delete mode 100644 src/SeqCli/Ingestion/TraceConstants.cs create mode 100644 src/SeqCli/Mapping/EventEntityJson.cs delete mode 100644 src/SeqCli/Mapping/MetricsMapping.cs delete mode 100644 src/SeqCli/Output/StripStructureTypeEnricher.cs delete mode 100644 src/SeqCli/Output/TracingFunctions.cs create mode 100644 src/SeqCli/PlainText/LogEvents/EventJsonBuilder.cs delete mode 100644 src/SeqCli/PlainText/LogEvents/LogEventBuilder.cs delete mode 100644 src/SeqCli/PlainText/LogEvents/TextOnlyException.cs rename src/SeqCli/PlainText/{PlainTextLogEventReader.cs => PlainTextEventReader.cs} (86%) rename src/SeqCli/{ => Sample}/Ingestion/BufferingSink.cs (50%) create mode 100644 src/SeqCli/Sample/Ingestion/MetricsMapping.cs create mode 100644 src/SeqCli/Syntax/EventJson.cs rename src/SeqCli/{Util/TextException.cs => Syntax/IEventEnricher.cs} (60%) rename src/SeqCli/{Output/RedundantEventTypeRemovalEnricher.cs => Syntax/LevelEnricher.cs} (64%) rename src/SeqCli/{Ingestion => Syntax}/ScalarPropertyEnricher.cs (59%) delete mode 100644 src/SeqCli/Syntax/SeqCliNameResolver.cs create mode 100644 src/SeqCli/Syntax/V1/TracingFunctions.cs delete mode 100644 src/SeqCli/Util/JsonNetDestructuringPolicy.cs create mode 100644 src/SeqCli/Util/JsonNodes.cs delete mode 100644 src/SeqCli/Util/LogEventPropertyFactory.cs create mode 100644 test/SeqCli.Tests/Ingestion/SerilogEventJsonTests.cs create mode 100644 test/SeqCli.Tests/PlainText/EventJsonBuilderTests.cs delete mode 100644 test/SeqCli.Tests/PlainText/LogEventBuilderTests.cs rename test/SeqCli.Tests/Support/{FixedLogEventReader.cs => FixedEventReader.cs} (73%) diff --git a/src/SeqCli/Apps/AppLoader.cs b/src/SeqCli/Apps/AppLoader.cs index c0a03ff5..143eb91a 100644 --- a/src/SeqCli/Apps/AppLoader.cs +++ b/src/SeqCli/Apps/AppLoader.cs @@ -29,12 +29,14 @@ class AppLoader : IDisposable readonly string _packageBinaryPath; // These are used for interop between the host process and the app. The - // app _must_ be able to load on the unified version. + // app _must_ be able to load on the unified version. Apps built against Seq.Syntax v1 + // bundle their own `Seq.Syntax.dll`, which loads side-by-side with the host's + // `Seq.Syntax.V2.dll`. readonly Assembly[] _contracts = [ typeof(SeqApp).Assembly, typeof(Log).Assembly, - typeof(SerilogExpression).Assembly + typeof(SeqExpression).Assembly ]; public AppLoader(string packageBinaryPath) diff --git a/src/SeqCli/Apps/Hosting/AppContainer.cs b/src/SeqCli/Apps/Hosting/AppContainer.cs index 8a58c2bd..2ba93fde 100644 --- a/src/SeqCli/Apps/Hosting/AppContainer.cs +++ b/src/SeqCli/Apps/Hosting/AppContainer.cs @@ -21,7 +21,6 @@ using Newtonsoft.Json.Linq; using Seq.Apps; using Seq.Apps.LogEvents; -using SeqCli.Mapping; using Serilog; using Serilog.Events; using Serilog.Formatting.Compact.Reader; @@ -143,7 +142,7 @@ LogEvent ReadSerilogEvent(string clef, out string eventId, out uint eventType) if (jobject.TryGetValue("@l", out var levelToken)) { jobject.Remove("@l"); - jobject.Add("@l", new JValue(LevelMapping.ToSerilogLevel(levelToken.Value()!).ToString())); + jobject.Add("@l", new JValue(SerilogLevelMapping.ToSerilogLevel(levelToken.Value()!).ToString())); } SanitizeTraceIdentifiers(jobject); diff --git a/src/SeqCli/Apps/Hosting/SerilogLevelMapping.cs b/src/SeqCli/Apps/Hosting/SerilogLevelMapping.cs new file mode 100644 index 00000000..f37a9f92 --- /dev/null +++ b/src/SeqCli/Apps/Hosting/SerilogLevelMapping.cs @@ -0,0 +1,41 @@ +// Copyright © Datalust and contributors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using SeqCli.Mapping; +using Serilog.Events; + +namespace SeqCli.Apps.Hosting; + +/// +/// Maps level names onto Serilog's level enum for hosted Seq apps relying on the older Serilog `LogEvent`-based +/// interface (newer apps should generally use raw JSON directly). +/// +static class SerilogLevelMapping +{ + public static LogEventLevel ToSerilogLevel(string level) + { + if (string.IsNullOrEmpty(level)) + return LogEventLevel.Information; + + return LevelMapping.ToFullLevelName(level) switch + { + "Trace" or "Verbose" => LogEventLevel.Verbose, + "Debug" => LogEventLevel.Debug, + "Warning" => LogEventLevel.Warning, + "Error" => LogEventLevel.Error, + "Fatal" or "Critical" or "Emergency" or "Alert" or "Panic" => LogEventLevel.Fatal, + _ => LogEventLevel.Information + }; + } +} diff --git a/src/SeqCli/Cli/Commands/IngestCommand.cs b/src/SeqCli/Cli/Commands/IngestCommand.cs index b965ddb3..ba81fa0a 100644 --- a/src/SeqCli/Cli/Commands/IngestCommand.cs +++ b/src/SeqCli/Cli/Commands/IngestCommand.cs @@ -14,18 +14,16 @@ using System; using System.Collections.Generic; +using System.Text.Json.Nodes; using System.Threading; using System.Threading.Tasks; using SeqCli.Api; using SeqCli.Cli.Features; using SeqCli.Config; using SeqCli.Ingestion; -using SeqCli.Mapping; using SeqCli.PlainText; using SeqCli.Syntax; using Serilog; -using Serilog.Core; -using Serilog.Events; namespace SeqCli.Cli.Commands; @@ -84,19 +82,19 @@ protected override async Task Run() { try { - var enrichers = new List(); - + var enrichers = new List(); + if (_level != null) - enrichers.Add(new ScalarPropertyEnricher(LevelMapping.SurrogateLevelProperty, _level)); - + enrichers.Add(new LevelEnricher(_level)); + foreach (var (name, value) in _properties.FlatProperties) enrichers.Add(new ScalarPropertyEnricher(name, value)); - Func? filter = null; + Func? filter = null; if (_filter != null) { var eval = SeqSyntax.CompileExpression(_filter); - filter = evt => Seq.Syntax.Expressions.ExpressionResult.IsTrue(eval(evt)); + filter = evt => eval(evt).IsTrue(); } var config = RuntimeConfigurationLoader.Load(_storagePath); @@ -112,9 +110,9 @@ protected override async Task Run() { using (input) { - ILogEventReader reader = _json - ? new JsonLogEventReader(input) - : new PlainTextLogEventReader(input, _pattern); + IEventReader reader = _json + ? new JsonEventReader(input) + : new PlainTextEventReader(input, _pattern); reader = new EnrichingReader(reader, enrichers); diff --git a/src/SeqCli/Cli/Commands/PrintCommand.cs b/src/SeqCli/Cli/Commands/PrintCommand.cs index 2740297f..0bc67f59 100644 --- a/src/SeqCli/Cli/Commands/PrintCommand.cs +++ b/src/SeqCli/Cli/Commands/PrintCommand.cs @@ -14,16 +14,16 @@ using System; using System.IO; +using System.Text.Json; +using System.Text.Json.Nodes; using System.Threading.Tasks; -using Newtonsoft.Json; -using Seq.Syntax.Expressions; using SeqCli.Cli.Features; using SeqCli.Config; using SeqCli.Ingestion; using SeqCli.Output; +using SeqCli.Syntax; using SeqCli.Util; using Serilog; -using Serilog.Events; namespace SeqCli.Cli.Commands; @@ -61,16 +61,16 @@ protected override async Task Run() { var config = RuntimeConfigurationLoader.Load(_storage); - Func? filter = null; + Func? filter = null; if (_filter != null) { - if (!SerilogExpression.TryCompile(_filter, out var compiled, out var error)) + if (!SeqSyntax.TryCompileExpression(_filter, out var compiled, out var error)) { Log.Error("The specified filter could not be compiled: {Error}", error); return 1; } - filter = evt => ExpressionResult.IsTrue(compiled(evt)); + filter = evt => compiled(evt).IsTrue(); } var template = _template == null ? null : PrintTemplate.InterpretEscapeChars(_template); @@ -80,7 +80,7 @@ protected override async Task Run() { using (input) { - var reader = new JsonLogEventReader(input); + var reader = new JsonEventReader(input); var isAtEnd = false; do @@ -90,12 +90,12 @@ protected override async Task Run() var result = await reader.TryReadAsync(); isAtEnd = result.IsAtEnd; - if (result.LogEvent != null && (filter == null || filter(result.LogEvent))) - output.WriteLogEvent(result.LogEvent); + if (result.Document != null && (filter == null || filter(result.Document))) + output.WriteEvent(result.Document); } catch (Exception ex) { - if (ex is not JsonReaderException && ex is not InvalidDataException || + if (ex is not JsonException && ex is not InvalidDataException || _invalidDataHandlingFeature.InvalidDataHandling != InvalidDataHandling.Ignore) throw; } diff --git a/src/SeqCli/Cli/Commands/TraceCommand.cs b/src/SeqCli/Cli/Commands/TraceCommand.cs index 6e255069..67541f3b 100644 --- a/src/SeqCli/Cli/Commands/TraceCommand.cs +++ b/src/SeqCli/Cli/Commands/TraceCommand.cs @@ -151,8 +151,8 @@ protected override async Task Run() } else { - foreach (var logEvent in TraceFormatter.ToLogEvents(subtreeRoot != null ? [subtreeRoot] : roots)) - output.WriteLogEvent(logEvent); + foreach (var eventJson in TraceFormatter.ToEventJson(subtreeRoot != null ? [subtreeRoot] : roots)) + output.WriteEvent(eventJson); } return 0; diff --git a/src/SeqCli/Csv/CsvWriter.cs b/src/SeqCli/Csv/CsvWriter.cs index 75f6553a..f87ae2a3 100644 --- a/src/SeqCli/Csv/CsvWriter.cs +++ b/src/SeqCli/Csv/CsvWriter.cs @@ -1,22 +1,34 @@ using System; -using System.Collections.Generic; using System.IO; using Seq.Api.Model.Data; +using Seq.Syntax.Templates.Themes; using SeqCli.Mcp.Data; -using SeqCli.Output; -using Serilog.Templates.Themes; namespace SeqCli.Csv; static class CsvWriter { + // Delimited output is written directly rather than rendered through a template, so styled + // runs are opened and closed here. + static void SetStyle(TextWriter output, TemplateTheme? theme, TemplateThemeStyle style) + { + if (theme?.Open(style) is { } open) + output.Write(open); + } + + static void ResetStyle(TextWriter output, TemplateTheme? theme, TemplateThemeStyle style) + { + if (theme?.Close(style) is { } close) + output.Write(close); + } + public static void WriteQueryResult(QueryResultPart result, Func stringify, TemplateTheme? theme, TextWriter output) { if (!string.IsNullOrWhiteSpace(result.Error)) { - theme?.Set(output, TemplateThemeStyle.Text); + SetStyle(output, theme, TemplateThemeStyle.Text); QueryResultHelper.WriteErrorResult(output, result); - theme?.Reset(output); + ResetStyle(output, theme, TemplateThemeStyle.Text); } var first = true; @@ -40,39 +52,39 @@ static void WriteCell(TextWriter output, TemplateTheme? theme, object? value, Fu } else { - theme?.Set(output, TemplateThemeStyle.TertiaryText); + SetStyle(output, theme, TemplateThemeStyle.TertiaryText); output.Write(','); - theme?.Reset(output); + ResetStyle(output, theme, TemplateThemeStyle.TertiaryText); } - - theme?.Set(output, TemplateThemeStyle.TertiaryText); + + SetStyle(output, theme, TemplateThemeStyle.TertiaryText); output.Write('"'); - theme?.Reset(output); + ResetStyle(output, theme, TemplateThemeStyle.TertiaryText); var valueAsString = stringify(value); - + var dataStyle = isHeadingRow ? TemplateThemeStyle.Name : TemplateThemeStyle.Text; var doubleQuote = valueAsString.IndexOf('"'); while (doubleQuote != -1) { - theme?.Set(output, dataStyle); + SetStyle(output, theme, dataStyle); output.Write(valueAsString[..doubleQuote]); - theme?.Reset(output); - - theme?.Set(output, TemplateThemeStyle.Scalar); + ResetStyle(output, theme, dataStyle); + + SetStyle(output, theme, TemplateThemeStyle.Scalar); output.Write("\"\""); - theme?.Reset(output); + ResetStyle(output, theme, TemplateThemeStyle.Scalar); valueAsString = valueAsString[(doubleQuote + 1)..]; doubleQuote = valueAsString.IndexOf('"'); } - - theme?.Set(output, dataStyle); + + SetStyle(output, theme, dataStyle); output.Write(valueAsString); - theme?.Reset(output); - - theme?.Set(output, TemplateThemeStyle.TertiaryText); + ResetStyle(output, theme, dataStyle); + + SetStyle(output, theme, TemplateThemeStyle.TertiaryText); output.Write('"'); - theme?.Reset(output); + ResetStyle(output, theme, TemplateThemeStyle.TertiaryText); } } \ No newline at end of file diff --git a/src/SeqCli/Forwarder/ForwarderModule.cs b/src/SeqCli/Forwarder/ForwarderModule.cs index 6bb7ef67..3980787d 100644 --- a/src/SeqCli/Forwarder/ForwarderModule.cs +++ b/src/SeqCli/Forwarder/ForwarderModule.cs @@ -21,9 +21,8 @@ using SeqCli.Forwarder.Channel; using SeqCli.Forwarder.Web.Api; using SeqCli.Forwarder.Web.Host; +using SeqCli.Syntax; using Serilog; -using Serilog.Formatting; -using Serilog.Templates; namespace SeqCli.Forwarder; @@ -68,17 +67,17 @@ protected override void Load(ContainerBuilder builder) Log.ForContext().Warning("Configured to expose ingestion log via HTTP API"); builder.RegisterType().As(); - var ingestionLogTemplate = $"[{{@t:o}} {{@l:u3}}] {{@m}}{Environment.NewLine}"; + var ingestionLogTemplate = $"[{{@Timestamp:o}} {{@Level:u3}}] {{@Message}}{Environment.NewLine}"; if (_config.Forwarder.Diagnostics.IngestionLogShowDetail) { Log.ForContext().Warning("Including full client, payload, and error detail in the ingestion log"); ingestionLogTemplate += $"{{#if ClientHostIP is not null}}Client IP address: {{ClientHostIP}}{Environment.NewLine}{{#end}}" + $"{{#if DocumentStart is not null}}First {{StartToLog}} characters of payload: {{DocumentStart:l}}{Environment.NewLine}{{#end}}" + - "{@x}"; + "{@Exception}"; } - builder.Register(_ => new ExpressionTemplate(ingestionLogTemplate)).As(); + builder.Register(_ => SeqSyntax.ParseTemplate(ingestionLogTemplate)); } builder.Register(c => diff --git a/src/SeqCli/Forwarder/Web/Api/IngestionLogEndpoints.cs b/src/SeqCli/Forwarder/Web/Api/IngestionLogEndpoints.cs index cf30acfb..eb98cc58 100644 --- a/src/SeqCli/Forwarder/Web/Api/IngestionLogEndpoints.cs +++ b/src/SeqCli/Forwarder/Web/Api/IngestionLogEndpoints.cs @@ -16,17 +16,18 @@ using System.Text; using Microsoft.AspNetCore.Builder; using Microsoft.AspNetCore.Http; +using Seq.Syntax.Templates; using SeqCli.Forwarder.Diagnostics; -using Serilog.Formatting; +using SeqCli.Ingestion; namespace SeqCli.Forwarder.Web.Api; class IngestionLogEndpoints : IMapEndpoints { - readonly ITextFormatter _formatter; + readonly ExpressionTemplate _formatter; readonly Encoding _utf8 = new UTF8Encoding(false); - public IngestionLogEndpoints(ITextFormatter formatter) + public IngestionLogEndpoints(ExpressionTemplate formatter) { _formatter = formatter; } @@ -45,7 +46,7 @@ public void MapEndpoints(WebApplication app) using var log = new StringWriter(); foreach (var logEvent in events) { - _formatter.Format(logEvent, log); + _formatter.Format(SerilogEventJson.ToEventJson(logEvent), log); } return Results.Content(log.ToString(), "text/plain", _utf8); diff --git a/src/SeqCli/Ingestion/BatchResult.cs b/src/SeqCli/Ingestion/BatchResult.cs index 0c4b52ec..daef0697 100644 --- a/src/SeqCli/Ingestion/BatchResult.cs +++ b/src/SeqCli/Ingestion/BatchResult.cs @@ -1,15 +1,15 @@ -using Serilog.Events; +using System.Text.Json.Nodes; namespace SeqCli.Ingestion; struct BatchResult { - public LogEvent[] LogEvents { get; } + public JsonObject[] Documents { get; } public bool IsLast { get; } - public BatchResult(LogEvent[] logEvents, bool isLast) + public BatchResult(JsonObject[] documents, bool isLast) { - LogEvents = logEvents; + Documents = documents; IsLast = isLast; } -} \ No newline at end of file +} diff --git a/src/SeqCli/Ingestion/EnrichingReader.cs b/src/SeqCli/Ingestion/EnrichingReader.cs index 198ab234..63a25ca3 100644 --- a/src/SeqCli/Ingestion/EnrichingReader.cs +++ b/src/SeqCli/Ingestion/EnrichingReader.cs @@ -1,18 +1,18 @@ using System; using System.Collections.Generic; using System.Threading.Tasks; -using Serilog.Core; +using SeqCli.Syntax; namespace SeqCli.Ingestion; -class EnrichingReader : ILogEventReader +class EnrichingReader : IEventReader { - readonly ILogEventReader _inner; - readonly IReadOnlyCollection _enrichers; + readonly IEventReader _inner; + readonly IReadOnlyCollection _enrichers; public EnrichingReader( - ILogEventReader inner, - IReadOnlyCollection enrichers) + IEventReader inner, + IReadOnlyCollection enrichers) { _inner = inner ?? throw new ArgumentNullException(nameof(inner)); _enrichers = enrichers ?? throw new ArgumentNullException(nameof(enrichers)); @@ -22,13 +22,12 @@ public async Task TryReadAsync() { var result = await _inner.TryReadAsync(); - if (result.LogEvent != null) + if (result.Document != null) { foreach (var enricher in _enrichers) - // We're breaking the nullability contract of `ILogEventEnricher.Enrich()`, here. - enricher.Enrich(result.LogEvent, null!); + enricher.Enrich(result.Document); } return result; } -} \ No newline at end of file +} diff --git a/src/SeqCli/Ingestion/ILogEventReader.cs b/src/SeqCli/Ingestion/IEventReader.cs similarity index 79% rename from src/SeqCli/Ingestion/ILogEventReader.cs rename to src/SeqCli/Ingestion/IEventReader.cs index a92b09b9..0ca24530 100644 --- a/src/SeqCli/Ingestion/ILogEventReader.cs +++ b/src/SeqCli/Ingestion/IEventReader.cs @@ -2,7 +2,7 @@ namespace SeqCli.Ingestion; -interface ILogEventReader +interface IEventReader { Task TryReadAsync(); } \ No newline at end of file diff --git a/src/SeqCli/Ingestion/JsonEventReader.cs b/src/SeqCli/Ingestion/JsonEventReader.cs new file mode 100644 index 00000000..b7a102f6 --- /dev/null +++ b/src/SeqCli/Ingestion/JsonEventReader.cs @@ -0,0 +1,64 @@ +// Copyright © Datalust and contributors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Globalization; +using System.IO; +using System.Text.Json.Nodes; +using System.Threading.Tasks; +using SeqCli.PlainText.Framing; +using Superpower; +using Superpower.Model; + +namespace SeqCli.Ingestion; + +class JsonEventReader : IEventReader +{ + static readonly TimeSpan TrailingLineArrivalDeadline = TimeSpan.FromMilliseconds(10); + + readonly FrameReader _reader; + + public JsonEventReader(TextReader input) + { + _reader = new FrameReader( + input ?? throw new ArgumentNullException(nameof(input)), + Parse.Return(TextSpan.None), + TrailingLineArrivalDeadline); + } + + public async Task TryReadAsync() + { + var frame = await _reader.TryReadAsync(); + if (!frame.HasValue) + return new ReadResult(null, frame.IsAtEnd); + + if (frame.IsOrphan) + throw new InvalidDataException($"A line arrived late or could not be parsed: `{frame.Value.Trim()}`."); + + return new ReadResult(ReadFromJson(frame.Value), frame.IsAtEnd); + } + + public static JsonObject ReadFromJson(string json) + { + if (JsonNode.Parse(json) is not JsonObject eventJson) + throw new InvalidDataException($"The line is not a JSON object: `{json.Trim()}`."); + + if (!eventJson.ContainsKey("@t")) + eventJson["@t"] = DateTime.UtcNow.ToString("O", CultureInfo.InvariantCulture); + + SerilogTracingConventions.LiftSpanProperties(eventJson); + + return eventJson; + } +} diff --git a/src/SeqCli/Ingestion/JsonLogEventReader.cs b/src/SeqCli/Ingestion/JsonLogEventReader.cs deleted file mode 100644 index 719da10c..00000000 --- a/src/SeqCli/Ingestion/JsonLogEventReader.cs +++ /dev/null @@ -1,94 +0,0 @@ -// Copyright © Datalust and contributors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -using System; -using System.Globalization; -using System.IO; -using System.Threading.Tasks; -using Newtonsoft.Json; -using Newtonsoft.Json.Linq; -using SeqCli.Mapping; -using SeqCli.PlainText.Framing; -using Serilog.Events; -using Serilog.Formatting.Compact.Reader; -using Superpower; -using Superpower.Model; - -namespace SeqCli.Ingestion; - -class JsonLogEventReader : ILogEventReader -{ - static readonly TimeSpan TrailingLineArrivalDeadline = TimeSpan.FromMilliseconds(10); - static readonly JsonSerializer _serializer = JsonSerializer.Create(new JsonSerializerSettings - { - DateParseHandling = DateParseHandling.None, - Culture = CultureInfo.InvariantCulture - }); - - readonly FrameReader _reader; - - public JsonLogEventReader(TextReader input) - { - _reader = new FrameReader( - input ?? throw new ArgumentNullException(nameof(input)), - Parse.Return(TextSpan.None), - TrailingLineArrivalDeadline); - } - - public async Task TryReadAsync() - { - var frame = await _reader.TryReadAsync(); - if (!frame.HasValue) - return new ReadResult(null, frame.IsAtEnd); - - if (frame.IsOrphan) - throw new InvalidDataException($"A line arrived late or could not be parsed: `{frame.Value.Trim()}`."); - - var frameValue = new JsonTextReader(new StringReader(frame.Value)); - if (!(_serializer.Deserialize(frameValue) is JObject jobject)) - throw new InvalidDataException($"The line is not a JSON object: `{frame.Value.Trim()}`."); - - var evt = ReadFromJObject(jobject); - return new ReadResult(evt, frame.IsAtEnd); - } - - public static LogEvent ReadFromJson(string json) - { - var frameValue = new JsonTextReader(new StringReader(json)); - if (_serializer.Deserialize(frameValue) is not JObject jObject) - throw new InvalidDataException($"The line is not a JSON object: `{json.Trim()}`."); - - return ReadFromJObject(jObject); - } - - static LogEvent ReadFromJObject(JObject jObject) - { - if (!jObject.TryGetValue("@t", out _)) - jObject.Add("@t", new JValue(DateTime.UtcNow.ToString("O"))); - - if (jObject.TryGetValue("@l", out var levelToken)) - { - var originalLevel = levelToken.Value()!; - jObject.Remove("@l"); - - var serilogLevel = LevelMapping.ToSerilogLevel(originalLevel); - if (serilogLevel != LogEventLevel.Information) - jObject.Add("@l", new JValue(serilogLevel.ToString())); - - jObject.Add(LevelMapping.SurrogateLevelProperty, originalLevel); - } - - return LogEventReader.ReadFromJObject(jObject); - } -} \ No newline at end of file diff --git a/src/SeqCli/Ingestion/LogShipper.cs b/src/SeqCli/Ingestion/LogShipper.cs index f0a19741..68674fee 100644 --- a/src/SeqCli/Ingestion/LogShipper.cs +++ b/src/SeqCli/Ingestion/LogShipper.cs @@ -19,22 +19,18 @@ using System.Net.Http; using System.Net.Http.Headers; using System.Text; +using System.Text.Json.Nodes; using System.Threading; using System.Threading.Tasks; using Newtonsoft.Json; using Seq.Api; using SeqCli.Api; -using SeqCli.Output; using Serilog; -using Serilog.Events; -using Serilog.Formatting; namespace SeqCli.Ingestion; static class LogShipper { - static readonly ITextFormatter JsonFormatter = TextFormatters.Json(null); - public static async Task ShipBufferAsync( SeqConnection connection, string? apiKey, @@ -49,7 +45,7 @@ public static async Task ShipBufferAsync( ContentType = new MediaTypeHeaderValue(ApiConstants.ClefMediaType, "utf-8") } }; - + var retries = 0; while (true) { @@ -87,22 +83,22 @@ public static async Task ShipBufferAsync( { sendFailureLog.Error(ex, "Failed to ship a batch"); } - + var millisecondsDelay = (int)Math.Min(Math.Pow(2, retries) * 2000, 60000); sendFailureLog.Information("Backing off connection schedule; will retry in {MillisecondsDelay}", millisecondsDelay); await Task.Delay(millisecondsDelay, cancellationToken); retries += 1; } } - + public static async Task ShipEventsAsync( SeqConnection connection, string? apiKey, - ILogEventReader reader, + IEventReader reader, InvalidDataHandling invalidDataHandling, SendFailureHandling sendFailureHandling, int batchSize, - Func? filter, + Func? filter, CancellationToken cancellationToken) { const int maxEmptyBatchWaitMS = 2000; @@ -116,10 +112,10 @@ public static async Task ShipEventsAsync( var statusCode = await SendBatchAsync( connection, apiKey, - batch.LogEvents, + batch.Documents, sendFailureHandling != SendFailureHandling.Ignore ? Log.Logger : null, cancellationToken); - + sendSucceeded = (int)statusCode is >= 200 and < 300; } catch (Exception ex) @@ -146,7 +142,7 @@ public static async Task ShipEventsAsync( if (batch.IsLast) break; - + batch = await ReadBatchAsync(reader, filter, batchSize, invalidDataHandling, maxEmptyBatchWaitMS); } @@ -154,15 +150,15 @@ public static async Task ShipEventsAsync( } static async Task ReadBatchAsync( - ILogEventReader reader, - Func? filter, + IEventReader reader, + Func? filter, int count, InvalidDataHandling invalidDataHandling, int maxWaitMS) { - var batch = new List(); + var batch = new List(); var isLast = false; - + // Avoid consuming stacks of CPU unnecessarily when there's no work to do. We do eventually yield // an empty batch, because level switching relies on this. var totalWaitMS = 0; @@ -175,7 +171,7 @@ static async Task ReadBatchAsync( { var rr = await reader.TryReadAsync(); isLast = rr.IsAtEnd; - var evt = rr.LogEvent; + var evt = rr.Document; if (evt == null) { if (isLast || batch.Count != 0 || totalWaitMS > maxWaitMS) @@ -195,7 +191,7 @@ static async Task ReadBatchAsync( } catch (Exception ex) { - if (ex is JsonReaderException || ex is InvalidDataException) + if (ex is System.Text.Json.JsonException || ex is InvalidDataException) { if (invalidDataHandling == InvalidDataHandling.Ignore) continue; @@ -211,7 +207,7 @@ static async Task ReadBatchAsync( static async Task SendBatchAsync( SeqConnection connection, string? apiKey, - IReadOnlyCollection batch, + IReadOnlyCollection batch, ILogger? sendFailureLog, CancellationToken cancellationToken) { @@ -223,7 +219,7 @@ static async Task SendBatchAsync( using (var builder = new StringWriter()) { foreach (var evt in batch) - JsonFormatter.Format(evt, builder); + builder.WriteLine(evt.ToJsonString()); content = new StringContent(builder.ToString(), Encoding.UTF8, ApiConstants.ClefMediaType); } @@ -264,4 +260,4 @@ static async Task SendAsync(SeqConnection connection, string? ap sendFailureLog.Error("Shipping failed with status code {StatusCode} ({ReasonPhrase})", result.StatusCode, result.ReasonPhrase); return result.StatusCode; } -} \ No newline at end of file +} diff --git a/src/SeqCli/Ingestion/ReadResult.cs b/src/SeqCli/Ingestion/ReadResult.cs index 87e10076..a44d30f4 100644 --- a/src/SeqCli/Ingestion/ReadResult.cs +++ b/src/SeqCli/Ingestion/ReadResult.cs @@ -1,15 +1,20 @@ -using Serilog.Events; +using System.Text.Json.Nodes; namespace SeqCli.Ingestion; readonly struct ReadResult { - public LogEvent? LogEvent { get; } + /// + /// The event, as a JSON document in Seq's emission schema, or null if no event + /// is available. + /// + public JsonObject? Document { get; } + public bool IsAtEnd { get; } - public ReadResult(LogEvent? logEvent, bool isAtEnd) + public ReadResult(JsonObject? document, bool isAtEnd) { - LogEvent = logEvent; + Document = document; IsAtEnd = isAtEnd; } -} \ No newline at end of file +} diff --git a/src/SeqCli/Ingestion/SerilogEventJson.cs b/src/SeqCli/Ingestion/SerilogEventJson.cs new file mode 100644 index 00000000..3a166419 --- /dev/null +++ b/src/SeqCli/Ingestion/SerilogEventJson.cs @@ -0,0 +1,91 @@ +// Copyright © Datalust and contributors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System.Globalization; +using System.Linq; +using System.Text.Json.Nodes; +using SeqCli.Syntax; +using Serilog.Events; + +namespace SeqCli.Ingestion; + +/// +/// Converts Serilog events produced within seqcli itself — the sample ingest simulation +/// and the forwarder's diagnostic ingestion log — into event JSON documents in Seq's emission +/// schema. Externally-supplied event data never passes through here: it's read directly into +/// JSON documents. +/// +static class SerilogEventJson +{ + public static JsonObject ToEventJson(LogEvent logEvent) + { + var eventJson = new JsonObject + { + ["@t"] = logEvent.Timestamp.ToString("o", CultureInfo.InvariantCulture), + ["@mt"] = logEvent.MessageTemplate.Text + }; + + if (logEvent.Level != LogEventLevel.Information) + eventJson["@l"] = logEvent.Level.ToString(); + + if (logEvent.Exception != null) + eventJson["@x"] = logEvent.Exception.ToString(); + + if (logEvent.TraceId is { } traceId) + eventJson["@tr"] = traceId.ToHexString(); + + if (logEvent.SpanId is { } spanId) + eventJson["@sp"] = spanId.ToHexString(); + + foreach (var (name, value) in logEvent.Properties) + EventJson.SetUserProperty(eventJson, name, ToJsonNode(value)); + + SerilogTracingConventions.LiftSpanProperties(eventJson); + + return eventJson; + } + + public static JsonNode? ToJsonNode(LogEventPropertyValue value) + { + switch (value) + { + case ScalarValue scalar: + return EventJson.CreateScalar(scalar.Value); + + case SequenceValue sequence: + return new JsonArray(sequence.Elements.Select(ToJsonNode).ToArray()); + + case StructureValue structure: + { + var result = new JsonObject(); + foreach (var property in structure.Properties) + result[property.Name] = ToJsonNode(property.Value); + if (structure.TypeTag != null) + result["$type"] = structure.TypeTag; + return result; + } + + case DictionaryValue dictionary: + { + var result = new JsonObject(); + foreach (var (key, element) in dictionary.Elements) + result[key.Value?.ToString() ?? "null"] = ToJsonNode(element); + return result; + } + + default: + return EventJson.CreateScalar(value.ToString()); + } + } +} diff --git a/src/SeqCli/Ingestion/SerilogTracingConventions.cs b/src/SeqCli/Ingestion/SerilogTracingConventions.cs new file mode 100644 index 00000000..e6eb029e --- /dev/null +++ b/src/SeqCli/Ingestion/SerilogTracingConventions.cs @@ -0,0 +1,46 @@ +// Copyright © Datalust and contributors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System.Text.Json.Nodes; + +namespace SeqCli.Ingestion; + +/// +/// SerilogTracing emits span fields as regular event properties, because Serilog's data model +/// has nowhere else to put them. Events passing through seqcli lift these into the reified +/// @st and @ps fields so that they're recognized as spans by Seq and by seqcli's +/// own output formatting. +/// +static class SerilogTracingConventions +{ + internal const string ParentSpanIdProperty = "ParentSpanId"; + + internal const string SpanStartTimestampProperty = "SpanStartTimestamp"; + + public static void LiftSpanProperties(JsonObject eventJson) + { + LiftProperty(eventJson, SpanStartTimestampProperty, "@st"); + LiftProperty(eventJson, ParentSpanIdProperty, "@ps"); + } + + static void LiftProperty(JsonObject eventJson, string propertyName, string reifiedName) + { + if (eventJson.TryGetPropertyValue(propertyName, out var value)) + { + eventJson.Remove(propertyName); + if (!eventJson.ContainsKey(reifiedName)) + eventJson[reifiedName] = value; + } + } +} diff --git a/src/SeqCli/Ingestion/StaticMessageTemplateReader.cs b/src/SeqCli/Ingestion/StaticMessageTemplateReader.cs index 973bff60..d5d62591 100644 --- a/src/SeqCli/Ingestion/StaticMessageTemplateReader.cs +++ b/src/SeqCli/Ingestion/StaticMessageTemplateReader.cs @@ -1,37 +1,29 @@ using System; -using System.Linq; using System.Threading.Tasks; -using SeqCli.Util; -using Serilog.Events; -using Serilog.Parsing; namespace SeqCli.Ingestion; -class StaticMessageTemplateReader : ILogEventReader +class StaticMessageTemplateReader : IEventReader { - readonly ILogEventReader _inner; - readonly MessageTemplate _messageTemplate; + readonly IEventReader _inner; + readonly string _messageTemplate; - public StaticMessageTemplateReader(ILogEventReader inner, string messageTemplate) + public StaticMessageTemplateReader(IEventReader inner, string messageTemplate) { _inner = inner ?? throw new ArgumentNullException(nameof(inner)); - _messageTemplate = new MessageTemplateParser().Parse(messageTemplate); + _messageTemplate = messageTemplate ?? throw new ArgumentNullException(nameof(messageTemplate)); } public async Task TryReadAsync() { var result = await _inner.TryReadAsync(); - if (result.LogEvent == null) - return result; + if (result.Document != null) + { + result.Document.Remove("@m"); + result.Document["@mt"] = _messageTemplate; + } - var evt = new LogEvent( - result.LogEvent.Timestamp, - result.LogEvent.Level, - result.LogEvent.Exception, - _messageTemplate, - result.LogEvent.Properties.Select(kv => LogEventPropertyFactory.SafeCreate(kv.Key, kv.Value))); - - return new ReadResult(evt, result.IsAtEnd); + return result; } -} \ No newline at end of file +} diff --git a/src/SeqCli/Ingestion/TraceConstants.cs b/src/SeqCli/Ingestion/TraceConstants.cs deleted file mode 100644 index 55fe7f34..00000000 --- a/src/SeqCli/Ingestion/TraceConstants.cs +++ /dev/null @@ -1,8 +0,0 @@ -namespace SeqCli.Ingestion; - -static class TraceConstants -{ - internal const string ParentSpanIdProperty = "ParentSpanId"; - - internal const string SpanStartTimestampProperty = "SpanStartTimestamp"; -} diff --git a/src/SeqCli/Mapping/EventEntityJson.cs b/src/SeqCli/Mapping/EventEntityJson.cs new file mode 100644 index 00000000..e84bb3c9 --- /dev/null +++ b/src/SeqCli/Mapping/EventEntityJson.cs @@ -0,0 +1,104 @@ +// Copyright © Datalust Pty Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Text; +using System.Text.Json.Nodes; +using Seq.Api.Model.Events; +using Seq.Api.Model.Shared; +using SeqCli.Syntax; +using SeqCli.Util; + +namespace SeqCli.Mapping; + +/// +/// Converts events retrieved from the Seq API into event JSON documents in Seq's emission +/// (CLEF) schema, ready for filtering and formatting with Seq.Syntax. +/// +static class EventEntityJson +{ + public static JsonObject ToEventJson(EventEntity evt) + { + // Timestamps are shown in local time, matching earlier seqcli versions. + var eventJson = new JsonObject + { + ["@t"] = DateTimeOffset.ParseExact(evt.Timestamp, "o", CultureInfo.InvariantCulture) + .ToLocalTime().ToString("o", CultureInfo.InvariantCulture) + }; + + if (evt.MessageTemplateTokens != null) + eventJson["@mt"] = ToMessageTemplateText(evt.MessageTemplateTokens); + + // By the emission convention, `Information` levels are omitted; any other level keeps + // the spelling it was ingested with. + if (!string.IsNullOrWhiteSpace(evt.Level) && evt.Level != "Information") + eventJson["@l"] = evt.Level; + + if (!string.IsNullOrWhiteSpace(evt.Exception)) + eventJson["@x"] = evt.Exception; + + if (!string.IsNullOrWhiteSpace(evt.TraceId)) + eventJson["@tr"] = evt.TraceId; + + if (!string.IsNullOrWhiteSpace(evt.SpanId)) + eventJson["@sp"] = evt.SpanId; + + if (!string.IsNullOrWhiteSpace(evt.ParentId)) + eventJson["@ps"] = evt.ParentId; + + if (!string.IsNullOrWhiteSpace(evt.Start)) + eventJson["@st"] = evt.Start; + + if (!string.IsNullOrWhiteSpace(evt.SpanKind)) + eventJson["@sk"] = evt.SpanKind; + + if (evt.Resource?.Count > 0) + eventJson["@ra"] = ToPropertiesObject(evt.Resource); + + if (evt.Scope?.Count > 0) + eventJson["@sa"] = ToPropertiesObject(evt.Scope); + + if (evt.Properties != null) + { + foreach (var property in evt.Properties) + EventJson.SetUserProperty(eventJson, property.Name, JsonNodes.FromApiValue(property.Value)); + } + + return eventJson; + } + + static string ToMessageTemplateText(List tokens) + { + var text = new StringBuilder(); + foreach (var token in tokens) + { + if (token.Text != null) + text.Append(token.Text.Replace("{", "{{").Replace("}", "}}")); + else + text.Append(token.RawText ?? $"{{{token.PropertyName}}}"); + } + + return text.ToString(); + } + + static JsonObject ToPropertiesObject(List properties) + { + var result = new JsonObject(); + foreach (var property in properties) + result[property.Name] = JsonNodes.FromApiValue(property.Value); + return result; + } +} diff --git a/src/SeqCli/Mapping/LevelMapping.cs b/src/SeqCli/Mapping/LevelMapping.cs index ff79087b..7faa47b4 100644 --- a/src/SeqCli/Mapping/LevelMapping.cs +++ b/src/SeqCli/Mapping/LevelMapping.cs @@ -1,4 +1,4 @@ -// Copyright © Datalust and contributors. +// Copyright © Datalust and contributors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -14,80 +14,74 @@ using System; using System.Collections.Generic; -using Serilog.Events; namespace SeqCli.Mapping; +/// +/// Recognizes the level spellings found in event data from various sources (info, +/// WARN, trce, …) and maps them to canonical Seq level names. Level values +/// themselves are preserved verbatim throughout the pipeline; the canonical name is used +/// where a normalized form is needed. +/// public static class LevelMapping { - // Use a "hygienic" name for the original level value to avoid collisions - internal static readonly string SurrogateLevelProperty = $"_SeqcliOriginalLevel_{Guid.NewGuid():N}"; - - static readonly Dictionary LevelsByName = + static readonly Dictionary LevelsByName = new(StringComparer.OrdinalIgnoreCase) { - ["t"] = ("Trace", LogEventLevel.Verbose), - ["tr"] = ("Trace", LogEventLevel.Verbose), - ["trc"] = ("Trace", LogEventLevel.Verbose), - ["trce"] = ("Trace", LogEventLevel.Verbose), - ["trace"] = ("Trace", LogEventLevel.Verbose), - ["v"] = ("Verbose", LogEventLevel.Verbose), - ["ver"] = ("Verbose", LogEventLevel.Verbose), - ["vrb"] = ("Verbose", LogEventLevel.Verbose), - ["verb"] = ("Verbose", LogEventLevel.Verbose), - ["verbose"] = ("Verbose", LogEventLevel.Verbose), - ["d"] = ("Debug", LogEventLevel.Debug), - ["de"] = ("Debug", LogEventLevel.Debug), - ["dbg"] = ("Debug", LogEventLevel.Debug), - ["deb"] = ("Debug", LogEventLevel.Debug), - ["dbug"] = ("Debug", LogEventLevel.Debug), - ["debu"] = ("Debug", LogEventLevel.Debug), - ["debug"] = ("Debug", LogEventLevel.Debug), - ["i"] = ("Information", LogEventLevel.Information), - ["in"] = ("Information", LogEventLevel.Information), - ["inf"] = ("Information", LogEventLevel.Information), - ["info"] = ("Information", LogEventLevel.Information), - ["information"] = ("Information", LogEventLevel.Information), - ["notice"] = ("Notice", LogEventLevel.Information), - ["w"] = ("Warning", LogEventLevel.Warning), - ["wa"] = ("Warning", LogEventLevel.Warning), - ["war"] = ("Warning", LogEventLevel.Warning), - ["wrn"] = ("Warning", LogEventLevel.Warning), - ["warn"] = ("Warning", LogEventLevel.Warning), - ["warning"] = ("Warning", LogEventLevel.Warning), - ["e"] = ("Error", LogEventLevel.Error), - ["er"] = ("Error", LogEventLevel.Error), - ["err"] = ("Error", LogEventLevel.Error), - ["erro"] = ("Error", LogEventLevel.Error), - ["eror"] = ("Error", LogEventLevel.Error), - ["error"] = ("Error", LogEventLevel.Error), - ["f"] = ("Fatal", LogEventLevel.Fatal), - ["fa"] = ("Fatal", LogEventLevel.Fatal), - ["ftl"] = ("Fatal", LogEventLevel.Fatal), - ["fat"] = ("Fatal", LogEventLevel.Fatal), - ["fatl"] = ("Fatal", LogEventLevel.Fatal), - ["fatal"] = ("Fatal", LogEventLevel.Fatal), - ["c"] = ("Critical", LogEventLevel.Fatal), - ["cr"] = ("Critical", LogEventLevel.Fatal), - ["crt"] = ("Critical", LogEventLevel.Fatal), - ["cri"] = ("Critical", LogEventLevel.Fatal), - ["crit"] = ("Critical", LogEventLevel.Fatal), - ["critical"] = ("Critical", LogEventLevel.Fatal), - ["emerg"] = ("Emergency", LogEventLevel.Fatal), - ["alert"] = ("Alert", LogEventLevel.Fatal), - ["panic"] = ("Panic", LogEventLevel.Fatal) + ["t"] = "Trace", + ["tr"] = "Trace", + ["trc"] = "Trace", + ["trce"] = "Trace", + ["trace"] = "Trace", + ["v"] = "Verbose", + ["ver"] = "Verbose", + ["vrb"] = "Verbose", + ["verb"] = "Verbose", + ["verbose"] = "Verbose", + ["d"] = "Debug", + ["de"] = "Debug", + ["dbg"] = "Debug", + ["deb"] = "Debug", + ["dbug"] = "Debug", + ["debu"] = "Debug", + ["debug"] = "Debug", + ["i"] = "Information", + ["in"] = "Information", + ["inf"] = "Information", + ["info"] = "Information", + ["information"] = "Information", + ["notice"] = "Notice", + ["w"] = "Warning", + ["wa"] = "Warning", + ["war"] = "Warning", + ["wrn"] = "Warning", + ["warn"] = "Warning", + ["warning"] = "Warning", + ["e"] = "Error", + ["er"] = "Error", + ["err"] = "Error", + ["erro"] = "Error", + ["eror"] = "Error", + ["error"] = "Error", + ["f"] = "Fatal", + ["fa"] = "Fatal", + ["ftl"] = "Fatal", + ["fat"] = "Fatal", + ["fatl"] = "Fatal", + ["fatal"] = "Fatal", + ["c"] = "Critical", + ["cr"] = "Critical", + ["crt"] = "Critical", + ["cri"] = "Critical", + ["crit"] = "Critical", + ["critical"] = "Critical", + ["emerg"] = "Emergency", + ["alert"] = "Alert", + ["panic"] = "Panic" }; - public static LogEventLevel ToSerilogLevel(string level) - { - if (string.IsNullOrEmpty(level)) - return LogEventLevel.Information; - - return LevelsByName.TryGetValue(level, out var m) ? m.Item2 : LogEventLevel.Information; - } - public static string ToFullLevelName(string level) { - return LevelsByName.TryGetValue(level, out var m) ? m.Item1 : level; + return LevelsByName.TryGetValue(level, out var m) ? m : level; } -} \ No newline at end of file +} diff --git a/src/SeqCli/Mapping/MetricsMapping.cs b/src/SeqCli/Mapping/MetricsMapping.cs deleted file mode 100644 index fe104666..00000000 --- a/src/SeqCli/Mapping/MetricsMapping.cs +++ /dev/null @@ -1,8 +0,0 @@ -using System; - -namespace SeqCli.Mapping; - -public static class MetricsMapping -{ - internal static readonly string SurrogateDefinitionsProperty = $"_SeqcliMetricDefinitions_{Guid.NewGuid():N}"; -} diff --git a/src/SeqCli/Mcp/Tools/Search/SearchTools.cs b/src/SeqCli/Mcp/Tools/Search/SearchTools.cs index 7dc9671d..58a3b663 100644 --- a/src/SeqCli/Mcp/Tools/Search/SearchTools.cs +++ b/src/SeqCli/Mcp/Tools/Search/SearchTools.cs @@ -30,8 +30,8 @@ using SeqCli.Mapping; using SeqCli.Output; using SeqCli.Signals; +using SeqCli.Syntax; using Serilog; -using Serilog.Events; using NativeFormatter = SeqCli.Output.NativeFormatter; // ReSharper disable UnusedMember.Global @@ -42,8 +42,9 @@ namespace SeqCli.Mcp.Tools.Search; class SearchTools(McpSession session, SeqConnection connection) { const string ResultIdPropertyName = "__seqcli_ResultId"; - static readonly ExpressionTemplate SearchResultFormatter = new ( - $"{{{ResultIdPropertyName}}} [{{UtcDateTime(@t)}} {{{LevelMapping.SurrogateLevelProperty}}}] {{@m}}{Environment.NewLine}{{#if @x is not null}}{{Substring(ToString(@x), 0, 512)}}...{Environment.NewLine}{{#end}}" + static readonly ExpressionTemplate SearchResultFormatter = SeqSyntax.ParseTemplate( + $"{{{ResultIdPropertyName}}} [{{UtcDateTime(@Timestamp)}} {{@Level}}] {{@Message}}{Environment.NewLine}" + + $"{{#if @Exception is not null}}{{Substring(ToString(@Exception), 0, 512)}}...{Environment.NewLine}{{#end}}" ); [McpServerTool(Name = "seq_new_session", ReadOnly = true, Title = "Begin a new Search/Query Session")] @@ -181,12 +182,10 @@ public async Task SearchEventsAsync( foreach (var result in takenResults) { var resultId = session.ImportSearchResult(result); - - var serilogEvent = OutputFormat.ToSerilogEvent(result); - OutputFormat.FlattenPropertiesUsedWithDottedNames(result, serilogEvent); - serilogEvent.AddOrUpdateProperty(new LogEventProperty(ResultIdPropertyName, new ScalarValue(resultId))); - serilogEvent.AddOrUpdateProperty(new LogEventProperty(LevelMapping.SurrogateLevelProperty, new ScalarValue(result.Level ?? "Information"))); - SearchResultFormatter.Format(serilogEvent, responseText); + + var eventJson = EventEntityJson.ToEventJson(result); + eventJson[ResultIdPropertyName] = resultId; + SearchResultFormatter.Format(eventJson, responseText); } return new CallToolResult diff --git a/src/SeqCli/Output/FlareTheme.cs b/src/SeqCli/Output/FlareTheme.cs index 38d1e578..a1026fee 100644 --- a/src/SeqCli/Output/FlareTheme.cs +++ b/src/SeqCli/Output/FlareTheme.cs @@ -13,13 +13,12 @@ // limitations under the License. using System.Collections.Generic; -using System.IO; -using Serilog.Templates.Themes; +using Seq.Syntax.Templates.Themes; namespace SeqCli.Output; /// -/// Flare is Seq's embedded stream/columnar database. This theme is derived from one build originally +/// Flare is Seq's embedded stream/columnar database. This theme is derived from one built originally /// for the flaretl command-line tooling used there. /// static class FlareTheme @@ -45,26 +44,5 @@ static class FlareTheme [TemplateThemeStyle.LevelFatal] = "\e[38;5;0197m\e[48;5;0238m" }; - public static readonly TemplateTheme SeqCli = new(FlareThemeStyles); - - // `CsvWriter` implements its own theming behavior because the required APIs are not public in Serilog.Expressions. - // The best way forward for this is likely to be porting theming to Seq.Syntax, and exposing the required APIs there. - - const string AnsiStyleResetSequence = "\e[0m"; - - // The passed-in theme is ignored because SerilogExpressions themes are opaque. All formatting uses the SeqCli theme. - // ReSharper disable once UnusedParameter.Global - extension(TemplateTheme theme) - { - public void Set(TextWriter output, TemplateThemeStyle style) - { - if (FlareThemeStyles.TryGetValue(style, out var styleSequence)) - output.Write(styleSequence); - } - - public void Reset(TextWriter output) - { - output.Write(AnsiStyleResetSequence); - } - } -} \ No newline at end of file + public static readonly TemplateTheme SeqCli = new AnsiTheme(FlareThemeStyles); +} diff --git a/src/SeqCli/Output/OutputFormat.cs b/src/SeqCli/Output/OutputFormat.cs index 7b9c5ea7..af137838 100644 --- a/src/SeqCli/Output/OutputFormat.cs +++ b/src/SeqCli/Output/OutputFormat.cs @@ -15,24 +15,21 @@ using System; using System.Collections; using System.Collections.Generic; -using System.Diagnostics; using System.Globalization; -using System.Linq; +using System.Text.Json.Nodes; using Newtonsoft.Json; using Newtonsoft.Json.Converters; using Newtonsoft.Json.Linq; using Seq.Api.Model; using Seq.Api.Model.Data; using Seq.Api.Model.Events; +using Seq.Syntax.Templates; +using Seq.Syntax.Templates.Encoding; +using Seq.Syntax.Templates.Themes; using SeqCli.Config; using SeqCli.Csv; using SeqCli.Mapping; using SeqCli.Util; -using Serilog; -using Serilog.Core; -using Serilog.Events; -using Serilog.Parsing; -using Serilog.Templates.Themes; namespace SeqCli.Output; @@ -40,10 +37,10 @@ sealed class OutputFormat { // See https://no-color.org for semantics. const string NoColorEnvironmentVariable = "NO_COLOR"; - + readonly OutputSyntax _syntax; - readonly string? _plainTextTemplate; - readonly Logger _formatter; + readonly ExpressionTemplate? _eventFormatter; + readonly ExpressionTemplate _jsonValueFormatter; readonly JsonSerializer _serializer = JsonSerializer.CreateDefault(new JsonSerializerSettings { @@ -92,7 +89,6 @@ internal OutputFormat( bool allowAnsiEscapes) { _syntax = syntax; - _plainTextTemplate = plainTextTemplate; var resolvedNoColor = ResolveNoColor(noColor, forceColor, outputConfig, noColorSetInEnvironment, allowAnsiEscapes); var applyThemeToRedirectedOutput = !resolvedNoColor && (forceColor ?? outputConfig.ForceColor); @@ -102,12 +98,20 @@ internal OutputFormat( ? FlareTheme.SeqCli : null; - _formatter = CreateOutputLogger(); + _eventFormatter = Json + ? TextFormatters.Json(TemplateTheme) + : Text + ? TextFormatters.Plain(TemplateTheme, plainTextTemplate) + : null; + + _jsonValueFormatter = new ExpressionTemplate( + "{Value}" + Environment.NewLine, + encoder: TemplateTheme != null ? TemplateOutputEncoder.Ansi(TemplateTheme) : null); } static bool NoColorSetInEnvironment() => !string.IsNullOrEmpty(Environment.GetEnvironmentVariable(NoColorEnvironmentVariable)); - + internal static bool ResolveNoColor( bool? noColorFlag, bool? forceColorFlag, @@ -135,27 +139,6 @@ internal static bool ResolveNoColor( public bool RequiresRender => Native; - Logger CreateOutputLogger() - { - var outputConfiguration = new LoggerConfiguration() - .MinimumLevel.Is(LevelAlias.Minimum) - .Enrich.With(); - - if (Json) - { - outputConfiguration.WriteTo.Console(TextFormatters.Json(TemplateTheme)); - } - else if (Text) - { - outputConfiguration.WriteTo.Console(TextFormatters.Plain(TemplateTheme, _plainTextTemplate)); - } - - // The logger is not configured for Native output, which avoids it. Ideally we'll shift away from using - // Serilog here, and move Text/Json over to EventEntity-driven formatters, too. - - return outputConfiguration.CreateLogger(); - } - public void WriteEntity(Entity entity) { if (entity == null) throw new ArgumentNullException(nameof(entity)); @@ -163,18 +146,11 @@ public void WriteEntity(Entity entity) var jo = JObject.FromObject( entity, _serializer); - + if (Json) { jo.Remove("Links"); - - var writer = new LoggerConfiguration() - .Destructure.With() - .Destructure.ToMaximumDepth(10000) - .Enrich.With() - .WriteTo.Console(TextFormatters.Plain(TemplateTheme, "{@m}" + Environment.NewLine)) - .CreateLogger(); - writer.Information("{@Entity}", jo); + WriteJsonValue(JsonNodes.FromNewtonsoft(jo)); } else if (Text) { @@ -190,22 +166,14 @@ public void WriteEntity(Entity entity) public void WriteObject(object value) { if (value == null) throw new ArgumentNullException(nameof(value)); - + if (Json) { var jo = value is ICollection and not (IDictionary or JToken) ? (JToken)JArray.FromObject(value, _serializer) : JObject.FromObject(value, _serializer); - // Using the same method of JSON colorization as above - - var writer = new LoggerConfiguration() - .Destructure.With() - .Destructure.ToMaximumDepth(10000) - .Enrich.With() - .WriteTo.Console(TextFormatters.Plain(TemplateTheme, "{@m}" + Environment.NewLine)) - .CreateLogger(); - writer.Information("{@Entity}", jo); + WriteJsonValue(JsonNodes.FromNewtonsoft(jo)); } else if (Text) { @@ -218,6 +186,11 @@ public void WriteObject(object value) } } + void WriteJsonValue(JsonNode? value) + { + _jsonValueFormatter.Format(new JsonObject { ["Value"] = value }, Console.Out); + } + public void ListEntities(IEnumerable list) { foreach (var entity in list) @@ -225,7 +198,7 @@ public void ListEntities(IEnumerable list) WriteEntity(entity); } } - + // ReSharper disable once MemberCanBeMadeStatic.Global #pragma warning disable CA1822 public void WriteText(string? text) @@ -259,125 +232,15 @@ public void WriteEventEntity(EventEntity evt) } else { - var serilogEvent = ToSerilogEvent(evt); - - if (Text) - { - // Add flattened versions of structured properties that are referenced using dotted-name syntax in - // message templates, e.g. {user.name}. Serilog.Expressions template rendering doesn't otherwise - // support these. In text output mode, these aren't usually observable, though - // seqcli print --template="{@p}" will make them visible. - FlattenPropertiesUsedWithDottedNames(evt, serilogEvent); - } - - WriteLogEvent(serilogEvent); - } - } - - public void WriteLogEvent(LogEvent logEvent) - { - _formatter.Write(logEvent); - } - - public static LogEvent ToSerilogEvent(EventEntity evt) - { - ActivityTraceId traceId = default; - if (!string.IsNullOrWhiteSpace(evt.TraceId)) - traceId = ActivityTraceId.CreateFromString(evt.TraceId); - - ActivitySpanId spanId = default; - if (!string.IsNullOrWhiteSpace(evt.SpanId)) - spanId = ActivitySpanId.CreateFromString(evt.SpanId); - - var serilogEvent = new LogEvent( - DateTimeOffset.ParseExact(evt.Timestamp, "o", CultureInfo.InvariantCulture).ToLocalTime(), - LevelMapping.ToSerilogLevel(evt.Level), - string.IsNullOrWhiteSpace(evt.Exception) ? null : new TextException(evt.Exception), - new MessageTemplate(evt.MessageTemplateTokens.Select(ToMessageTemplateToken)), - evt.Properties - .Select(p => CreateProperty(p.Name, p.Value)), - traceId, - spanId - ); - - if (evt.Scope?.Count > 0) - serilogEvent.AddOrUpdateProperty(new("@sa", new StructureValue(evt.Scope.Select(p => CreateProperty(p.Name, p.Value))))); - - if (evt.Resource?.Count > 0) - serilogEvent.AddOrUpdateProperty(new("@ra", new StructureValue(evt.Resource.Select(p => CreateProperty(p.Name, p.Value))))); - - if (!string.IsNullOrWhiteSpace(evt.ParentId)) - serilogEvent.AddOrUpdateProperty(new("@ps", new ScalarValue(evt.ParentId))); - - if (!string.IsNullOrWhiteSpace(evt.Start)) - serilogEvent.AddOrUpdateProperty(new("@st", new ScalarValue(evt.Start))); - - if (!string.IsNullOrWhiteSpace(evt.SpanKind)) - serilogEvent.AddOrUpdateProperty(new("@sk", new ScalarValue(evt.SpanKind))); - - return serilogEvent; - } - - public static void FlattenPropertiesUsedWithDottedNames(EventEntity evt, LogEvent serilogEvent) - { - foreach (var token in evt.MessageTemplateTokens) - { - if (token.Text != null || token.PropertyName is not { } name || !name.Contains('.') || - serilogEvent.Properties.ContainsKey(name)) - { - continue; - } - - var steps = name.Split('.'); - var value = evt.Properties.FirstOrDefault(p => p.Name == steps[0])?.Value; - for (var i = 1; i < steps.Length; ++i) - { - value = (value as JObject)?.GetValue(steps[i]); - } - - if (value is JToken resolved) - { - // Existing flat-named properties, where present, win. - serilogEvent.AddPropertyIfAbsent(LogEventPropertyFactory.SafeCreate( - name, resolved is JValue scalar ? new ScalarValue(scalar.Value) : CreatePropertyValue(resolved))); - } + WriteEvent(EventEntityJson.ToEventJson(evt)); } } - static MessageTemplateToken ToMessageTemplateToken(MessageTemplateTokenPart token) - { - // Not ideal, we lose renderings, alignment etc. here. - - if (token.Text != null) - return new TextToken(token.Text); - return new PropertyToken(token.PropertyName, token.RawText ?? $"{{{token.PropertyName}}}"); - } - - static LogEventProperty CreateProperty(string name, object value) + public void WriteEvent(JsonObject eventJson) { - return LogEventPropertyFactory.SafeCreate(name, CreatePropertyValue(value)); + _eventFormatter?.Format(eventJson, Console.Out); } - internal static LogEventPropertyValue CreatePropertyValue(object value) - { - switch (value) - { - case JObject jo: - jo.TryGetValue("$typeTag", out var tt); - return new StructureValue( - jo.Properties() - .Where(kvp => kvp.Name != "$typeTag") - .Select(kvp => CreateProperty(kvp.Name, kvp.Value)), - (tt as JValue)?.Value as string); - - case JArray ja: - return new SequenceValue(ja.Select(CreatePropertyValue)); - - default: - return new ScalarValue(value); - } - } - static string Stringify(object? value) { return value switch diff --git a/src/SeqCli/Output/StripStructureTypeEnricher.cs b/src/SeqCli/Output/StripStructureTypeEnricher.cs deleted file mode 100644 index 352cd1bf..00000000 --- a/src/SeqCli/Output/StripStructureTypeEnricher.cs +++ /dev/null @@ -1,25 +0,0 @@ -using System.Linq; -using SeqCli.Util; -using Serilog.Core; -using Serilog.Data; -using Serilog.Events; - -namespace SeqCli.Output; - -public class StripStructureTypeEnricher : LogEventPropertyValueRewriter, ILogEventEnricher -{ - public void Enrich(LogEvent logEvent, ILogEventPropertyFactory propertyFactory) - { - foreach (var property in logEvent.Properties) - { - var updated = LogEventPropertyFactory.SafeCreate(property.Key, Visit(null, property.Value)); - logEvent.AddOrUpdateProperty(updated); - } - } - - protected override LogEventPropertyValue VisitStructureValue(object? state, StructureValue structure) - { - return new StructureValue(structure.Properties.Select(p => - LogEventPropertyFactory.SafeCreate(p.Name, Visit(null, p.Value)))); - } -} \ No newline at end of file diff --git a/src/SeqCli/Output/TextFormatters.cs b/src/SeqCli/Output/TextFormatters.cs index 86cfbee7..88025fdf 100644 --- a/src/SeqCli/Output/TextFormatters.cs +++ b/src/SeqCli/Output/TextFormatters.cs @@ -13,42 +13,32 @@ // limitations under the License. using System; -using SeqCli.Ingestion; -using SeqCli.Mapping; -using Serilog.Expressions; -using Serilog.Formatting; -using Serilog.Templates; -using Serilog.Templates.Themes; +using Seq.Syntax.Templates; +using Seq.Syntax.Templates.Encoding; +using Seq.Syntax.Templates.Themes; +using SeqCli.Syntax; namespace SeqCli.Output; -// This is the only usage of Serilog.Expressions remaining in seqcli; the upstream Seq.Syntax doesn't yet support -// tracing properties or theming. static class TextFormatters { - public static ITextFormatter Json(TemplateTheme? theme) => new ExpressionTemplate( - $"{{ " + - $"if {MetricsMapping.SurrogateDefinitionsProperty} is not null then " + - // Emit a metric sample - $"{{@t, @l: undefined(), @d: {MetricsMapping.SurrogateDefinitionsProperty}, ..rest()}} " + - $"else " + - // Emit a log or span - $"{{@t, @mt, @l: coalesce({LevelMapping.SurrogateLevelProperty}, if @l = 'Information' then undefined() else @l), @x, @sp, @tr, @ps: coalesce({TraceConstants.ParentSpanIdProperty}, @ps), @st: coalesce({TraceConstants.SpanStartTimestampProperty}, @st), ..rest()}} " + - $"}}" + - Environment.NewLine, - theme: theme, - // The `OutputFormat` constructor has already decided whether to colorize. - applyThemeWhenOutputIsRedirected: true - ); + /// + /// Newline-delimited CLEF output: the event JSON document is written verbatim, with theming + /// when a theme is supplied. + /// + public static ExpressionTemplate Json(TemplateTheme? theme) => new( + "{@Data}" + Environment.NewLine, + encoder: Encoder(theme)); + // Guarding on `@Elapsed` rather than the built-in `IsSpan()` shows elapsed time for any + // event carrying a span start timestamp, whether or not trace and span ids accompany it. static readonly string DefaultPlainTextOutputTemplate = - "[{@t:o} {@l:u3}] {@m}{#if IsSpan()} ({Milliseconds(Elapsed()):0.###} ms){#end}" + Environment.NewLine + "{@x}"; + "[{@Timestamp:o} {@Level:u3}] {@Message}{#if @Elapsed is not null} ({TotalMilliseconds(@Elapsed):0.###} ms){#end}" + + Environment.NewLine + "{@Exception}"; - public static ITextFormatter Plain(TemplateTheme? theme, string? outputTemplate) => new ExpressionTemplate( - outputTemplate ?? DefaultPlainTextOutputTemplate, - theme: theme, - nameResolver: new StaticMemberNameResolver(typeof(TracingFunctions)), - // The `OutputFormat` constructor has already decided whether to colorize. - applyThemeWhenOutputIsRedirected: true - ); -} \ No newline at end of file + public static ExpressionTemplate Plain(TemplateTheme? theme, string? outputTemplate) => + SeqSyntax.ParseTemplate(outputTemplate ?? DefaultPlainTextOutputTemplate, Encoder(theme)); + + static TemplateOutputEncoder? Encoder(TemplateTheme? theme) => + theme != null ? TemplateOutputEncoder.Ansi(theme) : null; +} diff --git a/src/SeqCli/Output/TraceFormatter.cs b/src/SeqCli/Output/TraceFormatter.cs index d2256e3d..ab8238b2 100644 --- a/src/SeqCli/Output/TraceFormatter.cs +++ b/src/SeqCli/Output/TraceFormatter.cs @@ -14,11 +14,11 @@ using System; using System.Collections.Generic; +using System.Globalization; using System.Text; -using SeqCli.Mapping; +using System.Text.Json.Nodes; using SeqCli.Traces; using SeqCli.Util; -using Serilog.Events; namespace SeqCli.Output; @@ -35,7 +35,7 @@ static class TraceFormatter public static string OutputTemplate(int columnCount) { - var template = new StringBuilder($"[{{@t:o}} {{@l:u3}}] {{{TreePrefixProperty}}}"); + var template = new StringBuilder($"[{{@Timestamp:o}} {{@Level:u3}}] {{{TreePrefixProperty}}}"); // `<> ''` is undefined, and hence falsy, when the property is missing; the guard thus // drops the column, and its trailing space, for both missing and empty values. @@ -45,22 +45,22 @@ public static string OutputTemplate(int columnCount) template.Append($"{{#if {column} <> ''}}{{{column}}} {{#end}}"); } - template.Append($"{{@m}}{{#if {ElapsedProperty} is not null}} ({{Milliseconds({ElapsedProperty}):0.###}} ms){{#end}}"); - template.Append(Environment.NewLine).Append("{@x}"); + template.Append($"{{@Message}}{{#if {ElapsedProperty} is not null}} ({{TotalMilliseconds({ElapsedProperty}):0.###}} ms){{#end}}"); + template.Append(Environment.NewLine).Append("{@Exception}"); return template.ToString(); } - public static IEnumerable ToLogEvents(IReadOnlyList roots) + public static IEnumerable ToEventJson(IReadOnlyList roots) { foreach (var root in roots) { - yield return ToLogEvent(root, root.Element.IsSpan ? "" : LogConnector); + yield return ToEventJson(root, root.Element.IsSpan ? "" : LogConnector); foreach (var descendant in WalkChildren(root, "")) yield return descendant; } } - static IEnumerable WalkChildren(TraceTreeNode parent, string indent) + static IEnumerable WalkChildren(TraceTreeNode parent, string indent) { for (var i = 0; i < parent.Children.Count; ++i) { @@ -71,40 +71,43 @@ static IEnumerable WalkChildren(TraceTreeNode parent, string indent) isLast ? LastSpanConnector : SpanConnector : LogConnector; - yield return ToLogEvent(child, indent + connector); + yield return ToEventJson(child, indent + connector); foreach (var descendant in WalkChildren(child, indent + (isLast ? Gap : Continuation))) yield return descendant; } } - static LogEvent ToLogEvent(TraceTreeNode treeNode, string treePrefix) + static JsonObject ToEventJson(TraceTreeNode treeNode, string treePrefix) { var evt = treeNode.Element; - var properties = new List + // Spans are positioned and shown at their start time. + var eventJson = new JsonObject { - new(TreePrefixProperty, new ScalarValue(treePrefix)) + ["@t"] = evt.SortKey.ToLocalTime().ToString("o", CultureInfo.InvariantCulture), + ["@mt"] = evt.MessageTemplate, + [TreePrefixProperty] = treePrefix }; - properties.AddRange(evt.TemplateProperties); + if (!string.IsNullOrEmpty(evt.Level)) + eventJson["@l"] = evt.Level; + + if (!string.IsNullOrWhiteSpace(evt.Exception)) + eventJson["@x"] = evt.Exception; + + foreach (var (name, value) in evt.TemplateProperties) + eventJson[name] = value?.DeepClone(); if (evt.Elapsed is { } elapsed) - properties.Add(new(ElapsedProperty, new ScalarValue(elapsed))); + eventJson[ElapsedProperty] = elapsed.ToString("c", CultureInfo.InvariantCulture); for (var i = 0; i < evt.Columns.Count; ++i) { if (evt.Columns[i] is { } value) - properties.Add(LogEventPropertyFactory.SafeCreate( - ColumnPropertyName(i), OutputFormat.CreatePropertyValue(value))); + eventJson[ColumnPropertyName(i)] = JsonNodes.FromApiValue(value); } - // Spans are positioned and shown at their start time. - return new LogEvent( - evt.SortKey.ToLocalTime(), - LevelMapping.ToSerilogLevel(evt.Level ?? ""), - string.IsNullOrWhiteSpace(evt.Exception) ? null : new TextException(evt.Exception), - evt.MessageTemplate, - properties); + return eventJson; } } diff --git a/src/SeqCli/Output/TracingFunctions.cs b/src/SeqCli/Output/TracingFunctions.cs deleted file mode 100644 index 5e2ba112..00000000 --- a/src/SeqCli/Output/TracingFunctions.cs +++ /dev/null @@ -1,55 +0,0 @@ -// Copyright © Datalust Pty Ltd -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -using System; -using System.Globalization; -using SeqCli.Ingestion; -using Serilog.Events; - -namespace SeqCli.Output; - -static class TracingFunctions -{ - public static LogEventPropertyValue? Elapsed(LogEvent logEvent) - { - if (logEvent.Properties.TryGetValue(TraceConstants.SpanStartTimestampProperty, out var sst) && - sst is ScalarValue { Value: DateTime spanStart }) - { - return new ScalarValue(logEvent.Timestamp - spanStart); - } - - if (logEvent.Properties.TryGetValue("@st", out var st) && - st is ScalarValue { Value: string spanStartIso } && - DateTimeOffset.TryParse(spanStartIso, CultureInfo.InvariantCulture, out var spanStartDto)) - { - return new ScalarValue(logEvent.Timestamp - spanStartDto); - } - - return null; - } - - public static LogEventPropertyValue? IsSpan(LogEvent logEvent) - { - return new ScalarValue(Elapsed(logEvent) != null); - } - - public static LogEventPropertyValue? Milliseconds(LogEventPropertyValue? timeSpan) - { - // Truncates instead of rounding. - if (timeSpan is ScalarValue { Value: TimeSpan ts }) - return new ScalarValue((decimal)ts.Ticks / TimeSpan.TicksPerMillisecond); - - return null; - } -} \ No newline at end of file diff --git a/src/SeqCli/PlainText/LogEvents/EventJsonBuilder.cs b/src/SeqCli/PlainText/LogEvents/EventJsonBuilder.cs new file mode 100644 index 00000000..caf4476e --- /dev/null +++ b/src/SeqCli/PlainText/LogEvents/EventJsonBuilder.cs @@ -0,0 +1,100 @@ +// Copyright © Datalust and contributors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Text.Json.Nodes; +using SeqCli.Syntax; +using Superpower.Model; + +namespace SeqCli.PlainText.LogEvents; + +/// +/// Assembles the values captured by a plain-text extraction pattern into an event JSON +/// document in Seq's emission schema. +/// +static class EventJsonBuilder +{ + public static JsonObject FromProperties(IDictionary properties, string? remainder) + { + var eventJson = new JsonObject + { + ["@t"] = GetTimestamp(properties).ToString("o", CultureInfo.InvariantCulture) + }; + + if (TryGetText(properties, ReifiedProperties.Level, out var level)) + eventJson["@l"] = level; + + if (TryGetText(properties, ReifiedProperties.Message, out var message)) + eventJson["@m"] = message; + + if (TryGetText(properties, ReifiedProperties.Exception, out var exception)) + eventJson["@x"] = exception; + + if (TryGetText(properties, ReifiedProperties.TraceId, out var traceId)) + eventJson["@tr"] = traceId; + + if (TryGetText(properties, ReifiedProperties.SpanId, out var spanId)) + eventJson["@sp"] = spanId; + + if (TryGetText(properties, ReifiedProperties.StartTimestamp, out var start)) + eventJson["@st"] = start; + + foreach (var (name, value) in properties) + { + if (!ReifiedProperties.IsReifiedProperty(name)) + EventJson.SetUserProperty(eventJson, name, CreateValue(value)); + } + + if (remainder != null) + EventJson.SetUserProperty(eventJson, "@unmatched", remainder); + + return eventJson; + } + + static JsonNode? CreateValue(object? value) + { + return value is TextSpan span + ? JsonValue.Create(span.ToStringValue()) + : EventJson.CreateScalar(value); + } + + static bool TryGetText(IDictionary properties, string name, out string text) + { + if (properties.TryGetValue(name, out var value) && value is TextSpan span) + { + text = span.ToStringValue(); + return true; + } + + text = ""; + return false; + } + + static DateTimeOffset GetTimestamp(IDictionary properties) + { + if (properties.TryGetValue(ReifiedProperties.Timestamp, out var t)) + { + if (t is TextSpan span && DateTimeOffset.TryParse(span.ToStringValue(), + CultureInfo.InvariantCulture, DateTimeStyles.AssumeLocal, out var ts)) + return ts; + + if (t is DateTimeOffset dto) + return dto; + } + + return DateTimeOffset.Now; + } +} diff --git a/src/SeqCli/PlainText/LogEvents/LogEventBuilder.cs b/src/SeqCli/PlainText/LogEvents/LogEventBuilder.cs deleted file mode 100644 index 1362716f..00000000 --- a/src/SeqCli/PlainText/LogEvents/LogEventBuilder.cs +++ /dev/null @@ -1,131 +0,0 @@ -// Copyright © Datalust and contributors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -using System; -using System.Collections.Generic; -using System.Diagnostics; -using System.Globalization; -using System.Linq; -using SeqCli.Mapping; -using SeqCli.Util; -using Serilog.Events; -using Serilog.Parsing; -using Superpower.Model; - -namespace SeqCli.PlainText.LogEvents; - -static class LogEventBuilder -{ - public static LogEvent FromProperties(IDictionary properties, string? remainder) - { - var timestamp = GetTimestamp(properties); - var level = GetLevel(properties); - var exception = TryGetException(properties); - var messageTemplate = GetMessageTemplate(properties); - var traceId = GetTraceId(properties); - var spanId = GetSpanId(properties); - var props = GetLogEventProperties(properties, remainder); - - var fallbackMappedLevel = level != null ? LevelMapping.ToSerilogLevel(level) : LogEventLevel.Information; - properties[LevelMapping.SurrogateLevelProperty] = level; - - return new LogEvent( - timestamp, - fallbackMappedLevel, - exception, - messageTemplate, - props, - traceId ?? default, - spanId ?? default - ); - } - - static readonly MessageTemplate NoMessage = new MessageTemplateParser().Parse(""); - - static MessageTemplate GetMessageTemplate(IDictionary properties) - { - if (properties.TryGetValue(ReifiedProperties.Message, out var m) && - m is TextSpan ts) - { - var text = ts.ToStringValue(); - return new MessageTemplate([new TextToken(text)]); - } - - return NoMessage; - } - - static string? GetLevel(IDictionary properties) - { - if (properties.TryGetValue(ReifiedProperties.Level, out var l) && - l is TextSpan ts) - return ts.ToStringValue(); - - return null; - } - - static ActivityTraceId? GetTraceId(IDictionary properties) - { - if (properties.TryGetValue(ReifiedProperties.TraceId, out var tr) && - tr is TextSpan ts) - return ActivityTraceId.CreateFromString(ts.ToStringValue()); - - return null; - } - - static ActivitySpanId? GetSpanId(IDictionary properties) - { - if (properties.TryGetValue(ReifiedProperties.SpanId, out var sp) && - sp is TextSpan ts) - return ActivitySpanId.CreateFromString(ts.ToStringValue()); - - return null; - } - - static Exception? TryGetException(IDictionary properties) - { - if (properties.TryGetValue(ReifiedProperties.Exception, out var x) && - x is TextSpan ts) - return new TextOnlyException(ts.ToStringValue()); - return null; - } - - static IEnumerable GetLogEventProperties(IDictionary properties, string? remainder) - { - var payload = properties - .Where(p => !ReifiedProperties.IsReifiedProperty(p.Key)) - .Select(p => LogEventPropertyFactory.SafeCreate(p.Key, new ScalarValue(p.Value))); - - if (remainder != null) - payload = payload.Concat(new[] - { - LogEventPropertyFactory.SafeCreate("@unmatched", new ScalarValue(remainder)) - }); - return payload; - } - - static DateTimeOffset GetTimestamp(IDictionary properties) - { - if (properties.TryGetValue(ReifiedProperties.Timestamp, out var t)) - { - if (t is TextSpan span && DateTimeOffset.TryParse(span.ToStringValue(), - CultureInfo.InvariantCulture, DateTimeStyles.AssumeLocal, out var ts)) - return ts; - - if (t is DateTimeOffset dto) - return dto; - } - - return DateTimeOffset.Now; - } -} \ No newline at end of file diff --git a/src/SeqCli/PlainText/LogEvents/TextOnlyException.cs b/src/SeqCli/PlainText/LogEvents/TextOnlyException.cs deleted file mode 100644 index 614c927f..00000000 --- a/src/SeqCli/PlainText/LogEvents/TextOnlyException.cs +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright © Datalust and contributors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -using System; - -namespace SeqCli.PlainText.LogEvents; - -class TextOnlyException : Exception -{ - readonly string _toStringValue; - - public TextOnlyException(string toStringValue) - { - _toStringValue = toStringValue ?? throw new ArgumentNullException(nameof(toStringValue)); - } - - public override string ToString() - { - return _toStringValue; - } -} \ No newline at end of file diff --git a/src/SeqCli/PlainText/PlainTextLogEventReader.cs b/src/SeqCli/PlainText/PlainTextEventReader.cs similarity index 86% rename from src/SeqCli/PlainText/PlainTextLogEventReader.cs rename to src/SeqCli/PlainText/PlainTextEventReader.cs index fae2df86..fbead08b 100644 --- a/src/SeqCli/PlainText/PlainTextLogEventReader.cs +++ b/src/SeqCli/PlainText/PlainTextEventReader.cs @@ -10,14 +10,14 @@ namespace SeqCli.PlainText; -class PlainTextLogEventReader : ILogEventReader +class PlainTextEventReader : IEventReader { static readonly TimeSpan TrailingLineArrivalDeadline = TimeSpan.FromMilliseconds(10); readonly NameValueExtractor _nameValueExtractor; readonly FrameReader _reader; - public PlainTextLogEventReader(TextReader input, string extractionPattern) + public PlainTextEventReader(TextReader input, string extractionPattern) { if (extractionPattern == null) throw new ArgumentNullException(nameof(extractionPattern)); _nameValueExtractor = ExtractionPatternInterpreter.CreateNameValueExtractor(ExtractionPatternParser.Parse(extractionPattern)); @@ -36,7 +36,7 @@ public async Task TryReadAsync() var (properties, remainder) = _nameValueExtractor.ExtractValues(frame.Value); - var evt = LogEventBuilder.FromProperties(properties, remainder); + var evt = EventJsonBuilder.FromProperties(properties, remainder); return new ReadResult(evt, frame.IsAtEnd); } } \ No newline at end of file diff --git a/src/SeqCli/Ingestion/BufferingSink.cs b/src/SeqCli/Sample/Ingestion/BufferingSink.cs similarity index 50% rename from src/SeqCli/Ingestion/BufferingSink.cs rename to src/SeqCli/Sample/Ingestion/BufferingSink.cs index 879b930c..cab3a4d3 100644 --- a/src/SeqCli/Ingestion/BufferingSink.cs +++ b/src/SeqCli/Sample/Ingestion/BufferingSink.cs @@ -1,31 +1,41 @@ -using System; +using System; using System.Collections.Concurrent; +using System.Text.Json.Nodes; using System.Threading.Tasks; +using SeqCli.Ingestion; using Serilog.Core; using Serilog.Events; -namespace SeqCli.Ingestion; +namespace SeqCli.Sample.Ingestion; -class BufferingSink: ILogEventSink, ILogEventReader, IDisposable +/// +/// Bridges the sample simulation's Serilog-based event generation into the +/// JSON-document-based shipping pipeline. +/// +class BufferingSink: ILogEventSink, IEventReader, IDisposable { - readonly ConcurrentQueue _queue = new(); + readonly ConcurrentQueue _queue = new(); const int QueueCapacity = 10000; volatile bool _disposed; - + public void Emit(LogEvent logEvent) { // No problem if this is racy - we can afford a bit of extra queue space. if (_disposed || _queue.Count > QueueCapacity) return; - - _queue.Enqueue(logEvent); + + var document = MetricsMapping.TryGetMetricSampleJson(logEvent, out var sample) + ? sample + : SerilogEventJson.ToEventJson(logEvent); + + _queue.Enqueue(document); } public Task TryReadAsync() { - if (!_queue.TryDequeue(out var logEvent)) + if (!_queue.TryDequeue(out var document)) return Task.FromResult(new ReadResult(null, _disposed)); - return Task.FromResult(new ReadResult(logEvent, _disposed)); + return Task.FromResult(new ReadResult(document, _disposed)); } public void Dispose() @@ -34,4 +44,4 @@ public void Dispose() _disposed = true; _queue.Clear(); } -} \ No newline at end of file +} diff --git a/src/SeqCli/Sample/Ingestion/MetricsMapping.cs b/src/SeqCli/Sample/Ingestion/MetricsMapping.cs new file mode 100644 index 00000000..8ba3771f --- /dev/null +++ b/src/SeqCli/Sample/Ingestion/MetricsMapping.cs @@ -0,0 +1,59 @@ +// Copyright © Datalust and contributors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Diagnostics.CodeAnalysis; +using System.Globalization; +using System.Text.Json.Nodes; +using SeqCli.Ingestion; +using SeqCli.Syntax; +using Serilog.Events; + +namespace SeqCli.Sample.Ingestion; + +/// +/// The sample simulation generates metric samples as Serilog events carrying their metric +/// definitions in a surrogate property, because Serilog's data model has no @d +/// equivalent. Events marked this way ship as metric samples rather than logs. +/// +static class MetricsMapping +{ + // Use a "hygienic" name for the definitions property to avoid collisions. + internal static readonly string SurrogateDefinitionsProperty = $"_SeqcliMetricDefinitions_{Guid.NewGuid():N}"; + + public static bool TryGetMetricSampleJson(LogEvent logEvent, [NotNullWhen(true)] out JsonObject? sample) + { + if (!logEvent.Properties.TryGetValue(SurrogateDefinitionsProperty, out var definitions)) + { + sample = null; + return false; + } + + // Metric samples carry only a timestamp, definitions, and their dimension/value + // properties; no message or level. + sample = new JsonObject + { + ["@t"] = logEvent.Timestamp.ToString("o", CultureInfo.InvariantCulture), + ["@d"] = SerilogEventJson.ToJsonNode(definitions) + }; + + foreach (var (name, value) in logEvent.Properties) + { + if (name != SurrogateDefinitionsProperty) + EventJson.SetUserProperty(sample, name, SerilogEventJson.ToJsonNode(value)); + } + + return true; + } +} diff --git a/src/SeqCli/Sample/Loader/Simulation.cs b/src/SeqCli/Sample/Loader/Simulation.cs index a54f0a28..23828632 100644 --- a/src/SeqCli/Sample/Loader/Simulation.cs +++ b/src/SeqCli/Sample/Loader/Simulation.cs @@ -17,7 +17,7 @@ using Roastery.Metrics; using Seq.Api; using SeqCli.Ingestion; -using SeqCli.Mapping; +using SeqCli.Sample.Ingestion; using Serilog; namespace SeqCli.Sample.Loader; diff --git a/src/SeqCli/SeqCli.csproj b/src/SeqCli/SeqCli.csproj index f666ccdd..0e97213a 100644 --- a/src/SeqCli/SeqCli.csproj +++ b/src/SeqCli/SeqCli.csproj @@ -31,7 +31,6 @@ - @@ -43,13 +42,10 @@ - + - - - diff --git a/src/SeqCli/Syntax/EventJson.cs b/src/SeqCli/Syntax/EventJson.cs new file mode 100644 index 00000000..282a3cbf --- /dev/null +++ b/src/SeqCli/Syntax/EventJson.cs @@ -0,0 +1,66 @@ +// Copyright © Datalust and contributors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Globalization; +using System.Text.Json.Nodes; + +namespace SeqCli.Syntax; + +/// +/// Helpers for constructing event JSON documents in Seq's emission (CLEF) schema, where +/// reified fields carry @-prefixed names and user-defined property names beginning +/// with @ are escaped with a second @. +/// +static class EventJson +{ + const string InvalidPropertyNameSubstitute = "(unnamed)"; + + public static string EscapeUserPropertyName(string name) + { + if (string.IsNullOrEmpty(name)) + return InvalidPropertyNameSubstitute; + + return name.StartsWith('@') ? $"@{name}" : name; + } + + public static void SetUserProperty(JsonObject eventJson, string name, JsonNode? value) + { + eventJson[EscapeUserPropertyName(name)] = value; + } + + public static JsonNode? CreateScalar(object? value) + { + return value switch + { + null => null, + string s => JsonValue.Create(s), + bool b => JsonValue.Create(b), + byte n => JsonValue.Create(n), + sbyte n => JsonValue.Create(n), + short n => JsonValue.Create(n), + ushort n => JsonValue.Create(n), + int n => JsonValue.Create(n), + uint n => JsonValue.Create(n), + long n => JsonValue.Create(n), + ulong n => JsonValue.Create(n), + float n => JsonValue.Create(n), + double n => JsonValue.Create(n), + decimal n => JsonValue.Create(n), + DateTime dt => JsonValue.Create(dt.ToString("o", CultureInfo.InvariantCulture)), + DateTimeOffset dto => JsonValue.Create(dto.ToString("o", CultureInfo.InvariantCulture)), + _ => JsonValue.Create(value.ToString()) + }; + } +} diff --git a/src/SeqCli/Util/TextException.cs b/src/SeqCli/Syntax/IEventEnricher.cs similarity index 60% rename from src/SeqCli/Util/TextException.cs rename to src/SeqCli/Syntax/IEventEnricher.cs index 2017129d..9d10d8e7 100644 --- a/src/SeqCli/Util/TextException.cs +++ b/src/SeqCli/Syntax/IEventEnricher.cs @@ -1,4 +1,4 @@ -// Copyright 2013-2015 Serilog Contributors +// Copyright © Datalust and contributors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,22 +12,15 @@ // See the License for the specific language governing permissions and // limitations under the License. -using System; +using System.Text.Json.Nodes; -namespace SeqCli.Util; +namespace SeqCli.Syntax; -class TextException : Exception +/// +/// Adds or updates fields on an event JSON document; the equivalent, in Seq's data model, of a +/// Serilog enricher. +/// +interface IEventEnricher { - readonly string _text; - - public TextException(string text) - : base("This exception type provides ToString() access to details only.") - { - _text = text; - } - - public override string ToString() - { - return _text; - } -} \ No newline at end of file + void Enrich(JsonObject eventJson); +} diff --git a/src/SeqCli/Output/RedundantEventTypeRemovalEnricher.cs b/src/SeqCli/Syntax/LevelEnricher.cs similarity index 64% rename from src/SeqCli/Output/RedundantEventTypeRemovalEnricher.cs rename to src/SeqCli/Syntax/LevelEnricher.cs index d32e6666..c09fdb35 100644 --- a/src/SeqCli/Output/RedundantEventTypeRemovalEnricher.cs +++ b/src/SeqCli/Syntax/LevelEnricher.cs @@ -1,4 +1,4 @@ -// Copyright © Datalust and contributors. +// Copyright © Datalust and contributors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,15 +12,17 @@ // See the License for the specific language governing permissions and // limitations under the License. -using Serilog.Core; -using Serilog.Events; +using System.Text.Json.Nodes; -namespace SeqCli.Output; +namespace SeqCli.Syntax; -public class RedundantEventTypeRemovalEnricher : ILogEventEnricher +/// +/// Overrides the event's @l level with a fixed value. +/// +class LevelEnricher(string level) : IEventEnricher { - public void Enrich(LogEvent logEvent, ILogEventPropertyFactory propertyFactory) + public void Enrich(JsonObject eventJson) { - logEvent.RemovePropertyIfPresent("@i"); + eventJson["@l"] = level; } -} \ No newline at end of file +} diff --git a/src/SeqCli/Ingestion/ScalarPropertyEnricher.cs b/src/SeqCli/Syntax/ScalarPropertyEnricher.cs similarity index 59% rename from src/SeqCli/Ingestion/ScalarPropertyEnricher.cs rename to src/SeqCli/Syntax/ScalarPropertyEnricher.cs index 7146c7e7..95490a3b 100644 --- a/src/SeqCli/Ingestion/ScalarPropertyEnricher.cs +++ b/src/SeqCli/Syntax/ScalarPropertyEnricher.cs @@ -1,4 +1,4 @@ -// Copyright © Datalust and contributors. +// Copyright © Datalust and contributors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,23 +12,23 @@ // See the License for the specific language governing permissions and // limitations under the License. -using SeqCli.Util; -using Serilog.Core; -using Serilog.Events; +using System.Text.Json.Nodes; -namespace SeqCli.Ingestion; +namespace SeqCli.Syntax; -class ScalarPropertyEnricher : ILogEventEnricher +class ScalarPropertyEnricher : IEventEnricher { - readonly LogEventProperty _property; + readonly string _name; + readonly object? _scalarValue; public ScalarPropertyEnricher(string name, object? scalarValue) { - _property = LogEventPropertyFactory.SafeCreate(name, new ScalarValue(scalarValue)); + _name = EventJson.EscapeUserPropertyName(name); + _scalarValue = scalarValue; } - public void Enrich(LogEvent logEvent, ILogEventPropertyFactory propertyFactory) + public void Enrich(JsonObject eventJson) { - logEvent.AddOrUpdateProperty(_property); + eventJson[_name] = EventJson.CreateScalar(_scalarValue); } -} \ No newline at end of file +} diff --git a/src/SeqCli/Syntax/SeqCliNameResolver.cs b/src/SeqCli/Syntax/SeqCliNameResolver.cs deleted file mode 100644 index 91b4abab..00000000 --- a/src/SeqCli/Syntax/SeqCliNameResolver.cs +++ /dev/null @@ -1,20 +0,0 @@ -using System.Diagnostics.CodeAnalysis; -using Seq.Syntax.Expressions; - -namespace SeqCli.Syntax; - -class SeqCliNameResolver: NameResolver -{ - public override bool TryResolveBuiltInPropertyName(string alias, [MaybeNullWhen(false)] out string target) - { - switch (alias) - { - case "@l": - target = "coalesce(SeqCliOriginalLevel, @l)"; - return true; - default: - target = null; - return false; - } - } -} diff --git a/src/SeqCli/Syntax/SeqSyntax.cs b/src/SeqCli/Syntax/SeqSyntax.cs index 3974b4ad..acbeab8c 100644 --- a/src/SeqCli/Syntax/SeqSyntax.cs +++ b/src/SeqCli/Syntax/SeqSyntax.cs @@ -1,11 +1,55 @@ -using Seq.Syntax.Expressions; +// Copyright © Datalust and contributors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Diagnostics.CodeAnalysis; +using Seq.Syntax.Expressions; +using Seq.Syntax.Templates; +using Seq.Syntax.Templates.Encoding; +using SeqCli.Syntax.V1; +using V1Compatibility = Seq.Syntax.Compatibility.V1; namespace SeqCli.Syntax; +/// +/// Compiles the expressions and templates accepted on the command line. Uses the Seq.Syntax v1 +/// compatibility shim so that established seqcli syntax — abbreviated built-in names like +/// @l, and the Elapsed()/Milliseconds() functions — keeps working. +/// static class SeqSyntax { public static CompiledExpression CompileExpression(string expression) { - return SerilogExpression.Compile(expression, nameResolver: new SeqCliNameResolver()); + if (!TryCompileExpression(expression, out var compiled, out var error)) + throw new ArgumentException(error); + + return compiled; + } + + public static bool TryCompileExpression( + string expression, + [MaybeNullWhen(false)] out CompiledExpression result, + [MaybeNullWhen(true)] out string error) + { + return V1Compatibility.TryCompileExpression(expression, formatProvider: null, TracingFunctions.Resolver, out result, out error); + } + + public static ExpressionTemplate ParseTemplate(string template, TemplateOutputEncoder? encoder = null) + { + if (!V1Compatibility.TryParseTemplate(template, culture: null, TracingFunctions.Resolver, encoder, out var parsed, out var error)) + throw new ArgumentException(error); + + return parsed; } -} \ No newline at end of file +} diff --git a/src/SeqCli/Syntax/V1/TracingFunctions.cs b/src/SeqCli/Syntax/V1/TracingFunctions.cs new file mode 100644 index 00000000..59193168 --- /dev/null +++ b/src/SeqCli/Syntax/V1/TracingFunctions.cs @@ -0,0 +1,58 @@ +// Copyright © Datalust Pty Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Globalization; +using System.Text.Json.Nodes; +using Seq.Syntax.Expressions; + +namespace SeqCli.Syntax.V1; + +/// +/// Functions carried over from earlier seqcli versions, where Seq.Syntax had no tracing +/// support of its own. Elapsed() and Milliseconds() remain only so that existing +/// user-supplied expressions and output templates keep working; the built-in @Elapsed +/// and TotalMilliseconds() replace them. +/// +static class TracingFunctions +{ + public static readonly NameResolver Resolver = new StaticMemberNameResolver(typeof(TracingFunctions)); + + public static EvaluationResult Elapsed(JsonObject eventJson) + { + if (GetTimestampField(eventJson, "@t") is { } timestamp && + GetTimestampField(eventJson, "@st") is { } start) + { + return JsonValue.Create(timestamp - start)!; + } + + return EvaluationResult.Undefined; + } + + public static EvaluationResult Milliseconds(TimeSpan timeSpan) + { + // Truncates instead of rounding. + return JsonValue.Create(timeSpan.Ticks / (decimal)TimeSpan.TicksPerMillisecond); + } + + static DateTimeOffset? GetTimestampField(JsonObject eventJson, string field) + { + return eventJson.TryGetPropertyValue(field, out var node) && + node is JsonValue value && + value.TryGetValue(out string? text) && + DateTimeOffset.TryParse(text, CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind, out var dto) + ? dto + : null; + } +} diff --git a/src/SeqCli/Traces/StructuredMessage.cs b/src/SeqCli/Traces/StructuredMessage.cs index 73654f04..38cae501 100644 --- a/src/SeqCli/Traces/StructuredMessage.cs +++ b/src/SeqCli/Traces/StructuredMessage.cs @@ -14,30 +14,30 @@ using System.Collections.Generic; using System.IO; +using System.Linq; +using System.Text.Json.Nodes; using Newtonsoft.Json.Linq; -using SeqCli.Output; using SeqCli.Util; -using Serilog.Events; -using Serilog.Parsing; namespace SeqCli.Traces; static class StructuredMessage { /// - /// Reads the token array produced by the Seq `@StructuredMessage` property - /// into a Serilog message template, along with the property values needed to render it. + /// Reads the token array produced by the Seq `@StructuredMessage` property into message + /// template text, along with the property values needed to render it. Dotted hole names + /// are stored as nested structures, matching how message rendering resolves them. /// - public static (MessageTemplate Message, IReadOnlyList Properties) Read(object? structuredMessage) + public static (string MessageTemplate, JsonObject Properties) Read(object? structuredMessage) { if (structuredMessage is null or JValue { Type: JTokenType.Null }) - return (new MessageTemplate([]), []); + return ("", new JsonObject()); if (structuredMessage is not JArray tokens) throw new InvalidDataException($"Expected a structured message but found `{structuredMessage}`."); - var templateTokens = new List(); - var properties = new List(); + var templateTokens = new List<(bool IsText, string Text)>(); + var properties = new JsonObject(); var propertyNames = new HashSet(); foreach (var token in tokens) @@ -48,14 +48,14 @@ public static (MessageTemplate Message, IReadOnlyList Properti throw new InvalidDataException("A message template hole is missing its `name`."); // Currently ignores `formatted`. - templateTokens.Add(new PropertyToken(name, (hole["raw"] as JValue)?.Value as string ?? $"{{{name}}}")); + templateTokens.Add((false, (hole["raw"] as JValue)?.Value as string ?? $"{{{name}}}")); if (hole.TryGetValue("value", out var value) && propertyNames.Add(name)) - properties.Add(LogEventPropertyFactory.SafeCreate(name, CreatePropertyValue(value))); + SetPathProperty(properties, name, JsonNodes.FromNewtonsoft(value)); } else if (token is JValue { Type: JTokenType.String } text) { - templateTokens.Add(new TextToken((string)text.Value!)); + templateTokens.Add((true, (string)text.Value!)); } else { @@ -65,27 +65,53 @@ public static (MessageTemplate Message, IReadOnlyList Properti TrimEnd(templateTokens); - return (new MessageTemplate(templateTokens), properties); + var templateText = string.Concat(templateTokens.Select(t => + t.IsText ? t.Text.Replace("{", "{{").Replace("}", "}}") : t.Text)); + + return (templateText, properties); + } + + // Message rendering resolves dotted hole names as paths into nested objects, so `a.b` + // becomes member `b` of object `a`. If placing a value along the path would collide with a + // non-object value, the hole is left unresolvable and renders as raw text. + static void SetPathProperty(JsonObject properties, string name, JsonNode? value) + { + var steps = name.Split('.'); + var target = properties; + for (var i = 0; i < steps.Length - 1; ++i) + { + if (target.TryGetPropertyValue(steps[i], out var next)) + { + if (next is not JsonObject nextObject) + return; + + target = nextObject; + } + else + { + var nextObject = new JsonObject(); + target[steps[i]] = nextObject; + target = nextObject; + } + } + + target[steps[^1]] = value; } - static void TrimEnd(List templateTokens) + static void TrimEnd(List<(bool IsText, string Text)> templateTokens) { - while (templateTokens.Count > 0 && templateTokens[^1] is TextToken text) + while (templateTokens.Count > 0 && templateTokens[^1] is (true, var text)) { - var trimmed = text.Text.TrimEnd(); - if (trimmed.Length == text.Text.Length) + var trimmed = text.TrimEnd(); + if (trimmed.Length == text.Length) break; templateTokens.RemoveAt(templateTokens.Count - 1); if (trimmed.Length > 0) { - templateTokens.Add(new TextToken(trimmed)); + templateTokens.Add((true, trimmed)); break; } } } - - static LogEventPropertyValue CreatePropertyValue(JToken value) => value is JValue scalar ? - new ScalarValue(scalar.Value) : - OutputFormat.CreatePropertyValue(value); } diff --git a/src/SeqCli/Traces/TraceTreeElement.cs b/src/SeqCli/Traces/TraceTreeElement.cs index 52a25a8f..df4a6756 100644 --- a/src/SeqCli/Traces/TraceTreeElement.cs +++ b/src/SeqCli/Traces/TraceTreeElement.cs @@ -14,7 +14,7 @@ using System; using System.Collections.Generic; -using Serilog.Events; +using System.Text.Json.Nodes; namespace SeqCli.Traces; @@ -22,8 +22,8 @@ record TraceTreeElement( string Id, DateTimeOffset Timestamp, string? Level, - MessageTemplate MessageTemplate, - IReadOnlyList TemplateProperties, + string MessageTemplate, + JsonObject TemplateProperties, string? Exception, string? SpanId, string? ParentId, diff --git a/src/SeqCli/Traces/TraceTreeJObjectConverter.cs b/src/SeqCli/Traces/TraceTreeJObjectConverter.cs index 4399c15b..5d170f83 100644 --- a/src/SeqCli/Traces/TraceTreeJObjectConverter.cs +++ b/src/SeqCli/Traces/TraceTreeJObjectConverter.cs @@ -15,17 +15,16 @@ using System.Collections.Generic; using System.Globalization; using System.IO; +using System.Text.Json.Nodes; using Newtonsoft.Json.Linq; -using SeqCli.Mapping; +using Seq.Syntax.Templates; using SeqCli.Output; -using Serilog.Events; -using Serilog.Formatting; namespace SeqCli.Traces; static class TraceTreeJObjectConverter { - static readonly ITextFormatter MessageFormatter = TextFormatters.Plain(theme: null, "{@m}"); + static readonly ExpressionTemplate MessageFormatter = TextFormatters.Plain(theme: null, "{@Message}"); public static JObject FromRoots(string traceId, IReadOnlyList roots, bool complete, bool includeTypeMarker, IReadOnlyList columns) { @@ -82,7 +81,7 @@ static JObject ToJson(TraceTreeNode node, bool includeTypeMarker, IReadOnlyList< json["parentSpanId"] = evt.ParentId; if (!string.IsNullOrEmpty(evt.Level)) - json["level"] = LevelMapping.ToFullLevelName(evt.Level); + json["level"] = evt.Level; if (evt.IsSpan) { @@ -131,15 +130,16 @@ static JObject ToJson(TraceTreeNode node, bool includeTypeMarker, IReadOnlyList< static string RenderMessage(TraceTreeElement evt) { - var logEvent = new LogEvent( - evt.SortKey, - LevelMapping.ToSerilogLevel(evt.Level ?? ""), - exception: null, - evt.MessageTemplate, - evt.TemplateProperties); + var eventJson = new JsonObject + { + ["@mt"] = evt.MessageTemplate + }; + + foreach (var (name, value) in evt.TemplateProperties) + eventJson[name] = value?.DeepClone(); var message = new StringWriter(); - MessageFormatter.Format(logEvent, message); + MessageFormatter.Format(eventJson, message); return message.ToString(); } } diff --git a/src/SeqCli/Util/JsonNetDestructuringPolicy.cs b/src/SeqCli/Util/JsonNetDestructuringPolicy.cs deleted file mode 100644 index 8d9bf7bc..00000000 --- a/src/SeqCli/Util/JsonNetDestructuringPolicy.cs +++ /dev/null @@ -1,91 +0,0 @@ -// Copyright 2015 Destructurama Contributors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -using System.Collections.Generic; -using System.Diagnostics.CodeAnalysis; -using System.Linq; -using Newtonsoft.Json.Linq; -using Serilog.Core; -using Serilog.Events; - -namespace SeqCli.Util; - -sealed class JsonNetDestructuringPolicy : IDestructuringPolicy -{ - public bool TryDestructure(object value, ILogEventPropertyValueFactory propertyValueFactory, [NotNullWhen(true)] out LogEventPropertyValue? result) - { - switch (value) - { - case JObject jo: - result = Destructure(jo, propertyValueFactory); - return true; - case JArray ja: - result = Destructure(ja, propertyValueFactory); - return true; - case JValue jv: - result = Destructure(jv, propertyValueFactory); - return true; - } - - result = null; - return false; - } - - static LogEventPropertyValue Destructure(JValue jv, ILogEventPropertyValueFactory propertyValueFactory) - { - return propertyValueFactory.CreatePropertyValue(jv.Value!, destructureObjects: true); - } - - static SequenceValue Destructure(JArray ja, ILogEventPropertyValueFactory propertyValueFactory) - { - var elems = ja.Select(t => propertyValueFactory.CreatePropertyValue(t, destructureObjects: true)); - return new SequenceValue(elems); - } - - static LogEventPropertyValue Destructure(JObject jo, ILogEventPropertyValueFactory propertyValueFactory) - { - string? typeTag = null; - var props = new List(jo.Count); - - foreach (var prop in jo.Properties()) - { - if (prop.Name == "$type") - { - if (prop.Value is JValue typeVal && typeVal.Value is string v) - { - typeTag = v; - continue; - } - } - else if (!LogEventProperty.IsValidName(prop.Name)) - { - return DestructureToDictionaryValue(jo, propertyValueFactory); - } - - props.Add(new LogEventProperty(prop.Name, propertyValueFactory.CreatePropertyValue(prop.Value, destructureObjects: true))); - } - - return new StructureValue(props, typeTag); - } - - static DictionaryValue DestructureToDictionaryValue(JObject jo, ILogEventPropertyValueFactory propertyValueFactory) - { - var elements = jo.Properties().Select( - prop => new KeyValuePair( - new ScalarValue(prop.Name), - propertyValueFactory.CreatePropertyValue(prop.Value, destructureObjects: true)) - ); - return new DictionaryValue(elements); - } -} \ No newline at end of file diff --git a/src/SeqCli/Util/JsonNodes.cs b/src/SeqCli/Util/JsonNodes.cs new file mode 100644 index 00000000..7129be90 --- /dev/null +++ b/src/SeqCli/Util/JsonNodes.cs @@ -0,0 +1,45 @@ +// Copyright © Datalust Pty Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System.Text.Json.Nodes; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; +using SeqCli.Syntax; + +namespace SeqCli.Util; + +static class JsonNodes +{ + public static JsonNode? FromNewtonsoft(JToken token) + { + if (token is JValue { Value: null }) + return null; + + return JsonNode.Parse(token.ToString(Formatting.None)); + } + + /// + /// Convert a value deserialized by the Seq API client — a Newtonsoft LINQ-to-JSON token, or + /// a plain CLR scalar — into its System.Text.Json equivalent. + /// + public static JsonNode? FromApiValue(object? value) + { + return value switch + { + null => null, + JToken token => FromNewtonsoft(token), + _ => EventJson.CreateScalar(value) + }; + } +} diff --git a/src/SeqCli/Util/LogEventPropertyFactory.cs b/src/SeqCli/Util/LogEventPropertyFactory.cs deleted file mode 100644 index 89c23987..00000000 --- a/src/SeqCli/Util/LogEventPropertyFactory.cs +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright © Datalust Pty Ltd and Contributors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -using System; -using Serilog.Events; - -namespace SeqCli.Util; - -static class LogEventPropertyFactory -{ - const string InvalidPropertyNameSubstitute = "(unnamed)"; - - public static LogEventProperty SafeCreate(string name, LogEventPropertyValue value) - { - if (value == null) throw new ArgumentNullException(nameof(value)); - - if (!LogEventProperty.IsValidName(name)) - name = InvalidPropertyNameSubstitute; - - return new LogEventProperty(name, value); - } -} \ No newline at end of file diff --git a/test/SeqCli.Tests/Csv/CsvWriterTests.cs b/test/SeqCli.Tests/Csv/CsvWriterTests.cs index cf4dbeb9..8d46d098 100644 --- a/test/SeqCli.Tests/Csv/CsvWriterTests.cs +++ b/test/SeqCli.Tests/Csv/CsvWriterTests.cs @@ -3,7 +3,7 @@ using System.IO; using Seq.Api.Model.Data; using SeqCli.Csv; -using Serilog.Templates.Themes; +using Seq.Syntax.Templates.Themes; using Xunit; namespace SeqCli.Tests.Csv; @@ -12,8 +12,8 @@ public class CsvWriterTests { const char Escape = '\x1b'; - // `CsvWriter` writes to the console without going through Serilog's console sink, so unlike the other - // output paths it has no opportunity to suppress the theme itself. + // `CsvWriter` writes delimited output directly rather than rendering a template, so unlike the + // other output paths it applies (or omits) the theme itself. [Fact] public void QueryResultsAreNotColorizedWhenOutputIsRedirected() { diff --git a/test/SeqCli.Tests/Ingestion/SerilogEventJsonTests.cs b/test/SeqCli.Tests/Ingestion/SerilogEventJsonTests.cs new file mode 100644 index 00000000..f27aaf1f --- /dev/null +++ b/test/SeqCli.Tests/Ingestion/SerilogEventJsonTests.cs @@ -0,0 +1,109 @@ +#nullable enable +using System; +using System.Linq; +using SeqCli.Ingestion; +using SeqCli.Sample.Ingestion; +using Serilog; +using Serilog.Events; +using Xunit; + +namespace SeqCli.Tests.Ingestion; + +public class SerilogEventJsonTests +{ + static LogEvent CaptureEvent(Action log) + { + LogEvent? captured = null; + var logger = new LoggerConfiguration() + .MinimumLevel.Verbose() + .WriteTo.Sink(new CapturingSink(evt => captured = evt)) + .CreateLogger(); + log(logger); + return captured ?? throw new InvalidOperationException("No event was captured."); + } + + class CapturingSink(Action capture) : Serilog.Core.ILogEventSink + { + public void Emit(LogEvent logEvent) => capture(logEvent); + } + + [Fact] + public void EventFieldsMapToTheEmissionSchema() + { + var evt = CaptureEvent(log => log.Warning(new Exception("Boom!"), "Hello, {Name}!", "world")); + var eventJson = SerilogEventJson.ToEventJson(evt); + + Assert.Equal(evt.Timestamp.ToString("o"), (string?)eventJson["@t"]); + Assert.Equal("Hello, {Name}!", (string?)eventJson["@mt"]); + Assert.Equal("Warning", (string?)eventJson["@l"]); + Assert.StartsWith("System.Exception: Boom!", (string?)eventJson["@x"]); + Assert.Equal("world", (string?)eventJson["Name"]); + } + + [Fact] + public void InformationLevelsAreOmitted() + { + var evt = CaptureEvent(log => log.Information("Hello")); + + Assert.False(SerilogEventJson.ToEventJson(evt).ContainsKey("@l")); + } + + [Fact] + public void StructuredValuesSerializeAsJson() + { + var evt = CaptureEvent(log => log.Information("{@Order} {Items}", + new { Id = 7, Total = 4.5 }, new[] { "a", "b" })); + var eventJson = SerilogEventJson.ToEventJson(evt); + + Assert.Equal(7, (int?)eventJson["Order"]!["Id"]); + Assert.Equal(4.5, (double?)eventJson["Order"]!["Total"]); + Assert.Equal(new[] { "a", "b" }, eventJson["Items"]!.AsArray().Select(i => (string?)i).ToArray()); + } + + [Fact] + public void SerilogTracingSpanPropertiesAreLifted() + { + var start = DateTime.UtcNow.AddMilliseconds(-25); + var evt = CaptureEvent(log => log + .ForContext("SpanStartTimestamp", start) + .ForContext("ParentSpanId", "8899aabbccddeeff") + .Information("GET /orders")); + var eventJson = SerilogEventJson.ToEventJson(evt); + + Assert.Equal(start.ToString("o"), (string?)eventJson["@st"]); + Assert.Equal("8899aabbccddeeff", (string?)eventJson["@ps"]); + Assert.False(eventJson.ContainsKey("SpanStartTimestamp")); + Assert.False(eventJson.ContainsKey("ParentSpanId")); + } + + [Fact] + public void MetricDefinitionsProduceMetricSamples() + { + var evt = CaptureEvent(log => log + .ForContext(MetricsMapping.SurrogateDefinitionsProperty, new { roasted_kg = new { unit = "kg" } }, destructureObjects: true) + .ForContext("roasted_kg", 42.5) + .Information("Metrics sampled")); + + Assert.True(MetricsMapping.TryGetMetricSampleJson(evt, out var eventJson)); + Assert.Equal("kg", (string?)eventJson["@d"]!["roasted_kg"]!["unit"]); + Assert.Equal(42.5, (double?)eventJson["roasted_kg"]); + Assert.False(eventJson.ContainsKey("@mt")); + Assert.False(eventJson.ContainsKey("@l")); + } + + [Fact] + public void PlainEventsAreNotMetricSamples() + { + var evt = CaptureEvent(log => log.Information("Hello")); + + Assert.False(MetricsMapping.TryGetMetricSampleJson(evt, out _)); + } + + [Fact] + public void PropertyNamesBeginningWithAtAreEscaped() + { + var evt = CaptureEvent(log => log.ForContext("@evil", "value").Information("Hello")); + + Assert.Equal("value", (string?)SerilogEventJson.ToEventJson(evt)["@@evil"]); + } +} diff --git a/test/SeqCli.Tests/Output/OutputFormatTests.cs b/test/SeqCli.Tests/Output/OutputFormatTests.cs index 4c08a64e..cb1cf55b 100644 --- a/test/SeqCli.Tests/Output/OutputFormatTests.cs +++ b/test/SeqCli.Tests/Output/OutputFormatTests.cs @@ -2,9 +2,9 @@ using Newtonsoft.Json.Linq; using Seq.Api.Model.Events; using SeqCli.Config; +using SeqCli.Mapping; using SeqCli.Output; using SeqCli.Tests.Support; -using Serilog.Events; using Xunit; #nullable enable @@ -128,11 +128,10 @@ static EventEntity MakeDottedHoleEvent(params (string Name, object? Value)[] pro static string RenderMessage(EventEntity evt) { - var serilogEvent = OutputFormat.ToSerilogEvent(evt); - OutputFormat.FlattenPropertiesUsedWithDottedNames(evt, serilogEvent); + var eventJson = EventEntityJson.ToEventJson(evt); var output = new StringWriter(); - TextFormatters.Plain(theme: null, "{@m}").Format(serilogEvent, output); + TextFormatters.Plain(theme: null, "{@m}").Format(eventJson, output); return output.ToString(); } @@ -146,36 +145,53 @@ public void DottedHoleNamesResolveThroughNestedStructures() } [Fact] - public void FlatPropertiesWinOverStructureTraversal() + public void UnresolvableDottedHolesRenderAsRawText() { - var evt = MakeDottedHoleEvent( - ("user.greeting.first", "G'day"), - ("user", JObject.Parse("""{"greeting": {"first": "Hello"}, "name": "Barney"}"""))); + var evt = MakeDottedHoleEvent(("user", JObject.Parse("""{"greeting": 42}"""))); - Assert.Equal("G'day Barney!", RenderMessage(evt)); + Assert.Equal("{user.greeting.first} {user.name}!", RenderMessage(evt)); + } + + static string CaptureConsoleOut(System.Action write) + { + var output = new StringWriter(); + var saved = System.Console.Out; + System.Console.SetOut(output); + try + { + write(); + } + finally + { + System.Console.SetOut(saved); + } + + return output.ToString(); } [Fact] - public void UnresolvableDottedHolesRenderAsRawText() + public void ObjectsAreWrittenAsSingleLineJson() { - var evt = MakeDottedHoleEvent(("user", JObject.Parse("""{"greeting": 42}"""))); + var format = Create(syntax: OutputSyntax.Json); - Assert.Equal("{user.greeting.first} {user.name}!", RenderMessage(evt)); + var written = CaptureConsoleOut(() => format.WriteObject( + new JObject(new JProperty("Title", "Errors"), new JProperty("Count", 42)))); + + Assert.Equal("""{"Title":"Errors","Count":42}""" + System.Environment.NewLine, written); } [Fact] - public void ResolvedScalarsAreUnwrappedFromTheirJsonRepresentation() + public void EntitiesAreWrittenAsJsonWithoutLinks() { - var evt = Some.MakeEvent(e => - { - e.MessageTemplateTokens = [new MessageTemplateTokenPart { PropertyName = "order.total" }]; - e.Properties = Some.MakeProperties(("order", JObject.Parse("""{"total": 42}"""))); - }); + var entity = new Seq.Api.Model.Signals.SignalEntity { Id = "signal-1", Title = "Errors" }; - var serilogEvent = OutputFormat.ToSerilogEvent(evt); - OutputFormat.FlattenPropertiesUsedWithDottedNames(evt, serilogEvent); + var format = Create(syntax: OutputSyntax.Json); + var written = CaptureConsoleOut(() => format.WriteEntity(entity)); - var scalar = Assert.IsType(serilogEvent.Properties["order.total"]); - Assert.Equal(42L, scalar.Value); + Assert.Contains("\"Id\":\"signal-1\"", written); + Assert.Contains("\"Title\":\"Errors\"", written); + Assert.DoesNotContain("Links", written); + Assert.EndsWith(System.Environment.NewLine, written); + Assert.Equal(written.TrimEnd(), written.TrimEnd().ReplaceLineEndings("")); } } diff --git a/test/SeqCli.Tests/Output/TextFormattersTests.cs b/test/SeqCli.Tests/Output/TextFormattersTests.cs index f3548e0d..7675926e 100644 --- a/test/SeqCli.Tests/Output/TextFormattersTests.cs +++ b/test/SeqCli.Tests/Output/TextFormattersTests.cs @@ -1,11 +1,11 @@ #nullable enable using System; using System.IO; +using System.Text.Json.Nodes; +using Seq.Syntax.Templates.Themes; +using SeqCli.Mapping; using SeqCli.Output; using SeqCli.Tests.Support; -using Serilog.Events; -using Serilog.Parsing; -using Serilog.Templates.Themes; using Xunit; namespace SeqCli.Tests.Output; @@ -13,7 +13,7 @@ namespace SeqCli.Tests.Output; public class TextFormattersTests { const char Escape = '\x1b'; - static readonly DateTimeOffset FixedTimestamp = new(2024, 1, 1, 10, 0, 1, 250, TimeSpan.Zero); + const string FixedTimestamp = "2024-01-01T10:00:01.2500000+00:00"; [Fact] public void ThemedJsonOutputIsColorizedRegardlessOfRedirection() @@ -27,12 +27,18 @@ public void UnthemedJsonOutputIsNotColorized() Assert.DoesNotContain(Escape, RenderJson(theme: null)); } + [Fact] + public void UnthemedJsonOutputIsTheEventDocumentVerbatim() + { + Assert.Equal( + """{"@t":"2024-01-01T10:00:01.2500000+00:00","@mt":"Hello, {Name}!","Name":"world"}""" + Environment.NewLine, + RenderJson(theme: null, SomeEventJson())); + } + [Fact] public void LogEventsAreFormattedWithTheDefaultTextTemplate() { - var evt = SomeLogEvent( - level: LogEventLevel.Warning, - properties: new LogEventProperty("Name", new ScalarValue("world"))); + var evt = SomeEventJson(level: "Warning"); Assert.Equal( $"[2024-01-01T10:00:01.2500000+00:00 WRN] Hello, world!{Environment.NewLine}", @@ -42,11 +48,7 @@ public void LogEventsAreFormattedWithTheDefaultTextTemplate() [Fact] public void ExceptionsAreIncludedInTextOutput() { - var evt = SomeLogEvent( - FixedTimestamp, - LogEventLevel.Error, - new Exception("Boom!"), - new LogEventProperty("Name", new ScalarValue("world"))); + var evt = SomeEventJson(level: "Error", exception: "System.Exception: Boom!"); Assert.Equal( $"[2024-01-01T10:00:01.2500000+00:00 ERR] Hello, world!{Environment.NewLine}System.Exception: Boom!{Environment.NewLine}", @@ -54,14 +56,10 @@ public void ExceptionsAreIncludedInTextOutput() } [Fact] - public void SpanElapsedTimeIsComputedFromTheStartTimestampProperty() + public void SpanElapsedTimeIsComputedFromTheStartTimestamp() { - // Events retrieved from the Seq API carry span start timestamps in ISO-8601 `@st` properties. - var evt = SomeLogEvent(FixedTimestamp, properties: - [ - new LogEventProperty("Name", new ScalarValue("world")), - new LogEventProperty("@st", new ScalarValue("2024-01-01T10:00:00.0000000Z")) - ]); + var evt = SomeEventJson(); + evt["@st"] = "2024-01-01T10:00:00.0000000Z"; Assert.Equal( $"[2024-01-01T10:00:01.2500000+00:00 INF] Hello, world! (1250 ms){Environment.NewLine}", @@ -69,53 +67,41 @@ public void SpanElapsedTimeIsComputedFromTheStartTimestampProperty() } [Fact] - public void SpanElapsedTimeIsComputedFromTheSurrogateStartTimestampProperty() + public void ACustomOutputTemplateReplacesTheDefault() { - // Ingested spans carry a surrogate `SpanStartTimestamp` property with a `DateTime` value. - var evt = SomeLogEvent(FixedTimestamp, properties: - [ - new LogEventProperty("Name", new ScalarValue("world")), - new LogEventProperty("SpanStartTimestamp", new ScalarValue( - FixedTimestamp.UtcDateTime.AddMilliseconds(-1.5))) - ]); - Assert.Equal( - $"[2024-01-01T10:00:01.2500000+00:00 INF] Hello, world! (1.5 ms){Environment.NewLine}", - RenderText(evt)); + $"INF Hello, world!{Environment.NewLine}", + RenderText(SomeEventJson(), $"{{@l:u3}} {{@m}}{Environment.NewLine}")); } - [Fact] - public void ACustomOutputTemplateReplacesTheDefault() + static JsonObject SomeEventJson(string? level = null, string? exception = null) { - var evt = SomeLogEvent(properties: new LogEventProperty("Name", new ScalarValue("world"))); + var evt = new JsonObject + { + ["@t"] = FixedTimestamp, + ["@mt"] = "Hello, {Name}!", + ["Name"] = "world" + }; - Assert.Equal($"INF Hello, world!{Environment.NewLine}", RenderText(evt, $"{{@l:u3}} {{@m}}{Environment.NewLine}")); - } + if (level != null) + evt["@l"] = level; - static LogEvent SomeLogEvent( - DateTimeOffset? timestamp = null, - LogEventLevel level = LogEventLevel.Information, - Exception? exception = null, - params LogEventProperty[] properties) - { - return new LogEvent( - timestamp ?? FixedTimestamp, - level, - exception, - new MessageTemplateParser().Parse("Hello, {Name}!"), - properties); + if (exception != null) + evt["@x"] = exception; + + return evt; } - static string RenderText(LogEvent evt, string? outputTemplate = null) + static string RenderText(JsonObject evt, string? outputTemplate = null) { var output = new StringWriter(); TextFormatters.Plain(theme: null, outputTemplate).Format(evt, output); return output.ToString(); } - static string RenderJson(TemplateTheme? theme) + static string RenderJson(TemplateTheme? theme, JsonObject? evt = null) { - var evt = OutputFormat.ToSerilogEvent(Some.MakeEvent(e => e.Properties = [])); + evt ??= EventEntityJson.ToEventJson(Some.MakeEvent(e => e.Properties = [])); var output = new StringWriter(); TextFormatters.Json(theme).Format(evt, output); diff --git a/test/SeqCli.Tests/Output/TraceFormatterTests.cs b/test/SeqCli.Tests/Output/TraceFormatterTests.cs index fce99356..9dc3a694 100644 --- a/test/SeqCli.Tests/Output/TraceFormatterTests.cs +++ b/test/SeqCli.Tests/Output/TraceFormatterTests.cs @@ -3,10 +3,9 @@ using System.Collections.Generic; using System.IO; using System.Linq; +using System.Text.Json.Nodes; using SeqCli.Output; using SeqCli.Traces; -using Serilog.Events; -using Serilog.Parsing; using Xunit; namespace SeqCli.Tests.Output; @@ -18,14 +17,14 @@ public class TraceFormatterTests static TraceTreeElement Span(string spanId, string? parentId, double startMs = 0, double elapsedMs = 1, string? message = null, IReadOnlyList? columns = null) => new($"event-span-{spanId}", T0.AddMilliseconds(startMs + elapsedMs), null, - new MessageTemplate([new TextToken(message ?? $"span {spanId}")]), [], + message ?? $"span {spanId}", new JsonObject(), null, spanId, parentId, T0.AddMilliseconds(startMs), TimeSpan.FromMilliseconds(elapsedMs), columns ?? []); static TraceTreeElement Log(string? spanId, double timestampMs, string message = "log", string? level = null, string? exception = null) => new($"event-log-{timestampMs}-{message}", T0.AddMilliseconds(timestampMs), level, - new MessageTemplate([new TextToken(message)]), [], exception, + message, new JsonObject(), exception, spanId, null, null, null, []); static string Render(params TraceTreeElement[] events) @@ -33,8 +32,8 @@ static string Render(params TraceTreeElement[] events) var output = new StringWriter(); var formatter = TextFormatters.Plain(theme: null, TraceFormatter.OutputTemplate(events.Max(e => e.Columns.Count))); - foreach (var logEvent in TraceFormatter.ToLogEvents(TraceTreeBuilder.Build(events))) - formatter.Format(logEvent, output); + foreach (var eventJson in TraceFormatter.ToEventJson(TraceTreeBuilder.Build(events))) + formatter.Format(eventJson, output); return output.ToString(); } @@ -115,12 +114,8 @@ public void MissingAndEmptyColumnValuesLeaveNoRedundantSpace(object? first) public void TemplateHolesAreFilledFromMessageProperties() { var evt = new TraceTreeElement("event-1", T0.AddMilliseconds(1.5), null, - new MessageTemplate([ - new TextToken("GET "), - new PropertyToken("Route", "{Route}"), - new TextToken(" as "), - new PropertyToken("User", "{User}")]), - [new LogEventProperty("Route", new ScalarValue("/orders"))], + "GET {Route} as {User}", + new JsonObject { ["Route"] = "/orders" }, null, "a", null, T0, TimeSpan.FromMilliseconds(1.5), []); Assert.Equal( diff --git a/test/SeqCli.Tests/PlainText/EventJsonBuilderTests.cs b/test/SeqCli.Tests/PlainText/EventJsonBuilderTests.cs new file mode 100644 index 00000000..0df85264 --- /dev/null +++ b/test/SeqCli.Tests/PlainText/EventJsonBuilderTests.cs @@ -0,0 +1,60 @@ +#nullable enable +using System; +using System.Collections.Generic; +using System.Globalization; +using SeqCli.PlainText.LogEvents; +using Superpower.Model; +using Xunit; + +namespace SeqCli.Tests.PlainText; + +public class EventJsonBuilderTests +{ + [Fact] + public void SuppliedValuesAreUsed() + { + var properties = new Dictionary + { + ["@t"] = new TextSpan("2018-02-01T13:00:00.123Z"), + ["@l"] = new TextSpan("WRN"), + ["@m"] = new TextSpan("Hello, world"), + ["@x"] = new TextSpan("EverythingFailedException"), + ["MachineName"] = new TextSpan("TP"), + ["Count"] = 42 + }; + + var remainder = "rem"; + var evt = EventJsonBuilder.FromProperties(properties, remainder); + + Assert.Equal("2018-02-01T13:00:00.1230000+00:00", + DateTimeOffset.Parse((string)evt["@t"]!, CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind) + .ToUniversalTime().ToString("o")); + Assert.Equal("Hello, world", (string?)evt["@m"]); + Assert.Equal("WRN", (string?)evt["@l"]); + Assert.Equal("EverythingFailedException", (string?)evt["@x"]); + Assert.Equal(42, (int?)evt["Count"]); + Assert.Equal("TP", (string?)evt["MachineName"]); + Assert.Equal("rem", (string?)evt["@@unmatched"]); + } + + [Fact] + public void MissingValuesAreDefaulted() + { + var evt = EventJsonBuilder.FromProperties(new Dictionary(), null); + + var timestamp = DateTimeOffset.Parse((string)evt["@t"]!, CultureInfo.InvariantCulture, + DateTimeStyles.RoundtripKind); + Assert.True(timestamp > DateTimeOffset.Now.AddSeconds(-5)); + Assert.False(evt.ContainsKey("@m")); + Assert.False(evt.ContainsKey("@l")); + Assert.False(evt.ContainsKey("@x")); + } + + [Fact] + public void DateTimeOffsetTimestampsAreAccepted() + { + var then = DateTimeOffset.Now.AddDays(-5); + var evt = EventJsonBuilder.FromProperties(new Dictionary{["@t"] = then}, null); + Assert.Equal(then.ToString("o", CultureInfo.InvariantCulture), (string?)evt["@t"]); + } +} diff --git a/test/SeqCli.Tests/PlainText/LogEventBuilderTests.cs b/test/SeqCli.Tests/PlainText/LogEventBuilderTests.cs deleted file mode 100644 index 75eaf8d5..00000000 --- a/test/SeqCli.Tests/PlainText/LogEventBuilderTests.cs +++ /dev/null @@ -1,56 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using SeqCli.PlainText.LogEvents; -using Serilog.Events; -using Superpower.Model; -using Xunit; - -namespace SeqCli.Tests.PlainText; - -public class LogEventBuilderTests -{ - [Fact] - public void SuppliedValuesAreUsed() - { - var properties = new Dictionary - { - ["@t"] = new TextSpan("2018-02-01T13:00:00.123Z"), - ["@l"] = new TextSpan("WRN"), - ["@m"] = new TextSpan("Hello, world"), - ["@x"] = new TextSpan("EverythingFailedException"), - ["MachineName"] = new TextSpan("TP"), - ["Count"] = 42 - }; - - var remainder = "rem"; - var evt = LogEventBuilder.FromProperties(properties, remainder); - - Assert.Equal("2018-02-01T13:00:00.1230000+00:00", evt.Timestamp.ToString("o")); - Assert.Equal("Hello, world", evt.RenderMessage()); - Assert.Equal(LogEventLevel.Warning, evt.Level); - Assert.Equal("EverythingFailedException", evt.Exception?.ToString()); - Assert.Equal(42, ((ScalarValue)evt.Properties["Count"]).Value); - Assert.Equal("TP", ((ScalarValue)evt.Properties["MachineName"]).Value!.ToString()); - Assert.Equal("rem", ((ScalarValue)evt.Properties["@unmatched"]).Value!.ToString()); - } - - [Fact] - public void MissingValuesAreDefaulted() - { - var evt = LogEventBuilder.FromProperties(new Dictionary(), null); - - Assert.True(evt.Timestamp > DateTimeOffset.Now.AddSeconds(-5)); - Assert.Equal("", evt.RenderMessage()); - Assert.Equal(LogEventLevel.Information, evt.Level); - Assert.Null(evt.Exception); - } - - [Fact] - public void DateTimeOffsetTimestampsAreAccepted() - { - var then = DateTimeOffset.Now.AddDays(-5); - var evt = LogEventBuilder.FromProperties(new Dictionary{["@t"] = then}, null); - Assert.Equal(then, evt.Timestamp); - } -} \ No newline at end of file diff --git a/test/SeqCli.Tests/PlainText/StaticMessageTemplateReaderTests.cs b/test/SeqCli.Tests/PlainText/StaticMessageTemplateReaderTests.cs index 29f99471..9c68a1d5 100644 --- a/test/SeqCli.Tests/PlainText/StaticMessageTemplateReaderTests.cs +++ b/test/SeqCli.Tests/PlainText/StaticMessageTemplateReaderTests.cs @@ -1,4 +1,5 @@ -using System.Threading.Tasks; +#nullable enable +using System.Threading.Tasks; using SeqCli.Ingestion; using SeqCli.Tests.Support; using Xunit; @@ -10,11 +11,13 @@ public class StaticMessageTemplateReaderTests [Fact] public async Task ReaderSubstitutesMessageTemplate() { - var evt = Some.LogEvent(); + var evt = Some.EventJson(); + evt["@m"] = "A pre-rendered message"; const string mt = "This is a message template"; - var reader = new FixedLogEventReader(new ReadResult(evt, false)); + var reader = new FixedEventReader(new ReadResult(evt, false)); var wrapper = new StaticMessageTemplateReader(reader, mt); var result = await wrapper.TryReadAsync(); - Assert.Equal(mt, result.LogEvent.MessageTemplate.Text); + Assert.Equal(mt, (string?)result.Document!["@mt"]); + Assert.False(result.Document.ContainsKey("@m")); } -} \ No newline at end of file +} diff --git a/test/SeqCli.Tests/Support/FixedLogEventReader.cs b/test/SeqCli.Tests/Support/FixedEventReader.cs similarity index 73% rename from test/SeqCli.Tests/Support/FixedLogEventReader.cs rename to test/SeqCli.Tests/Support/FixedEventReader.cs index 538c5b65..2c29398f 100644 --- a/test/SeqCli.Tests/Support/FixedLogEventReader.cs +++ b/test/SeqCli.Tests/Support/FixedEventReader.cs @@ -3,11 +3,11 @@ namespace SeqCli.Tests.Support; -class FixedLogEventReader : ILogEventReader +class FixedEventReader : IEventReader { readonly ReadResult _result; - public FixedLogEventReader(ReadResult result) + public FixedEventReader(ReadResult result) { _result = result; } diff --git a/test/SeqCli.Tests/Support/Some.cs b/test/SeqCli.Tests/Support/Some.cs index 7ba27d31..0a2aa24e 100644 --- a/test/SeqCli.Tests/Support/Some.cs +++ b/test/SeqCli.Tests/Support/Some.cs @@ -1,11 +1,10 @@ -using System; +using System; using System.Collections.Generic; using System.Linq; using System.Security.Cryptography; +using System.Text.Json.Nodes; using Seq.Api.Model.Events; using Seq.Api.Model.Shared; -using Serilog.Events; -using Serilog.Parsing; namespace SeqCli.Tests.Support; @@ -15,14 +14,13 @@ static class Some { static readonly RandomNumberGenerator Rng = RandomNumberGenerator.Create(); - public static LogEvent LogEvent() + public static JsonObject EventJson() { - return new LogEvent( - DateTimeOffset.UtcNow, - LogEventLevel.Information, - null, - new MessageTemplateParser().Parse("Test"), - Enumerable.Empty()); + return new JsonObject + { + ["@t"] = DateTimeOffset.UtcNow.ToString("o"), + ["@mt"] = "Test" + }; } public static string String() @@ -41,7 +39,7 @@ public static byte[] Bytes(int count) Rng.GetBytes(bytes); return bytes; } - + public static EventEntity MakeEvent(Action? configure = null) { var evt = new EventEntity @@ -58,4 +56,4 @@ public static EventEntity MakeEvent(Action? configure = null) public static List MakeProperties(params (string Name, object? Value)[] items) => items.Select(i => new EventPropertyPart(i.Name, i.Value)).ToList(); -} \ No newline at end of file +} diff --git a/test/SeqCli.Tests/Traces/StructuredMessageTests.cs b/test/SeqCli.Tests/Traces/StructuredMessageTests.cs index 731a0bb8..4180126e 100644 --- a/test/SeqCli.Tests/Traces/StructuredMessageTests.cs +++ b/test/SeqCli.Tests/Traces/StructuredMessageTests.cs @@ -1,10 +1,8 @@ #nullable enable using System.IO; -using System.Linq; +using System.Text.Json.Nodes; using Newtonsoft.Json.Linq; using SeqCli.Traces; -using Serilog.Events; -using Serilog.Parsing; using Xunit; namespace SeqCli.Tests.Traces; @@ -25,7 +23,7 @@ public void MissingStructuredMessagesReadAsEmpty() foreach (var cell in new object?[] { null, JValue.CreateNull() }) { var (message, properties) = StructuredMessage.Read(cell); - Assert.Empty(message.Tokens); + Assert.Equal("", message); Assert.Empty(properties); } } @@ -35,24 +33,29 @@ public void TextTokensAreRead() { var (message, properties) = StructuredMessage.Read(new JArray("Hello", ", ", "world")); - Assert.Equal("Hello, world", message.Text); - Assert.All(message.Tokens, token => Assert.IsType(token)); + Assert.Equal("Hello, world", message); Assert.Empty(properties); } + [Fact] + public void LiteralBracesAreEscapedInTemplateText() + { + var (message, _) = StructuredMessage.Read(new JArray("a {not-a-hole} b")); + + Assert.Equal("a {{not-a-hole}} b", message); + } + [Fact] public void HolesCarryRawTextAndValues() { var (message, properties) = StructuredMessage.Read(new JArray( "Hello, ", Hole("Name", "{Name:x}", "World"), "!")); - Assert.Equal("Hello, {Name:x}!", message.Text); - var hole = Assert.IsType(message.Tokens.ElementAt(1)); - Assert.Equal("Name", hole.PropertyName); + Assert.Equal("Hello, {Name:x}!", message); var property = Assert.Single(properties); - Assert.Equal("Name", property.Name); - Assert.Equal(new ScalarValue("World"), property.Value); + Assert.Equal("Name", property.Key); + Assert.Equal("World", (string?)property.Value); } [Fact] @@ -60,7 +63,7 @@ public void HolesWithoutValuesContributeNoProperties() { var (message, properties) = StructuredMessage.Read(new JArray(Hole("Name"))); - Assert.Equal("{Name}", message.Text); + Assert.Equal("{Name}", message); Assert.Empty(properties); } @@ -74,22 +77,32 @@ public void DuplicateHolesContributeASingleProperty() } [Fact] - public void ScalarHoleValuesAreUnwrapped() + public void ScalarHoleValuesAreRead() { var (_, properties) = StructuredMessage.Read(new JArray(Hole("Count", value: 42L))); - var scalar = Assert.IsType(Assert.Single(properties).Value); - Assert.Equal(42L, scalar.Value); + Assert.Equal(42L, (long?)Assert.Single(properties).Value); } [Fact] - public void StructuredHoleValuesBecomeStructures() + public void StructuredHoleValuesBecomeObjects() { var (_, properties) = StructuredMessage.Read(new JArray( Hole("Order", value: new JObject(new JProperty("Id", 7))))); - var structure = Assert.IsType(Assert.Single(properties).Value); - Assert.Equal("Id", Assert.Single(structure.Properties).Name); + var structure = Assert.IsType(Assert.Single(properties).Value); + Assert.Equal(7, (int?)structure["Id"]); + } + + [Fact] + public void DottedHoleNamesBecomeNestedObjects() + { + var (message, properties) = StructuredMessage.Read(new JArray( + Hole("user.name", value: "Barney"))); + + Assert.Equal("{user.name}", message); + var user = Assert.IsType(properties["user"]); + Assert.Equal("Barney", (string?)user["name"]); } [Fact] @@ -97,7 +110,7 @@ public void TrailingWhitespaceIsTrimmed() { var (message, _) = StructuredMessage.Read(new JArray("Hi ", "}", " \n")); - Assert.Equal("Hi }", message.Text); + Assert.Equal("Hi }}", message); } [Fact] @@ -105,7 +118,7 @@ public void WhitespaceOnlyMessagesReadAsEmpty() { var (message, _) = StructuredMessage.Read(new JArray(" ")); - Assert.Empty(message.Tokens); + Assert.Equal("", message); } [Fact] @@ -113,7 +126,7 @@ public void TrailingHolesAreNotTrimmed() { var (message, _) = StructuredMessage.Read(new JArray("Took ", Hole("Elapsed"))); - Assert.Equal("Took {Elapsed}", message.Text); + Assert.Equal("Took {Elapsed}", message); } [Fact] diff --git a/test/SeqCli.Tests/Traces/TraceQueryTests.cs b/test/SeqCli.Tests/Traces/TraceQueryTests.cs index fb282be5..84a5baab 100644 --- a/test/SeqCli.Tests/Traces/TraceQueryTests.cs +++ b/test/SeqCli.Tests/Traces/TraceQueryTests.cs @@ -3,7 +3,6 @@ using Newtonsoft.Json.Linq; using Seq.Api.Model.Data; using SeqCli.Traces; -using Serilog.Events; using Xunit; namespace SeqCli.Tests.Traces; @@ -85,7 +84,7 @@ public void SpanRowsAreRead() Assert.Equal("event-1", evt.Id); Assert.Equal(timestamp, evt.Timestamp); Assert.Equal("INFO", evt.Level); - Assert.Equal("Hello!", evt.MessageTemplate.Text); + Assert.Equal("Hello!", evt.MessageTemplate); Assert.Empty(evt.TemplateProperties); Assert.Null(evt.Exception); Assert.Equal("0011223344556677", evt.SpanId); @@ -157,10 +156,10 @@ public void StructuredMessageHolesBecomeTemplatePropertiesAndValues() var evt = Assert.Single(TraceQuery.ReadEvents(result, includeExceptions: true, [])); - Assert.Equal("Hello, {Name}!", evt.MessageTemplate.Text); + Assert.Equal("Hello, {Name}!", evt.MessageTemplate); var property = Assert.Single(evt.TemplateProperties); - Assert.Equal("Name", property.Name); - Assert.Equal(new ScalarValue("World"), property.Value); + Assert.Equal("Name", property.Key); + Assert.Equal("World", (string?)property.Value); } [Fact] diff --git a/test/SeqCli.Tests/Traces/TraceTreeBuilderTests.cs b/test/SeqCli.Tests/Traces/TraceTreeBuilderTests.cs index 2511109f..2f16c2f9 100644 --- a/test/SeqCli.Tests/Traces/TraceTreeBuilderTests.cs +++ b/test/SeqCli.Tests/Traces/TraceTreeBuilderTests.cs @@ -1,9 +1,8 @@ #nullable enable using System; using System.Linq; +using System.Text.Json.Nodes; using SeqCli.Traces; -using Serilog.Events; -using Serilog.Parsing; using Xunit; namespace SeqCli.Tests.Traces; @@ -14,12 +13,12 @@ public class TraceTreeBuilderTests static TraceTreeElement Span(string spanId, string? parentId, double startMs = 0, double elapsedMs = 1) => new($"event-span-{spanId}", T0.AddMilliseconds(startMs + elapsedMs), null, - new MessageTemplate([new TextToken($"span {spanId}")]), [], null, + $"span {spanId}", new JsonObject(), null, spanId, parentId, T0.AddMilliseconds(startMs), TimeSpan.FromMilliseconds(elapsedMs), []); static TraceTreeElement Log(string? spanId, double timestampMs, string message = "log") => new($"event-log-{timestampMs}-{message}", T0.AddMilliseconds(timestampMs), null, - new MessageTemplate([new TextToken(message)]), [], null, + message, new JsonObject(), null, spanId, null, null, null, []); [Fact] @@ -76,7 +75,7 @@ public void SiblingSpansAndLogsInterleaveChronologically() var root = Assert.Single(roots); Assert.Equal( ["first", "span c", "span b", "last"], - root.Children.Select(c => c.Element.MessageTemplate.Text).ToArray()); + root.Children.Select(c => c.Element.MessageTemplate).ToArray()); } [Fact] @@ -104,7 +103,7 @@ public void OrphanLogsBecomeRootsAlongsideSpans() Assert.Equal( ["span a", "first", "second", "span b"], - roots.Select(r => r.Element.MessageTemplate.Text).ToArray()); + roots.Select(r => r.Element.MessageTemplate).ToArray()); Assert.All(roots, r => Assert.Empty(r.Children)); } @@ -117,7 +116,7 @@ public void OrphanLogsWithNoRootSpanRemainAtRootLevel() ]); Assert.Equal(2, roots.Count); - Assert.Equal(["first", "second"], roots.Select(r => r.Element.MessageTemplate.Text).ToArray()); + Assert.Equal(["first", "second"], roots.Select(r => r.Element.MessageTemplate).ToArray()); } [Fact] diff --git a/test/SeqCli.Tests/Traces/TraceTreeJObjectConverterTests.cs b/test/SeqCli.Tests/Traces/TraceTreeJObjectConverterTests.cs index b0dfa1c9..6eb90582 100644 --- a/test/SeqCli.Tests/Traces/TraceTreeJObjectConverterTests.cs +++ b/test/SeqCli.Tests/Traces/TraceTreeJObjectConverterTests.cs @@ -2,10 +2,9 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Text.Json.Nodes; using Newtonsoft.Json.Linq; using SeqCli.Traces; -using Serilog.Events; -using Serilog.Parsing; using Xunit; namespace SeqCli.Tests.Traces; @@ -20,14 +19,14 @@ static TraceTreeElement Span(string spanId, string? parentId, double startMs = 0 string? message = null, string? level = null, string? exception = null, IReadOnlyList? columns = null) => new($"event-span-{spanId}", T0.AddMilliseconds(startMs + elapsedMs), level, - new MessageTemplate([new TextToken(message ?? $"span {spanId}")]), [], + message ?? $"span {spanId}", new JsonObject(), exception, spanId, parentId, T0.AddMilliseconds(startMs), TimeSpan.FromMilliseconds(elapsedMs), columns ?? []); static TraceTreeElement Log(string? spanId, double timestampMs, string message = "log", string? level = null, string? exception = null, IReadOnlyList? columns = null) => new($"event-log-{timestampMs}-{message}", T0.AddMilliseconds(timestampMs), level, - new MessageTemplate([new TextToken(message)]), [], exception, + message, new JsonObject(), exception, spanId, null, null, null, columns ?? []); static JObject ToJson(params TraceTreeElement[] events) => ToJson([], events); @@ -142,30 +141,13 @@ public void LogsWithNoCapturedEnclosingSpanBecomeOrphans() Assert.Equal("uncaptured", (string?)orphans[0]["spanId"]); Assert.Null(orphans[2]["spanId"]); } - - [Fact] - public void LevelsAreNormalizedToFullNames() - { - var document = ToJson( - Span("a", null), - Log("a", 1, level: "warn"), - Log("a", 2, level: "Nonstandard")); - - var children = (JArray)document["root"]!["children"]!; - Assert.Equal("Warning", (string?)children[0]["level"]); - Assert.Equal("Nonstandard", (string?)children[1]["level"]); - } - + [Fact] public void TemplateHolesAreFilledFromMessageProperties() { var evt = new TraceTreeElement("event-1", T0.AddMilliseconds(1.5), null, - new MessageTemplate([ - new TextToken("GET "), - new PropertyToken("Route", "{Route}"), - new TextToken(" as "), - new PropertyToken("User", "{User}")]), - [new LogEventProperty("Route", new ScalarValue("/orders"))], + "GET {Route} as {User}", + new JsonObject { ["Route"] = "/orders" }, null, "a", null, T0, TimeSpan.FromMilliseconds(1.5), []); var document = ToJson(evt); From 96b40197bb9daea2083d237e28d735524df0f5e3 Mon Sep 17 00:00:00 2001 From: Nicholas Blumhardt Date: Tue, 1 Sep 2026 15:55:07 +1000 Subject: [PATCH 02/15] Localize Serilog event conversion to the sample application - we don't want easy or obvious conversions into Serilog types, the set of scenarios that require this should be and stay vanishingly small --- src/SeqCli/Forwarder/ForwarderModule.cs | 21 ++----- .../Web/Api/IngestionLogEndpoints.cs | 58 +++++++++++++++++-- src/SeqCli/Ingestion/JsonEventReader.cs | 6 +- .../Ingestion/SerilogEventJson.cs | 2 +- .../Ingestion/SerilogTracingConventions.cs | 8 +-- .../SerilogEventJsonTests.cs | 3 +- 6 files changed, 63 insertions(+), 35 deletions(-) rename src/SeqCli/{ => Sample}/Ingestion/SerilogEventJson.cs (98%) rename src/SeqCli/{ => Sample}/Ingestion/SerilogTracingConventions.cs (78%) rename test/SeqCli.Tests/{Ingestion => Sample}/SerilogEventJsonTests.cs (98%) diff --git a/src/SeqCli/Forwarder/ForwarderModule.cs b/src/SeqCli/Forwarder/ForwarderModule.cs index 3980787d..9ed614b4 100644 --- a/src/SeqCli/Forwarder/ForwarderModule.cs +++ b/src/SeqCli/Forwarder/ForwarderModule.cs @@ -21,7 +21,6 @@ using SeqCli.Forwarder.Channel; using SeqCli.Forwarder.Web.Api; using SeqCli.Forwarder.Web.Host; -using SeqCli.Syntax; using Serilog; namespace SeqCli.Forwarder; @@ -65,25 +64,17 @@ protected override void Load(ContainerBuilder builder) if (_config.Forwarder.Diagnostics.ExposeIngestionLog) { Log.ForContext().Warning("Configured to expose ingestion log via HTTP API"); - builder.RegisterType().As(); - - var ingestionLogTemplate = $"[{{@Timestamp:o}} {{@Level:u3}}] {{@Message}}{Environment.NewLine}"; if (_config.Forwarder.Diagnostics.IngestionLogShowDetail) { Log.ForContext().Warning("Including full client, payload, and error detail in the ingestion log"); - ingestionLogTemplate += - $"{{#if ClientHostIP is not null}}Client IP address: {{ClientHostIP}}{Environment.NewLine}{{#end}}" + - $"{{#if DocumentStart is not null}}First {{StartToLog}} characters of payload: {{DocumentStart:l}}{Environment.NewLine}{{#end}}" + - "{@Exception}"; } - - builder.Register(_ => SeqSyntax.ParseTemplate(ingestionLogTemplate)); + + builder.Register(_ => new IngestionLogEndpoints(_config.Forwarder.Diagnostics.IngestionLogShowDetail)).As(); } - builder.Register(c => + builder.Register(_ => { - var config = c.Resolve(); - var baseUri = config.Connection.ServerUrl; + var baseUri = _config.Connection.ServerUrl; if (string.IsNullOrWhiteSpace(baseUri)) throw new ArgumentException("The destination Seq server URL must be configured in `SeqCli.json`."); @@ -94,13 +85,13 @@ protected override void Load(ContainerBuilder builder) // this expression, using an "or" operator. var hasSocketHandlerOption = - config.Connection.PooledConnectionLifetimeMilliseconds.HasValue; + _config.Connection.PooledConnectionLifetimeMilliseconds.HasValue; if (hasSocketHandlerOption) { var httpMessageHandler = new SocketsHttpHandler { - PooledConnectionLifetime = config.Connection.PooledConnectionLifetimeMilliseconds.HasValue ? TimeSpan.FromMilliseconds(config.Connection.PooledConnectionLifetimeMilliseconds.Value) : Timeout.InfiniteTimeSpan, + PooledConnectionLifetime = _config.Connection.PooledConnectionLifetimeMilliseconds.HasValue ? TimeSpan.FromMilliseconds(_config.Connection.PooledConnectionLifetimeMilliseconds.Value) : Timeout.InfiniteTimeSpan, }; return new HttpClient(httpMessageHandler) { BaseAddress = new Uri(baseUri) }; diff --git a/src/SeqCli/Forwarder/Web/Api/IngestionLogEndpoints.cs b/src/SeqCli/Forwarder/Web/Api/IngestionLogEndpoints.cs index eb98cc58..eba1c743 100644 --- a/src/SeqCli/Forwarder/Web/Api/IngestionLogEndpoints.cs +++ b/src/SeqCli/Forwarder/Web/Api/IngestionLogEndpoints.cs @@ -12,24 +12,25 @@ // See the License for the specific language governing permissions and // limitations under the License. +using System; +using System.Globalization; using System.IO; using System.Text; using Microsoft.AspNetCore.Builder; using Microsoft.AspNetCore.Http; -using Seq.Syntax.Templates; using SeqCli.Forwarder.Diagnostics; -using SeqCli.Ingestion; +using Serilog.Events; namespace SeqCli.Forwarder.Web.Api; class IngestionLogEndpoints : IMapEndpoints { - readonly ExpressionTemplate _formatter; + readonly bool _showDetail; readonly Encoding _utf8 = new UTF8Encoding(false); - public IngestionLogEndpoints(ExpressionTemplate formatter) + public IngestionLogEndpoints(bool showDetail) { - _formatter = formatter; + _showDetail = showDetail; } public void MapEndpoints(WebApplication app) @@ -46,10 +47,55 @@ public void MapEndpoints(WebApplication app) using var log = new StringWriter(); foreach (var logEvent in events) { - _formatter.Format(SerilogEventJson.ToEventJson(logEvent), log); + Format(logEvent, log); } return Results.Content(log.ToString(), "text/plain", _utf8); }); } + + void Format(LogEvent logEvent, TextWriter log) + { + log.Write($"[{logEvent.Timestamp:o} {Abbreviate(logEvent.Level)}] "); + + static string Abbreviate(LogEventLevel logEventLevel) + { + // Here because we don't want Serilog level conversion routines, or any other Serilog model conversion + // routines, to propagate. + return logEventLevel switch + { + LogEventLevel.Verbose => "VRB", + LogEventLevel.Debug => "DBG", + LogEventLevel.Information => "INF", + LogEventLevel.Warning => "WAR", + LogEventLevel.Error => "ERR", + LogEventLevel.Fatal => "FTL", + _ => throw new ArgumentOutOfRangeException(nameof(logEventLevel), logEventLevel, null) + }; + } + + logEvent.RenderMessage(log, CultureInfo.InvariantCulture); + log.WriteLine(); + if (_showDetail) + { + if (logEvent.Properties.TryGetValue("ClientHostIP", out var clientHostIPProperty) && + clientHostIPProperty is ScalarValue { Value: string clientHostIP}) + { + log.WriteLine($"Client IP address: {clientHostIP}"); + } + + if (logEvent.Properties.TryGetValue("DocumentStart", out var documentStartProperty) && + documentStartProperty is ScalarValue { Value: string documentStart} && + logEvent.Properties.TryGetValue("StartToLog", out var startToLogProperty) && + startToLogProperty is ScalarValue { Value: {} startToLog }) + { + log.WriteLine($"First {startToLog} characters of payload: {documentStart}"); + } + + if (logEvent.Exception is { } exception) + { + log.WriteLine(exception); + } + } + } } diff --git a/src/SeqCli/Ingestion/JsonEventReader.cs b/src/SeqCli/Ingestion/JsonEventReader.cs index b7a102f6..a8d4be8c 100644 --- a/src/SeqCli/Ingestion/JsonEventReader.cs +++ b/src/SeqCli/Ingestion/JsonEventReader.cs @@ -49,16 +49,14 @@ public async Task TryReadAsync() return new ReadResult(ReadFromJson(frame.Value), frame.IsAtEnd); } - public static JsonObject ReadFromJson(string json) + static JsonObject ReadFromJson(string json) { if (JsonNode.Parse(json) is not JsonObject eventJson) throw new InvalidDataException($"The line is not a JSON object: `{json.Trim()}`."); if (!eventJson.ContainsKey("@t")) eventJson["@t"] = DateTime.UtcNow.ToString("O", CultureInfo.InvariantCulture); - - SerilogTracingConventions.LiftSpanProperties(eventJson); - + return eventJson; } } diff --git a/src/SeqCli/Ingestion/SerilogEventJson.cs b/src/SeqCli/Sample/Ingestion/SerilogEventJson.cs similarity index 98% rename from src/SeqCli/Ingestion/SerilogEventJson.cs rename to src/SeqCli/Sample/Ingestion/SerilogEventJson.cs index 3a166419..4750e756 100644 --- a/src/SeqCli/Ingestion/SerilogEventJson.cs +++ b/src/SeqCli/Sample/Ingestion/SerilogEventJson.cs @@ -18,7 +18,7 @@ using SeqCli.Syntax; using Serilog.Events; -namespace SeqCli.Ingestion; +namespace SeqCli.Sample.Ingestion; /// /// Converts Serilog events produced within seqcli itself — the sample ingest simulation diff --git a/src/SeqCli/Ingestion/SerilogTracingConventions.cs b/src/SeqCli/Sample/Ingestion/SerilogTracingConventions.cs similarity index 78% rename from src/SeqCli/Ingestion/SerilogTracingConventions.cs rename to src/SeqCli/Sample/Ingestion/SerilogTracingConventions.cs index e6eb029e..f08bc184 100644 --- a/src/SeqCli/Ingestion/SerilogTracingConventions.cs +++ b/src/SeqCli/Sample/Ingestion/SerilogTracingConventions.cs @@ -14,14 +14,8 @@ using System.Text.Json.Nodes; -namespace SeqCli.Ingestion; +namespace SeqCli.Sample.Ingestion; -/// -/// SerilogTracing emits span fields as regular event properties, because Serilog's data model -/// has nowhere else to put them. Events passing through seqcli lift these into the reified -/// @st and @ps fields so that they're recognized as spans by Seq and by seqcli's -/// own output formatting. -/// static class SerilogTracingConventions { internal const string ParentSpanIdProperty = "ParentSpanId"; diff --git a/test/SeqCli.Tests/Ingestion/SerilogEventJsonTests.cs b/test/SeqCli.Tests/Sample/SerilogEventJsonTests.cs similarity index 98% rename from test/SeqCli.Tests/Ingestion/SerilogEventJsonTests.cs rename to test/SeqCli.Tests/Sample/SerilogEventJsonTests.cs index f27aaf1f..256d7a3f 100644 --- a/test/SeqCli.Tests/Ingestion/SerilogEventJsonTests.cs +++ b/test/SeqCli.Tests/Sample/SerilogEventJsonTests.cs @@ -1,13 +1,12 @@ #nullable enable using System; using System.Linq; -using SeqCli.Ingestion; using SeqCli.Sample.Ingestion; using Serilog; using Serilog.Events; using Xunit; -namespace SeqCli.Tests.Ingestion; +namespace SeqCli.Tests.Sample; public class SerilogEventJsonTests { From d53501719d21c2b915399618a2940a0c204e6ee8 Mon Sep 17 00:00:00 2001 From: Nicholas Blumhardt Date: Tue, 1 Sep 2026 16:16:19 +1000 Subject: [PATCH 03/15] Trim down more --- src/SeqCli/Apps/AppLoader.cs | 5 +-- src/SeqCli/Sample/Ingestion/BufferingSink.cs | 2 +- src/SeqCli/Sample/Ingestion/MetricsMapping.cs | 6 +-- .../Ingestion/SerilogTracingConventions.cs | 40 ------------------- ...SerilogEventJson.cs => SimulationEvent.cs} | 32 ++++++++++----- ...ntJsonTests.cs => SimulationEventTests.cs} | 12 +++--- 6 files changed, 35 insertions(+), 62 deletions(-) delete mode 100644 src/SeqCli/Sample/Ingestion/SerilogTracingConventions.cs rename src/SeqCli/Sample/Ingestion/{SerilogEventJson.cs => SimulationEvent.cs} (74%) rename test/SeqCli.Tests/Sample/{SerilogEventJsonTests.cs => SimulationEventTests.cs} (90%) diff --git a/src/SeqCli/Apps/AppLoader.cs b/src/SeqCli/Apps/AppLoader.cs index 143eb91a..e46cdfed 100644 --- a/src/SeqCli/Apps/AppLoader.cs +++ b/src/SeqCli/Apps/AppLoader.cs @@ -29,13 +29,12 @@ class AppLoader : IDisposable readonly string _packageBinaryPath; // These are used for interop between the host process and the app. The - // app _must_ be able to load on the unified version. Apps built against Seq.Syntax v1 - // bundle their own `Seq.Syntax.dll`, which loads side-by-side with the host's - // `Seq.Syntax.V2.dll`. + // app _must_ be able to load on the unified version. readonly Assembly[] _contracts = [ typeof(SeqApp).Assembly, typeof(Log).Assembly, + // Seq.Syntax uses version-specific assembly names to improve our chances of successful loading. typeof(SeqExpression).Assembly ]; diff --git a/src/SeqCli/Sample/Ingestion/BufferingSink.cs b/src/SeqCli/Sample/Ingestion/BufferingSink.cs index cab3a4d3..59b0225d 100644 --- a/src/SeqCli/Sample/Ingestion/BufferingSink.cs +++ b/src/SeqCli/Sample/Ingestion/BufferingSink.cs @@ -25,7 +25,7 @@ public void Emit(LogEvent logEvent) var document = MetricsMapping.TryGetMetricSampleJson(logEvent, out var sample) ? sample - : SerilogEventJson.ToEventJson(logEvent); + : SimulationEvent.ToJsonObject(logEvent); _queue.Enqueue(document); } diff --git a/src/SeqCli/Sample/Ingestion/MetricsMapping.cs b/src/SeqCli/Sample/Ingestion/MetricsMapping.cs index 8ba3771f..12fbb91f 100644 --- a/src/SeqCli/Sample/Ingestion/MetricsMapping.cs +++ b/src/SeqCli/Sample/Ingestion/MetricsMapping.cs @@ -1,4 +1,4 @@ -// Copyright © Datalust and contributors. +// Copyright © Datalust and contributors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -45,13 +45,13 @@ public static bool TryGetMetricSampleJson(LogEvent logEvent, [NotNullWhen(true)] sample = new JsonObject { ["@t"] = logEvent.Timestamp.ToString("o", CultureInfo.InvariantCulture), - ["@d"] = SerilogEventJson.ToJsonNode(definitions) + ["@d"] = SimulationEvent.ToJsonNode(definitions) }; foreach (var (name, value) in logEvent.Properties) { if (name != SurrogateDefinitionsProperty) - EventJson.SetUserProperty(sample, name, SerilogEventJson.ToJsonNode(value)); + EventJson.SetUserProperty(sample, name, SimulationEvent.ToJsonNode(value)); } return true; diff --git a/src/SeqCli/Sample/Ingestion/SerilogTracingConventions.cs b/src/SeqCli/Sample/Ingestion/SerilogTracingConventions.cs deleted file mode 100644 index f08bc184..00000000 --- a/src/SeqCli/Sample/Ingestion/SerilogTracingConventions.cs +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright © Datalust and contributors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -using System.Text.Json.Nodes; - -namespace SeqCli.Sample.Ingestion; - -static class SerilogTracingConventions -{ - internal const string ParentSpanIdProperty = "ParentSpanId"; - - internal const string SpanStartTimestampProperty = "SpanStartTimestamp"; - - public static void LiftSpanProperties(JsonObject eventJson) - { - LiftProperty(eventJson, SpanStartTimestampProperty, "@st"); - LiftProperty(eventJson, ParentSpanIdProperty, "@ps"); - } - - static void LiftProperty(JsonObject eventJson, string propertyName, string reifiedName) - { - if (eventJson.TryGetPropertyValue(propertyName, out var value)) - { - eventJson.Remove(propertyName); - if (!eventJson.ContainsKey(reifiedName)) - eventJson[reifiedName] = value; - } - } -} diff --git a/src/SeqCli/Sample/Ingestion/SerilogEventJson.cs b/src/SeqCli/Sample/Ingestion/SimulationEvent.cs similarity index 74% rename from src/SeqCli/Sample/Ingestion/SerilogEventJson.cs rename to src/SeqCli/Sample/Ingestion/SimulationEvent.cs index 4750e756..bc48e5cc 100644 --- a/src/SeqCli/Sample/Ingestion/SerilogEventJson.cs +++ b/src/SeqCli/Sample/Ingestion/SimulationEvent.cs @@ -20,15 +20,13 @@ namespace SeqCli.Sample.Ingestion; -/// -/// Converts Serilog events produced within seqcli itself — the sample ingest simulation -/// and the forwarder's diagnostic ingestion log — into event JSON documents in Seq's emission -/// schema. Externally-supplied event data never passes through here: it's read directly into -/// JSON documents. -/// -static class SerilogEventJson +/// Used only in the Roastery simulation; no other event data should ever be processed using this type. +static class SimulationEvent { - public static JsonObject ToEventJson(LogEvent logEvent) + const string ParentSpanIdProperty = "ParentSpanId", + SpanStartTimestampProperty = "SpanStartTimestamp"; + + public static JsonObject ToJsonObject(LogEvent logEvent) { var eventJson = new JsonObject { @@ -51,7 +49,7 @@ public static JsonObject ToEventJson(LogEvent logEvent) foreach (var (name, value) in logEvent.Properties) EventJson.SetUserProperty(eventJson, name, ToJsonNode(value)); - SerilogTracingConventions.LiftSpanProperties(eventJson); + LiftSpanProperties(eventJson); return eventJson; } @@ -88,4 +86,20 @@ public static JsonObject ToEventJson(LogEvent logEvent) return EventJson.CreateScalar(value.ToString()); } } + + static void LiftSpanProperties(JsonObject eventJson) + { + LiftProperty(eventJson, SpanStartTimestampProperty, "@st"); + LiftProperty(eventJson, ParentSpanIdProperty, "@ps"); + } + + static void LiftProperty(JsonObject eventJson, string propertyName, string reifiedName) + { + if (eventJson.TryGetPropertyValue(propertyName, out var value)) + { + eventJson.Remove(propertyName); + if (!eventJson.ContainsKey(reifiedName)) + eventJson[reifiedName] = value; + } + } } diff --git a/test/SeqCli.Tests/Sample/SerilogEventJsonTests.cs b/test/SeqCli.Tests/Sample/SimulationEventTests.cs similarity index 90% rename from test/SeqCli.Tests/Sample/SerilogEventJsonTests.cs rename to test/SeqCli.Tests/Sample/SimulationEventTests.cs index 256d7a3f..0a71da14 100644 --- a/test/SeqCli.Tests/Sample/SerilogEventJsonTests.cs +++ b/test/SeqCli.Tests/Sample/SimulationEventTests.cs @@ -8,7 +8,7 @@ namespace SeqCli.Tests.Sample; -public class SerilogEventJsonTests +public class SimulationEventTests { static LogEvent CaptureEvent(Action log) { @@ -30,7 +30,7 @@ class CapturingSink(Action capture) : Serilog.Core.ILogEventSink public void EventFieldsMapToTheEmissionSchema() { var evt = CaptureEvent(log => log.Warning(new Exception("Boom!"), "Hello, {Name}!", "world")); - var eventJson = SerilogEventJson.ToEventJson(evt); + var eventJson = SimulationEvent.ToJsonObject(evt); Assert.Equal(evt.Timestamp.ToString("o"), (string?)eventJson["@t"]); Assert.Equal("Hello, {Name}!", (string?)eventJson["@mt"]); @@ -44,7 +44,7 @@ public void InformationLevelsAreOmitted() { var evt = CaptureEvent(log => log.Information("Hello")); - Assert.False(SerilogEventJson.ToEventJson(evt).ContainsKey("@l")); + Assert.False(SimulationEvent.ToJsonObject(evt).ContainsKey("@l")); } [Fact] @@ -52,7 +52,7 @@ public void StructuredValuesSerializeAsJson() { var evt = CaptureEvent(log => log.Information("{@Order} {Items}", new { Id = 7, Total = 4.5 }, new[] { "a", "b" })); - var eventJson = SerilogEventJson.ToEventJson(evt); + var eventJson = SimulationEvent.ToJsonObject(evt); Assert.Equal(7, (int?)eventJson["Order"]!["Id"]); Assert.Equal(4.5, (double?)eventJson["Order"]!["Total"]); @@ -67,7 +67,7 @@ public void SerilogTracingSpanPropertiesAreLifted() .ForContext("SpanStartTimestamp", start) .ForContext("ParentSpanId", "8899aabbccddeeff") .Information("GET /orders")); - var eventJson = SerilogEventJson.ToEventJson(evt); + var eventJson = SimulationEvent.ToJsonObject(evt); Assert.Equal(start.ToString("o"), (string?)eventJson["@st"]); Assert.Equal("8899aabbccddeeff", (string?)eventJson["@ps"]); @@ -103,6 +103,6 @@ public void PropertyNamesBeginningWithAtAreEscaped() { var evt = CaptureEvent(log => log.ForContext("@evil", "value").Information("Hello")); - Assert.Equal("value", (string?)SerilogEventJson.ToEventJson(evt)["@@evil"]); + Assert.Equal("value", (string?)SimulationEvent.ToJsonObject(evt)["@@evil"]); } } From 33b115d46b2735bda88308eda08dfea2a346e7ac Mon Sep 17 00:00:00 2001 From: Nicholas Blumhardt Date: Tue, 1 Sep 2026 16:30:52 +1000 Subject: [PATCH 04/15] More tidy-up --- src/SeqCli/Cli/Commands/TailCommand.cs | 8 ++++++-- src/SeqCli/Mapping/EventEntityJson.cs | 11 ++++++----- src/SeqCli/Mcp/Tools/Search/SearchTools.cs | 1 - 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/src/SeqCli/Cli/Commands/TailCommand.cs b/src/SeqCli/Cli/Commands/TailCommand.cs index 9d4d4957..291433ba 100644 --- a/src/SeqCli/Cli/Commands/TailCommand.cs +++ b/src/SeqCli/Cli/Commands/TailCommand.cs @@ -13,6 +13,8 @@ // limitations under the License. using System; +using System.IO; +using System.Text.Json.Nodes; using System.Threading; using System.Threading.Tasks; using SeqCli.Api; @@ -63,13 +65,15 @@ protected override async Task Run() try { - await foreach (var evt in connection.Events.StreamAsync( + await foreach (var evt in connection.Events.StreamDocumentsAsync( filter: strict, signal: _signal.Signal, render: true, + clef: true, cancellationToken: cancel.Token)) { - output.WriteEventEntity(evt); + var eventJson = JsonNode.Parse(evt)?.AsObject() ?? throw new InvalidDataException("Non-JSON document received."); + output.WriteEvent(eventJson); } } catch (OperationCanceledException) diff --git a/src/SeqCli/Mapping/EventEntityJson.cs b/src/SeqCli/Mapping/EventEntityJson.cs index e84bb3c9..68a5a485 100644 --- a/src/SeqCli/Mapping/EventEntityJson.cs +++ b/src/SeqCli/Mapping/EventEntityJson.cs @@ -19,22 +19,25 @@ using System.Text.Json.Nodes; using Seq.Api.Model.Events; using Seq.Api.Model.Shared; +using SeqCli.Output; using SeqCli.Syntax; using SeqCli.Util; namespace SeqCli.Mapping; /// -/// Converts events retrieved from the Seq API into event JSON documents in Seq's emission -/// (CLEF) schema, ready for filtering and formatting with Seq.Syntax. +/// Converts event entities into compact JSON format for further processing. This class is only necessary because +/// Seq.Api doesn't yet provide a simple compact-JSON based result format for searches. Once we've filled +/// that gap, this class, and can be removed. /// static class EventEntityJson { public static JsonObject ToEventJson(EventEntity evt) { - // Timestamps are shown in local time, matching earlier seqcli versions. var eventJson = new JsonObject { + // Earlier versions relied on Serilog output formatting to show timestamps in local time; we'll need + // to consider adding some compensating mechanism to `Seq.Syntax`. ["@t"] = DateTimeOffset.ParseExact(evt.Timestamp, "o", CultureInfo.InvariantCulture) .ToLocalTime().ToString("o", CultureInfo.InvariantCulture) }; @@ -42,8 +45,6 @@ public static JsonObject ToEventJson(EventEntity evt) if (evt.MessageTemplateTokens != null) eventJson["@mt"] = ToMessageTemplateText(evt.MessageTemplateTokens); - // By the emission convention, `Information` levels are omitted; any other level keeps - // the spelling it was ingested with. if (!string.IsNullOrWhiteSpace(evt.Level) && evt.Level != "Information") eventJson["@l"] = evt.Level; diff --git a/src/SeqCli/Mcp/Tools/Search/SearchTools.cs b/src/SeqCli/Mcp/Tools/Search/SearchTools.cs index 58a3b663..9ceda5e5 100644 --- a/src/SeqCli/Mcp/Tools/Search/SearchTools.cs +++ b/src/SeqCli/Mcp/Tools/Search/SearchTools.cs @@ -28,7 +28,6 @@ using Seq.Api.Model.Signals; using Seq.Syntax.Templates; using SeqCli.Mapping; -using SeqCli.Output; using SeqCli.Signals; using SeqCli.Syntax; using Serilog; From 8b828b33a78d4739f1ba8a44e9bbffebba70121c Mon Sep 17 00:00:00 2001 From: Nicholas Blumhardt Date: Tue, 1 Sep 2026 16:41:32 +1000 Subject: [PATCH 05/15] More cleanup --- src/SeqCli/Apps/Hosting/AppContainer.cs | 8 ++-- src/SeqCli/Apps/Hosting/EventFormat.cs | 2 +- .../Apps/Hosting/SerilogLevelMapping.cs | 41 ------------------- .../Cli/Commands/Alert/CreateCommand.cs | 2 +- .../Cli/Commands/ApiKey/CreateCommand.cs | 2 +- src/SeqCli/Mapping/LevelMapping.cs | 23 +++++++++-- src/SeqCli/PlainText/Extraction/Matchers.cs | 3 +- 7 files changed, 29 insertions(+), 52 deletions(-) delete mode 100644 src/SeqCli/Apps/Hosting/SerilogLevelMapping.cs diff --git a/src/SeqCli/Apps/Hosting/AppContainer.cs b/src/SeqCli/Apps/Hosting/AppContainer.cs index 2ba93fde..b65029f5 100644 --- a/src/SeqCli/Apps/Hosting/AppContainer.cs +++ b/src/SeqCli/Apps/Hosting/AppContainer.cs @@ -21,6 +21,7 @@ using Newtonsoft.Json.Linq; using Seq.Apps; using Seq.Apps.LogEvents; +using SeqCli.Mapping; using Serilog; using Serilog.Events; using Serilog.Formatting.Compact.Reader; @@ -108,11 +109,11 @@ async Task SendTypedEventAsync(string clef) { if (_seqApp is ISubscribeTo led) { - led.On(EventFormat.FromRaw(eventId, eventType, serilogEvent)); + led.On(EventFormat.FromSerilogLogEvent(eventId, eventType, serilogEvent)); } else if (_seqApp is ISubscribeToAsync leda) { - await leda.OnAsync(EventFormat.FromRaw(eventId, eventType, serilogEvent)); + await leda.OnAsync(EventFormat.FromSerilogLogEvent(eventId, eventType, serilogEvent)); } else if (_seqApp is ISubscribeTo sled) { @@ -142,7 +143,8 @@ LogEvent ReadSerilogEvent(string clef, out string eventId, out uint eventType) if (jobject.TryGetValue("@l", out var levelToken)) { jobject.Remove("@l"); - jobject.Add("@l", new JValue(SerilogLevelMapping.ToSerilogLevel(levelToken.Value()!).ToString())); + // The Seq.Api `LogEventLevel` enum intentionally matches the Serilog one. + jobject.Add("@l", new JValue(LevelMapping.ToSeqApiLogEventLevel(levelToken.Value()!).ToString())); } SanitizeTraceIdentifiers(jobject); diff --git a/src/SeqCli/Apps/Hosting/EventFormat.cs b/src/SeqCli/Apps/Hosting/EventFormat.cs index 1ff23253..67a37b4d 100644 --- a/src/SeqCli/Apps/Hosting/EventFormat.cs +++ b/src/SeqCli/Apps/Hosting/EventFormat.cs @@ -24,7 +24,7 @@ namespace SeqCli.Apps.Hosting; static class EventFormat { - public static Event FromRaw(string eventId, uint eventType, LogEvent raw) + public static Event FromSerilogLogEvent(string eventId, uint eventType, LogEvent raw) { var properties = new Dictionary(); foreach (var prop in raw.Properties) diff --git a/src/SeqCli/Apps/Hosting/SerilogLevelMapping.cs b/src/SeqCli/Apps/Hosting/SerilogLevelMapping.cs deleted file mode 100644 index f37a9f92..00000000 --- a/src/SeqCli/Apps/Hosting/SerilogLevelMapping.cs +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright © Datalust and contributors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -using SeqCli.Mapping; -using Serilog.Events; - -namespace SeqCli.Apps.Hosting; - -/// -/// Maps level names onto Serilog's level enum for hosted Seq apps relying on the older Serilog `LogEvent`-based -/// interface (newer apps should generally use raw JSON directly). -/// -static class SerilogLevelMapping -{ - public static LogEventLevel ToSerilogLevel(string level) - { - if (string.IsNullOrEmpty(level)) - return LogEventLevel.Information; - - return LevelMapping.ToFullLevelName(level) switch - { - "Trace" or "Verbose" => LogEventLevel.Verbose, - "Debug" => LogEventLevel.Debug, - "Warning" => LogEventLevel.Warning, - "Error" => LogEventLevel.Error, - "Fatal" or "Critical" or "Emergency" or "Alert" or "Panic" => LogEventLevel.Fatal, - _ => LogEventLevel.Information - }; - } -} diff --git a/src/SeqCli/Cli/Commands/Alert/CreateCommand.cs b/src/SeqCli/Cli/Commands/Alert/CreateCommand.cs index 49ceba1d..e7de0520 100644 --- a/src/SeqCli/Cli/Commands/Alert/CreateCommand.cs +++ b/src/SeqCli/Cli/Commands/Alert/CreateCommand.cs @@ -178,7 +178,7 @@ protected override async Task Run() alert.Having = _having; if (_notificationLevel != null) - alert.NotificationLevel = Enum.Parse(LevelMapping.ToFullLevelName(_notificationLevel)); + alert.NotificationLevel = LevelMapping.ToSeqApiLogEventLevel(_notificationLevel); if (_suppressionTime != null) alert.SuppressionTime = DurationMoniker.ToTimeSpan(_suppressionTime); diff --git a/src/SeqCli/Cli/Commands/ApiKey/CreateCommand.cs b/src/SeqCli/Cli/Commands/ApiKey/CreateCommand.cs index fe514434..03c3cf25 100644 --- a/src/SeqCli/Cli/Commands/ApiKey/CreateCommand.cs +++ b/src/SeqCli/Cli/Commands/ApiKey/CreateCommand.cs @@ -125,7 +125,7 @@ protected override async Task Run() if (_level != null) { - apiKey.InputSettings.MinimumLevel = Enum.Parse(LevelMapping.ToFullLevelName(_level)); + apiKey.InputSettings.MinimumLevel = LevelMapping.ToSeqApiLogEventLevel(_level); } apiKey.AssignedPermissions.Clear(); diff --git a/src/SeqCli/Mapping/LevelMapping.cs b/src/SeqCli/Mapping/LevelMapping.cs index 7faa47b4..2d866366 100644 --- a/src/SeqCli/Mapping/LevelMapping.cs +++ b/src/SeqCli/Mapping/LevelMapping.cs @@ -14,14 +14,12 @@ using System; using System.Collections.Generic; +using Seq.Api.Model.LogEvents; namespace SeqCli.Mapping; /// -/// Recognizes the level spellings found in event data from various sources (info, -/// WARN, trce, …) and maps them to canonical Seq level names. Level values -/// themselves are preserved verbatim throughout the pipeline; the canonical name is used -/// where a normalized form is needed. +/// Some Seq API /// public static class LevelMapping { @@ -80,8 +78,25 @@ public static class LevelMapping ["panic"] = "Panic" }; + // Intended only for use by ingest extraction patterns. public static string ToFullLevelName(string level) { return LevelsByName.TryGetValue(level, out var m) ? m : level; } + + public static LogEventLevel ToSeqApiLogEventLevel(string level) + { + if (string.IsNullOrEmpty(level)) + return LogEventLevel.Information; + + return ToFullLevelName(level) switch + { + "Trace" or "Verbose" => LogEventLevel.Verbose, + "Debug" => LogEventLevel.Debug, + "Warning" => LogEventLevel.Warning, + "Error" => LogEventLevel.Error, + "Fatal" or "Critical" or "Emergency" or "Alert" or "Panic" => LogEventLevel.Fatal, + _ => LogEventLevel.Information + }; + } } diff --git a/src/SeqCli/PlainText/Extraction/Matchers.cs b/src/SeqCli/PlainText/Extraction/Matchers.cs index c2326f01..5ae49e6e 100644 --- a/src/SeqCli/PlainText/Extraction/Matchers.cs +++ b/src/SeqCli/PlainText/Extraction/Matchers.cs @@ -8,6 +8,7 @@ using Superpower; using Superpower.Model; using Superpower.Parsers; +// ReSharper disable MemberCanBePrivate.Global namespace SeqCli.PlainText.Extraction; @@ -122,7 +123,7 @@ static class Matchers // Equivalent to :* at end-of-pattern public static TextParser MultiLineContent { get; } = - Span.WithAll(ch => true) + Span.WithAll(_ => true) .Select(span => (object?)span); [Matcher("n")] From 7ebd2ad63a324eb979b040de772964ac579b7d4d Mon Sep 17 00:00:00 2001 From: Nicholas Blumhardt Date: Tue, 1 Sep 2026 16:48:19 +1000 Subject: [PATCH 06/15] More clean-up --- .../JsonNodes.cs => Api/ToSystemTextJson.cs} | 24 +++++++++---------- src/SeqCli/Mapping/EventEntityJson.cs | 5 ++-- src/SeqCli/Output/OutputFormat.cs | 5 ++-- src/SeqCli/Output/TraceFormatter.cs | 3 ++- src/SeqCli/Traces/StructuredMessage.cs | 3 ++- 5 files changed, 22 insertions(+), 18 deletions(-) rename src/SeqCli/{Util/JsonNodes.cs => Api/ToSystemTextJson.cs} (83%) diff --git a/src/SeqCli/Util/JsonNodes.cs b/src/SeqCli/Api/ToSystemTextJson.cs similarity index 83% rename from src/SeqCli/Util/JsonNodes.cs rename to src/SeqCli/Api/ToSystemTextJson.cs index 7129be90..56d92eda 100644 --- a/src/SeqCli/Util/JsonNodes.cs +++ b/src/SeqCli/Api/ToSystemTextJson.cs @@ -17,21 +17,12 @@ using Newtonsoft.Json.Linq; using SeqCli.Syntax; -namespace SeqCli.Util; +namespace SeqCli.Api; -static class JsonNodes +static class ToSystemTextJson { - public static JsonNode? FromNewtonsoft(JToken token) - { - if (token is JValue { Value: null }) - return null; - - return JsonNode.Parse(token.ToString(Formatting.None)); - } - /// - /// Convert a value deserialized by the Seq API client — a Newtonsoft LINQ-to-JSON token, or - /// a plain CLR scalar — into its System.Text.Json equivalent. + /// Convert a value deserialized by the Seq API client into its `System.Text.Json` equivalent. /// public static JsonNode? FromApiValue(object? value) { @@ -42,4 +33,13 @@ static class JsonNodes _ => EventJson.CreateScalar(value) }; } + + /// Conversion helper for values retrieved through the Seq API client. + public static JsonNode? FromNewtonsoft(JToken token) + { + if (token is JValue { Value: null }) + return null; + + return JsonNode.Parse(token.ToString(Formatting.None)); + } } diff --git a/src/SeqCli/Mapping/EventEntityJson.cs b/src/SeqCli/Mapping/EventEntityJson.cs index 68a5a485..05433965 100644 --- a/src/SeqCli/Mapping/EventEntityJson.cs +++ b/src/SeqCli/Mapping/EventEntityJson.cs @@ -19,6 +19,7 @@ using System.Text.Json.Nodes; using Seq.Api.Model.Events; using Seq.Api.Model.Shared; +using SeqCli.Api; using SeqCli.Output; using SeqCli.Syntax; using SeqCli.Util; @@ -75,7 +76,7 @@ public static JsonObject ToEventJson(EventEntity evt) if (evt.Properties != null) { foreach (var property in evt.Properties) - EventJson.SetUserProperty(eventJson, property.Name, JsonNodes.FromApiValue(property.Value)); + EventJson.SetUserProperty(eventJson, property.Name, ToSystemTextJson.FromApiValue(property.Value)); } return eventJson; @@ -99,7 +100,7 @@ static JsonObject ToPropertiesObject(List properties) { var result = new JsonObject(); foreach (var property in properties) - result[property.Name] = JsonNodes.FromApiValue(property.Value); + result[property.Name] = ToSystemTextJson.FromApiValue(property.Value); return result; } } diff --git a/src/SeqCli/Output/OutputFormat.cs b/src/SeqCli/Output/OutputFormat.cs index af137838..c57bd175 100644 --- a/src/SeqCli/Output/OutputFormat.cs +++ b/src/SeqCli/Output/OutputFormat.cs @@ -26,6 +26,7 @@ using Seq.Syntax.Templates; using Seq.Syntax.Templates.Encoding; using Seq.Syntax.Templates.Themes; +using SeqCli.Api; using SeqCli.Config; using SeqCli.Csv; using SeqCli.Mapping; @@ -150,7 +151,7 @@ public void WriteEntity(Entity entity) if (Json) { jo.Remove("Links"); - WriteJsonValue(JsonNodes.FromNewtonsoft(jo)); + WriteJsonValue(ToSystemTextJson.FromNewtonsoft(jo)); } else if (Text) { @@ -173,7 +174,7 @@ public void WriteObject(object value) (JToken)JArray.FromObject(value, _serializer) : JObject.FromObject(value, _serializer); - WriteJsonValue(JsonNodes.FromNewtonsoft(jo)); + WriteJsonValue(ToSystemTextJson.FromNewtonsoft(jo)); } else if (Text) { diff --git a/src/SeqCli/Output/TraceFormatter.cs b/src/SeqCli/Output/TraceFormatter.cs index ab8238b2..9f11263c 100644 --- a/src/SeqCli/Output/TraceFormatter.cs +++ b/src/SeqCli/Output/TraceFormatter.cs @@ -17,6 +17,7 @@ using System.Globalization; using System.Text; using System.Text.Json.Nodes; +using SeqCli.Api; using SeqCli.Traces; using SeqCli.Util; @@ -105,7 +106,7 @@ static JsonObject ToEventJson(TraceTreeNode treeNode, string treePrefix) for (var i = 0; i < evt.Columns.Count; ++i) { if (evt.Columns[i] is { } value) - eventJson[ColumnPropertyName(i)] = JsonNodes.FromApiValue(value); + eventJson[ColumnPropertyName(i)] = ToSystemTextJson.FromApiValue(value); } return eventJson; diff --git a/src/SeqCli/Traces/StructuredMessage.cs b/src/SeqCli/Traces/StructuredMessage.cs index 38cae501..a4a5a281 100644 --- a/src/SeqCli/Traces/StructuredMessage.cs +++ b/src/SeqCli/Traces/StructuredMessage.cs @@ -17,6 +17,7 @@ using System.Linq; using System.Text.Json.Nodes; using Newtonsoft.Json.Linq; +using SeqCli.Api; using SeqCli.Util; namespace SeqCli.Traces; @@ -51,7 +52,7 @@ public static (string MessageTemplate, JsonObject Properties) Read(object? struc templateTokens.Add((false, (hole["raw"] as JValue)?.Value as string ?? $"{{{name}}}")); if (hole.TryGetValue("value", out var value) && propertyNames.Add(name)) - SetPathProperty(properties, name, JsonNodes.FromNewtonsoft(value)); + SetPathProperty(properties, name, ToSystemTextJson.FromNewtonsoft(value)); } else if (token is JValue { Type: JTokenType.String } text) { From 5aadf5788e00c5fcca6a1cc5db71212b885f2fcf Mon Sep 17 00:00:00 2001 From: Nicholas Blumhardt Date: Tue, 1 Sep 2026 16:56:21 +1000 Subject: [PATCH 07/15] No need to maintain compat with undocumented internal functions we previously used for trace formatting --- src/SeqCli/Syntax/SeqSyntax.cs | 7 ++- src/SeqCli/Syntax/V1/TracingFunctions.cs | 58 ------------------------ 2 files changed, 3 insertions(+), 62 deletions(-) delete mode 100644 src/SeqCli/Syntax/V1/TracingFunctions.cs diff --git a/src/SeqCli/Syntax/SeqSyntax.cs b/src/SeqCli/Syntax/SeqSyntax.cs index acbeab8c..4d05d078 100644 --- a/src/SeqCli/Syntax/SeqSyntax.cs +++ b/src/SeqCli/Syntax/SeqSyntax.cs @@ -17,8 +17,7 @@ using Seq.Syntax.Expressions; using Seq.Syntax.Templates; using Seq.Syntax.Templates.Encoding; -using SeqCli.Syntax.V1; -using V1Compatibility = Seq.Syntax.Compatibility.V1; +using Seq.Syntax.Compatibility; namespace SeqCli.Syntax; @@ -42,12 +41,12 @@ public static bool TryCompileExpression( [MaybeNullWhen(false)] out CompiledExpression result, [MaybeNullWhen(true)] out string error) { - return V1Compatibility.TryCompileExpression(expression, formatProvider: null, TracingFunctions.Resolver, out result, out error); + return V1.TryCompileExpression(expression, formatProvider: null, null, out result, out error); } public static ExpressionTemplate ParseTemplate(string template, TemplateOutputEncoder? encoder = null) { - if (!V1Compatibility.TryParseTemplate(template, culture: null, TracingFunctions.Resolver, encoder, out var parsed, out var error)) + if (!V1.TryParseTemplate(template, culture: null, null, encoder, out var parsed, out var error)) throw new ArgumentException(error); return parsed; diff --git a/src/SeqCli/Syntax/V1/TracingFunctions.cs b/src/SeqCli/Syntax/V1/TracingFunctions.cs deleted file mode 100644 index 59193168..00000000 --- a/src/SeqCli/Syntax/V1/TracingFunctions.cs +++ /dev/null @@ -1,58 +0,0 @@ -// Copyright © Datalust Pty Ltd -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -using System; -using System.Globalization; -using System.Text.Json.Nodes; -using Seq.Syntax.Expressions; - -namespace SeqCli.Syntax.V1; - -/// -/// Functions carried over from earlier seqcli versions, where Seq.Syntax had no tracing -/// support of its own. Elapsed() and Milliseconds() remain only so that existing -/// user-supplied expressions and output templates keep working; the built-in @Elapsed -/// and TotalMilliseconds() replace them. -/// -static class TracingFunctions -{ - public static readonly NameResolver Resolver = new StaticMemberNameResolver(typeof(TracingFunctions)); - - public static EvaluationResult Elapsed(JsonObject eventJson) - { - if (GetTimestampField(eventJson, "@t") is { } timestamp && - GetTimestampField(eventJson, "@st") is { } start) - { - return JsonValue.Create(timestamp - start)!; - } - - return EvaluationResult.Undefined; - } - - public static EvaluationResult Milliseconds(TimeSpan timeSpan) - { - // Truncates instead of rounding. - return JsonValue.Create(timeSpan.Ticks / (decimal)TimeSpan.TicksPerMillisecond); - } - - static DateTimeOffset? GetTimestampField(JsonObject eventJson, string field) - { - return eventJson.TryGetPropertyValue(field, out var node) && - node is JsonValue value && - value.TryGetValue(out string? text) && - DateTimeOffset.TryParse(text, CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind, out var dto) - ? dto - : null; - } -} From 8d9fad05844dc6df3d6f8eaae360785471a22ec1 Mon Sep 17 00:00:00 2001 From: Nicholas Blumhardt Date: Tue, 1 Sep 2026 17:05:10 +1000 Subject: [PATCH 08/15] Separate Data/ from Syntax/ - still not particularly cohesive, but should point us in the right direction --- src/SeqCli/Api/ToSystemTextJson.cs | 3 ++- src/SeqCli/Cli/Commands/IngestCommand.cs | 1 + .../{Syntax/EventJson.cs => Data/EventJsonDocument.cs} | 9 ++------- src/SeqCli/{Syntax => Data}/IEventEnricher.cs | 2 +- src/SeqCli/{Syntax => Data}/ScalarPropertyEnricher.cs | 6 +++--- src/SeqCli/Ingestion/EnrichingReader.cs | 1 + src/SeqCli/Mapping/EventEntityJson.cs | 3 ++- src/SeqCli/PlainText/LogEvents/EventJsonBuilder.cs | 7 ++++--- src/SeqCli/Sample/Ingestion/MetricsMapping.cs | 3 ++- src/SeqCli/Sample/Ingestion/SimulationEvent.cs | 7 ++++--- src/SeqCli/Syntax/LevelEnricher.cs | 1 + 11 files changed, 23 insertions(+), 20 deletions(-) rename src/SeqCli/{Syntax/EventJson.cs => Data/EventJsonDocument.cs} (87%) rename src/SeqCli/{Syntax => Data}/IEventEnricher.cs (97%) rename src/SeqCli/{Syntax => Data}/ScalarPropertyEnricher.cs (85%) diff --git a/src/SeqCli/Api/ToSystemTextJson.cs b/src/SeqCli/Api/ToSystemTextJson.cs index 56d92eda..35716f03 100644 --- a/src/SeqCli/Api/ToSystemTextJson.cs +++ b/src/SeqCli/Api/ToSystemTextJson.cs @@ -15,6 +15,7 @@ using System.Text.Json.Nodes; using Newtonsoft.Json; using Newtonsoft.Json.Linq; +using SeqCli.Data; using SeqCli.Syntax; namespace SeqCli.Api; @@ -30,7 +31,7 @@ static class ToSystemTextJson { null => null, JToken token => FromNewtonsoft(token), - _ => EventJson.CreateScalar(value) + _ => EventJsonDocument.CreateScalar(value) }; } diff --git a/src/SeqCli/Cli/Commands/IngestCommand.cs b/src/SeqCli/Cli/Commands/IngestCommand.cs index ba81fa0a..e0ad35a6 100644 --- a/src/SeqCli/Cli/Commands/IngestCommand.cs +++ b/src/SeqCli/Cli/Commands/IngestCommand.cs @@ -20,6 +20,7 @@ using SeqCli.Api; using SeqCli.Cli.Features; using SeqCli.Config; +using SeqCli.Data; using SeqCli.Ingestion; using SeqCli.PlainText; using SeqCli.Syntax; diff --git a/src/SeqCli/Syntax/EventJson.cs b/src/SeqCli/Data/EventJsonDocument.cs similarity index 87% rename from src/SeqCli/Syntax/EventJson.cs rename to src/SeqCli/Data/EventJsonDocument.cs index 282a3cbf..d4ac9c08 100644 --- a/src/SeqCli/Syntax/EventJson.cs +++ b/src/SeqCli/Data/EventJsonDocument.cs @@ -16,14 +16,9 @@ using System.Globalization; using System.Text.Json.Nodes; -namespace SeqCli.Syntax; +namespace SeqCli.Data; -/// -/// Helpers for constructing event JSON documents in Seq's emission (CLEF) schema, where -/// reified fields carry @-prefixed names and user-defined property names beginning -/// with @ are escaped with a second @. -/// -static class EventJson +static class EventJsonDocument { const string InvalidPropertyNameSubstitute = "(unnamed)"; diff --git a/src/SeqCli/Syntax/IEventEnricher.cs b/src/SeqCli/Data/IEventEnricher.cs similarity index 97% rename from src/SeqCli/Syntax/IEventEnricher.cs rename to src/SeqCli/Data/IEventEnricher.cs index 9d10d8e7..55c8584b 100644 --- a/src/SeqCli/Syntax/IEventEnricher.cs +++ b/src/SeqCli/Data/IEventEnricher.cs @@ -14,7 +14,7 @@ using System.Text.Json.Nodes; -namespace SeqCli.Syntax; +namespace SeqCli.Data; /// /// Adds or updates fields on an event JSON document; the equivalent, in Seq's data model, of a diff --git a/src/SeqCli/Syntax/ScalarPropertyEnricher.cs b/src/SeqCli/Data/ScalarPropertyEnricher.cs similarity index 85% rename from src/SeqCli/Syntax/ScalarPropertyEnricher.cs rename to src/SeqCli/Data/ScalarPropertyEnricher.cs index 95490a3b..6e9193d3 100644 --- a/src/SeqCli/Syntax/ScalarPropertyEnricher.cs +++ b/src/SeqCli/Data/ScalarPropertyEnricher.cs @@ -14,7 +14,7 @@ using System.Text.Json.Nodes; -namespace SeqCli.Syntax; +namespace SeqCli.Data; class ScalarPropertyEnricher : IEventEnricher { @@ -23,12 +23,12 @@ class ScalarPropertyEnricher : IEventEnricher public ScalarPropertyEnricher(string name, object? scalarValue) { - _name = EventJson.EscapeUserPropertyName(name); + _name = EventJsonDocument.EscapeUserPropertyName(name); _scalarValue = scalarValue; } public void Enrich(JsonObject eventJson) { - eventJson[_name] = EventJson.CreateScalar(_scalarValue); + eventJson[_name] = EventJsonDocument.CreateScalar(_scalarValue); } } diff --git a/src/SeqCli/Ingestion/EnrichingReader.cs b/src/SeqCli/Ingestion/EnrichingReader.cs index 63a25ca3..dcbbebae 100644 --- a/src/SeqCli/Ingestion/EnrichingReader.cs +++ b/src/SeqCli/Ingestion/EnrichingReader.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Threading.Tasks; +using SeqCli.Data; using SeqCli.Syntax; namespace SeqCli.Ingestion; diff --git a/src/SeqCli/Mapping/EventEntityJson.cs b/src/SeqCli/Mapping/EventEntityJson.cs index 05433965..093046dc 100644 --- a/src/SeqCli/Mapping/EventEntityJson.cs +++ b/src/SeqCli/Mapping/EventEntityJson.cs @@ -20,6 +20,7 @@ using Seq.Api.Model.Events; using Seq.Api.Model.Shared; using SeqCli.Api; +using SeqCli.Data; using SeqCli.Output; using SeqCli.Syntax; using SeqCli.Util; @@ -76,7 +77,7 @@ public static JsonObject ToEventJson(EventEntity evt) if (evt.Properties != null) { foreach (var property in evt.Properties) - EventJson.SetUserProperty(eventJson, property.Name, ToSystemTextJson.FromApiValue(property.Value)); + EventJsonDocument.SetUserProperty(eventJson, property.Name, ToSystemTextJson.FromApiValue(property.Value)); } return eventJson; diff --git a/src/SeqCli/PlainText/LogEvents/EventJsonBuilder.cs b/src/SeqCli/PlainText/LogEvents/EventJsonBuilder.cs index caf4476e..caf55b5e 100644 --- a/src/SeqCli/PlainText/LogEvents/EventJsonBuilder.cs +++ b/src/SeqCli/PlainText/LogEvents/EventJsonBuilder.cs @@ -16,6 +16,7 @@ using System.Collections.Generic; using System.Globalization; using System.Text.Json.Nodes; +using SeqCli.Data; using SeqCli.Syntax; using Superpower.Model; @@ -55,11 +56,11 @@ public static JsonObject FromProperties(IDictionary properties, foreach (var (name, value) in properties) { if (!ReifiedProperties.IsReifiedProperty(name)) - EventJson.SetUserProperty(eventJson, name, CreateValue(value)); + EventJsonDocument.SetUserProperty(eventJson, name, CreateValue(value)); } if (remainder != null) - EventJson.SetUserProperty(eventJson, "@unmatched", remainder); + EventJsonDocument.SetUserProperty(eventJson, "@unmatched", remainder); return eventJson; } @@ -68,7 +69,7 @@ public static JsonObject FromProperties(IDictionary properties, { return value is TextSpan span ? JsonValue.Create(span.ToStringValue()) - : EventJson.CreateScalar(value); + : EventJsonDocument.CreateScalar(value); } static bool TryGetText(IDictionary properties, string name, out string text) diff --git a/src/SeqCli/Sample/Ingestion/MetricsMapping.cs b/src/SeqCli/Sample/Ingestion/MetricsMapping.cs index 12fbb91f..0271b2f2 100644 --- a/src/SeqCli/Sample/Ingestion/MetricsMapping.cs +++ b/src/SeqCli/Sample/Ingestion/MetricsMapping.cs @@ -16,6 +16,7 @@ using System.Diagnostics.CodeAnalysis; using System.Globalization; using System.Text.Json.Nodes; +using SeqCli.Data; using SeqCli.Ingestion; using SeqCli.Syntax; using Serilog.Events; @@ -51,7 +52,7 @@ public static bool TryGetMetricSampleJson(LogEvent logEvent, [NotNullWhen(true)] foreach (var (name, value) in logEvent.Properties) { if (name != SurrogateDefinitionsProperty) - EventJson.SetUserProperty(sample, name, SimulationEvent.ToJsonNode(value)); + EventJsonDocument.SetUserProperty(sample, name, SimulationEvent.ToJsonNode(value)); } return true; diff --git a/src/SeqCli/Sample/Ingestion/SimulationEvent.cs b/src/SeqCli/Sample/Ingestion/SimulationEvent.cs index bc48e5cc..b6a3656e 100644 --- a/src/SeqCli/Sample/Ingestion/SimulationEvent.cs +++ b/src/SeqCli/Sample/Ingestion/SimulationEvent.cs @@ -15,6 +15,7 @@ using System.Globalization; using System.Linq; using System.Text.Json.Nodes; +using SeqCli.Data; using SeqCli.Syntax; using Serilog.Events; @@ -47,7 +48,7 @@ public static JsonObject ToJsonObject(LogEvent logEvent) eventJson["@sp"] = spanId.ToHexString(); foreach (var (name, value) in logEvent.Properties) - EventJson.SetUserProperty(eventJson, name, ToJsonNode(value)); + EventJsonDocument.SetUserProperty(eventJson, name, ToJsonNode(value)); LiftSpanProperties(eventJson); @@ -59,7 +60,7 @@ public static JsonObject ToJsonObject(LogEvent logEvent) switch (value) { case ScalarValue scalar: - return EventJson.CreateScalar(scalar.Value); + return EventJsonDocument.CreateScalar(scalar.Value); case SequenceValue sequence: return new JsonArray(sequence.Elements.Select(ToJsonNode).ToArray()); @@ -83,7 +84,7 @@ public static JsonObject ToJsonObject(LogEvent logEvent) } default: - return EventJson.CreateScalar(value.ToString()); + return EventJsonDocument.CreateScalar(value.ToString()); } } diff --git a/src/SeqCli/Syntax/LevelEnricher.cs b/src/SeqCli/Syntax/LevelEnricher.cs index c09fdb35..8a1606bb 100644 --- a/src/SeqCli/Syntax/LevelEnricher.cs +++ b/src/SeqCli/Syntax/LevelEnricher.cs @@ -13,6 +13,7 @@ // limitations under the License. using System.Text.Json.Nodes; +using SeqCli.Data; namespace SeqCli.Syntax; From 45551a20d3cf595c15be31d362a4b0e565552f0a Mon Sep 17 00:00:00 2001 From: Nicholas Blumhardt Date: Tue, 1 Sep 2026 17:27:58 +1000 Subject: [PATCH 09/15] Drop some brittle test scaffolding --- test/SeqCli.Tests/Output/OutputFormatTests.cs | 43 ------------------- 1 file changed, 43 deletions(-) diff --git a/test/SeqCli.Tests/Output/OutputFormatTests.cs b/test/SeqCli.Tests/Output/OutputFormatTests.cs index cb1cf55b..a6777bef 100644 --- a/test/SeqCli.Tests/Output/OutputFormatTests.cs +++ b/test/SeqCli.Tests/Output/OutputFormatTests.cs @@ -151,47 +151,4 @@ public void UnresolvableDottedHolesRenderAsRawText() Assert.Equal("{user.greeting.first} {user.name}!", RenderMessage(evt)); } - - static string CaptureConsoleOut(System.Action write) - { - var output = new StringWriter(); - var saved = System.Console.Out; - System.Console.SetOut(output); - try - { - write(); - } - finally - { - System.Console.SetOut(saved); - } - - return output.ToString(); - } - - [Fact] - public void ObjectsAreWrittenAsSingleLineJson() - { - var format = Create(syntax: OutputSyntax.Json); - - var written = CaptureConsoleOut(() => format.WriteObject( - new JObject(new JProperty("Title", "Errors"), new JProperty("Count", 42)))); - - Assert.Equal("""{"Title":"Errors","Count":42}""" + System.Environment.NewLine, written); - } - - [Fact] - public void EntitiesAreWrittenAsJsonWithoutLinks() - { - var entity = new Seq.Api.Model.Signals.SignalEntity { Id = "signal-1", Title = "Errors" }; - - var format = Create(syntax: OutputSyntax.Json); - var written = CaptureConsoleOut(() => format.WriteEntity(entity)); - - Assert.Contains("\"Id\":\"signal-1\"", written); - Assert.Contains("\"Title\":\"Errors\"", written); - Assert.DoesNotContain("Links", written); - Assert.EndsWith(System.Environment.NewLine, written); - Assert.Equal(written.TrimEnd(), written.TrimEnd().ReplaceLineEndings("")); - } } From 575be722f383d0823d90003391fe0b5a6ae48c7c Mon Sep 17 00:00:00 2001 From: Nicholas Blumhardt Date: Wed, 2 Sep 2026 07:17:26 +1000 Subject: [PATCH 10/15] Fix namespace --- src/SeqCli/{Syntax => Data}/LevelEnricher.cs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) rename src/SeqCli/{Syntax => Data}/LevelEnricher.cs (95%) diff --git a/src/SeqCli/Syntax/LevelEnricher.cs b/src/SeqCli/Data/LevelEnricher.cs similarity index 95% rename from src/SeqCli/Syntax/LevelEnricher.cs rename to src/SeqCli/Data/LevelEnricher.cs index 8a1606bb..b50df51c 100644 --- a/src/SeqCli/Syntax/LevelEnricher.cs +++ b/src/SeqCli/Data/LevelEnricher.cs @@ -13,9 +13,8 @@ // limitations under the License. using System.Text.Json.Nodes; -using SeqCli.Data; -namespace SeqCli.Syntax; +namespace SeqCli.Data; /// /// Overrides the event's @l level with a fixed value. From 8d598c623029ed733cd71e915493990d8938c7ba Mon Sep 17 00:00:00 2001 From: Nicholas Blumhardt Date: Wed, 2 Sep 2026 07:19:09 +1000 Subject: [PATCH 11/15] Remove half comment --- src/SeqCli/Mapping/LevelMapping.cs | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/SeqCli/Mapping/LevelMapping.cs b/src/SeqCli/Mapping/LevelMapping.cs index 2d866366..33639c3a 100644 --- a/src/SeqCli/Mapping/LevelMapping.cs +++ b/src/SeqCli/Mapping/LevelMapping.cs @@ -18,9 +18,6 @@ namespace SeqCli.Mapping; -/// -/// Some Seq API -/// public static class LevelMapping { static readonly Dictionary LevelsByName = From 725b97710e7232747a3602a2810254c4970974a6 Mon Sep 17 00:00:00 2001 From: Nicholas Blumhardt Date: Wed, 2 Sep 2026 07:22:37 +1000 Subject: [PATCH 12/15] Namespaces, using --- src/Roastery/Data/Database.cs | 1 - src/Roastery/Util/Distribution.cs | 1 - src/Roastery/Web/RequestLoggingMiddleware.cs | 1 - src/SeqCli/Api/ToSystemTextJson.cs | 1 - src/SeqCli/Cli/Commands/Alert/CreateCommand.cs | 1 - src/SeqCli/Cli/Commands/ApiKey/CreateCommand.cs | 1 - src/SeqCli/Ingestion/EnrichingReader.cs | 1 - src/SeqCli/Mapping/EventEntityJson.cs | 2 -- src/SeqCli/Output/OutputFormat.cs | 1 - src/SeqCli/Output/TraceFormatter.cs | 1 - src/SeqCli/PlainText/{LogEvents => }/EventJsonBuilder.cs | 3 +-- src/SeqCli/PlainText/PlainTextEventReader.cs | 2 +- src/SeqCli/Sample/Ingestion/MetricsMapping.cs | 2 -- src/SeqCli/Sample/Ingestion/SimulationEvent.cs | 1 - src/SeqCli/Traces/StructuredMessage.cs | 1 - test/SeqCli.EndToEnd/Events/EventsDeleteTestCase.cs | 1 - .../Forwarder/ForwarderSimpleIngestionTestCase.cs | 1 - test/SeqCli.EndToEnd/Mcp/McpMetricsBasicsTestCase.cs | 1 - test/SeqCli.EndToEnd/Settings/SettingBasicsTestCase.cs | 3 +-- test/SeqCli.EndToEnd/Skills/SkillsInstallTestCase.cs | 1 - test/SeqCli.EndToEnd/User/UserCreateRemoveTestCase.cs | 1 - test/SeqCli.Tests/Forwarder/Storage/BufferTests.cs | 1 - test/SeqCli.Tests/PlainText/EventJsonBuilderTests.cs | 3 ++- .../PlainText/ExtractionPatternInterpreterTests.cs | 1 - test/SeqCli.Tests/PlainText/ExtractionPatternParserTests.cs | 3 +-- test/SeqCli.Tests/Syntax/AliasedExpressionParserTests.cs | 1 - 26 files changed, 6 insertions(+), 31 deletions(-) rename src/SeqCli/PlainText/{LogEvents => }/EventJsonBuilder.cs (98%) diff --git a/src/Roastery/Data/Database.cs b/src/Roastery/Data/Database.cs index 9234f1ce..251b67f1 100644 --- a/src/Roastery/Data/Database.cs +++ b/src/Roastery/Data/Database.cs @@ -1,6 +1,5 @@ using System; using System.Collections.Generic; -using System.Diagnostics; using System.Globalization; using System.Linq; using System.Reflection; diff --git a/src/Roastery/Util/Distribution.cs b/src/Roastery/Util/Distribution.cs index b20ffb49..5264cd14 100644 --- a/src/Roastery/Util/Distribution.cs +++ b/src/Roastery/Util/Distribution.cs @@ -1,7 +1,6 @@ using System; using System.Collections.Generic; using System.Runtime.CompilerServices; -using System.Threading; namespace Roastery.Util; diff --git a/src/Roastery/Web/RequestLoggingMiddleware.cs b/src/Roastery/Web/RequestLoggingMiddleware.cs index c75a5002..92d50bf3 100644 --- a/src/Roastery/Web/RequestLoggingMiddleware.cs +++ b/src/Roastery/Web/RequestLoggingMiddleware.cs @@ -1,6 +1,5 @@ using System; using System.Diagnostics; -using System.Diagnostics.Metrics; using System.Net; using System.Threading.Tasks; using Roastery.Metrics; diff --git a/src/SeqCli/Api/ToSystemTextJson.cs b/src/SeqCli/Api/ToSystemTextJson.cs index 35716f03..52773250 100644 --- a/src/SeqCli/Api/ToSystemTextJson.cs +++ b/src/SeqCli/Api/ToSystemTextJson.cs @@ -16,7 +16,6 @@ using Newtonsoft.Json; using Newtonsoft.Json.Linq; using SeqCli.Data; -using SeqCli.Syntax; namespace SeqCli.Api; diff --git a/src/SeqCli/Cli/Commands/Alert/CreateCommand.cs b/src/SeqCli/Cli/Commands/Alert/CreateCommand.cs index e7de0520..e74d833a 100644 --- a/src/SeqCli/Cli/Commands/Alert/CreateCommand.cs +++ b/src/SeqCli/Cli/Commands/Alert/CreateCommand.cs @@ -17,7 +17,6 @@ using System.Linq; using System.Threading.Tasks; using Seq.Api.Model.Alerting; -using Seq.Api.Model.LogEvents; using Seq.Api.Model.Shared; using SeqCli.Api; using SeqCli.Cli.Features; diff --git a/src/SeqCli/Cli/Commands/ApiKey/CreateCommand.cs b/src/SeqCli/Cli/Commands/ApiKey/CreateCommand.cs index 03c3cf25..cf928d8c 100644 --- a/src/SeqCli/Cli/Commands/ApiKey/CreateCommand.cs +++ b/src/SeqCli/Cli/Commands/ApiKey/CreateCommand.cs @@ -16,7 +16,6 @@ using System.Linq; using System.Threading.Tasks; using Seq.Api; -using Seq.Api.Model.LogEvents; using Seq.Api.Model.Security; using Seq.Api.Model.Shared; using SeqCli.Api; diff --git a/src/SeqCli/Ingestion/EnrichingReader.cs b/src/SeqCli/Ingestion/EnrichingReader.cs index dcbbebae..6207bf4c 100644 --- a/src/SeqCli/Ingestion/EnrichingReader.cs +++ b/src/SeqCli/Ingestion/EnrichingReader.cs @@ -2,7 +2,6 @@ using System.Collections.Generic; using System.Threading.Tasks; using SeqCli.Data; -using SeqCli.Syntax; namespace SeqCli.Ingestion; diff --git a/src/SeqCli/Mapping/EventEntityJson.cs b/src/SeqCli/Mapping/EventEntityJson.cs index 093046dc..169d7a49 100644 --- a/src/SeqCli/Mapping/EventEntityJson.cs +++ b/src/SeqCli/Mapping/EventEntityJson.cs @@ -22,8 +22,6 @@ using SeqCli.Api; using SeqCli.Data; using SeqCli.Output; -using SeqCli.Syntax; -using SeqCli.Util; namespace SeqCli.Mapping; diff --git a/src/SeqCli/Output/OutputFormat.cs b/src/SeqCli/Output/OutputFormat.cs index c57bd175..9efb3b97 100644 --- a/src/SeqCli/Output/OutputFormat.cs +++ b/src/SeqCli/Output/OutputFormat.cs @@ -30,7 +30,6 @@ using SeqCli.Config; using SeqCli.Csv; using SeqCli.Mapping; -using SeqCli.Util; namespace SeqCli.Output; diff --git a/src/SeqCli/Output/TraceFormatter.cs b/src/SeqCli/Output/TraceFormatter.cs index 9f11263c..c0a575ef 100644 --- a/src/SeqCli/Output/TraceFormatter.cs +++ b/src/SeqCli/Output/TraceFormatter.cs @@ -19,7 +19,6 @@ using System.Text.Json.Nodes; using SeqCli.Api; using SeqCli.Traces; -using SeqCli.Util; namespace SeqCli.Output; diff --git a/src/SeqCli/PlainText/LogEvents/EventJsonBuilder.cs b/src/SeqCli/PlainText/EventJsonBuilder.cs similarity index 98% rename from src/SeqCli/PlainText/LogEvents/EventJsonBuilder.cs rename to src/SeqCli/PlainText/EventJsonBuilder.cs index caf55b5e..bff33836 100644 --- a/src/SeqCli/PlainText/LogEvents/EventJsonBuilder.cs +++ b/src/SeqCli/PlainText/EventJsonBuilder.cs @@ -17,10 +17,9 @@ using System.Globalization; using System.Text.Json.Nodes; using SeqCli.Data; -using SeqCli.Syntax; using Superpower.Model; -namespace SeqCli.PlainText.LogEvents; +namespace SeqCli.PlainText; /// /// Assembles the values captured by a plain-text extraction pattern into an event JSON diff --git a/src/SeqCli/PlainText/PlainTextEventReader.cs b/src/SeqCli/PlainText/PlainTextEventReader.cs index fbead08b..21da4e89 100644 --- a/src/SeqCli/PlainText/PlainTextEventReader.cs +++ b/src/SeqCli/PlainText/PlainTextEventReader.cs @@ -1,10 +1,10 @@ using System; using System.IO; using System.Threading.Tasks; +using SeqCli.Data; using SeqCli.Ingestion; using SeqCli.PlainText.Extraction; using SeqCli.PlainText.Framing; -using SeqCli.PlainText.LogEvents; using SeqCli.PlainText.Parsers; using SeqCli.PlainText.Patterns; diff --git a/src/SeqCli/Sample/Ingestion/MetricsMapping.cs b/src/SeqCli/Sample/Ingestion/MetricsMapping.cs index 0271b2f2..6782e6c5 100644 --- a/src/SeqCli/Sample/Ingestion/MetricsMapping.cs +++ b/src/SeqCli/Sample/Ingestion/MetricsMapping.cs @@ -17,8 +17,6 @@ using System.Globalization; using System.Text.Json.Nodes; using SeqCli.Data; -using SeqCli.Ingestion; -using SeqCli.Syntax; using Serilog.Events; namespace SeqCli.Sample.Ingestion; diff --git a/src/SeqCli/Sample/Ingestion/SimulationEvent.cs b/src/SeqCli/Sample/Ingestion/SimulationEvent.cs index b6a3656e..89bc110c 100644 --- a/src/SeqCli/Sample/Ingestion/SimulationEvent.cs +++ b/src/SeqCli/Sample/Ingestion/SimulationEvent.cs @@ -16,7 +16,6 @@ using System.Linq; using System.Text.Json.Nodes; using SeqCli.Data; -using SeqCli.Syntax; using Serilog.Events; namespace SeqCli.Sample.Ingestion; diff --git a/src/SeqCli/Traces/StructuredMessage.cs b/src/SeqCli/Traces/StructuredMessage.cs index a4a5a281..702c9006 100644 --- a/src/SeqCli/Traces/StructuredMessage.cs +++ b/src/SeqCli/Traces/StructuredMessage.cs @@ -18,7 +18,6 @@ using System.Text.Json.Nodes; using Newtonsoft.Json.Linq; using SeqCli.Api; -using SeqCli.Util; namespace SeqCli.Traces; diff --git a/test/SeqCli.EndToEnd/Events/EventsDeleteTestCase.cs b/test/SeqCli.EndToEnd/Events/EventsDeleteTestCase.cs index 29b585c9..61d8281b 100644 --- a/test/SeqCli.EndToEnd/Events/EventsDeleteTestCase.cs +++ b/test/SeqCli.EndToEnd/Events/EventsDeleteTestCase.cs @@ -1,4 +1,3 @@ -using System; using System.IO; using System.Threading.Tasks; using Seq.Api; diff --git a/test/SeqCli.EndToEnd/Forwarder/ForwarderSimpleIngestionTestCase.cs b/test/SeqCli.EndToEnd/Forwarder/ForwarderSimpleIngestionTestCase.cs index bb199942..d90065e5 100644 --- a/test/SeqCli.EndToEnd/Forwarder/ForwarderSimpleIngestionTestCase.cs +++ b/test/SeqCli.EndToEnd/Forwarder/ForwarderSimpleIngestionTestCase.cs @@ -1,5 +1,4 @@ using System; -using System.Globalization; using System.Threading.Tasks; using Seq.Api; using SeqCli.EndToEnd.Support; diff --git a/test/SeqCli.EndToEnd/Mcp/McpMetricsBasicsTestCase.cs b/test/SeqCli.EndToEnd/Mcp/McpMetricsBasicsTestCase.cs index 77fc5f69..f1fa88fc 100644 --- a/test/SeqCli.EndToEnd/Mcp/McpMetricsBasicsTestCase.cs +++ b/test/SeqCli.EndToEnd/Mcp/McpMetricsBasicsTestCase.cs @@ -1,6 +1,5 @@ using System; using System.Collections.Generic; -using System.Text.Json.Nodes; using System.Threading.Tasks; using JetBrains.Annotations; using ModelContextProtocol.Client; diff --git a/test/SeqCli.EndToEnd/Settings/SettingBasicsTestCase.cs b/test/SeqCli.EndToEnd/Settings/SettingBasicsTestCase.cs index f387a400..1de07801 100644 --- a/test/SeqCli.EndToEnd/Settings/SettingBasicsTestCase.cs +++ b/test/SeqCli.EndToEnd/Settings/SettingBasicsTestCase.cs @@ -1,5 +1,4 @@ -using System; -using System.Threading.Tasks; +using System.Threading.Tasks; using Seq.Api; using SeqCli.EndToEnd.Support; using Serilog; diff --git a/test/SeqCli.EndToEnd/Skills/SkillsInstallTestCase.cs b/test/SeqCli.EndToEnd/Skills/SkillsInstallTestCase.cs index 88871bc1..5bca2a32 100644 --- a/test/SeqCli.EndToEnd/Skills/SkillsInstallTestCase.cs +++ b/test/SeqCli.EndToEnd/Skills/SkillsInstallTestCase.cs @@ -1,6 +1,5 @@ using System.IO; using System.Threading.Tasks; -using JetBrains.Annotations; using Seq.Api; using SeqCli.EndToEnd.Support; using Serilog; diff --git a/test/SeqCli.EndToEnd/User/UserCreateRemoveTestCase.cs b/test/SeqCli.EndToEnd/User/UserCreateRemoveTestCase.cs index ddbf95d2..25e3fb99 100644 --- a/test/SeqCli.EndToEnd/User/UserCreateRemoveTestCase.cs +++ b/test/SeqCli.EndToEnd/User/UserCreateRemoveTestCase.cs @@ -4,7 +4,6 @@ using SeqCli.EndToEnd.Support; using Serilog; using Xunit; -using System.IO; using System.Linq; namespace SeqCli.EndToEnd.User; diff --git a/test/SeqCli.Tests/Forwarder/Storage/BufferTests.cs b/test/SeqCli.Tests/Forwarder/Storage/BufferTests.cs index 60dee141..7e6bf24e 100644 --- a/test/SeqCli.Tests/Forwarder/Storage/BufferTests.cs +++ b/test/SeqCli.Tests/Forwarder/Storage/BufferTests.cs @@ -1,5 +1,4 @@ using System.Linq; -using SeqCli.Forwarder.Filesystem.System; using SeqCli.Forwarder.Storage; using SeqCli.Tests.Forwarder.Filesystem; using Xunit; diff --git a/test/SeqCli.Tests/PlainText/EventJsonBuilderTests.cs b/test/SeqCli.Tests/PlainText/EventJsonBuilderTests.cs index 0df85264..0e392eb9 100644 --- a/test/SeqCli.Tests/PlainText/EventJsonBuilderTests.cs +++ b/test/SeqCli.Tests/PlainText/EventJsonBuilderTests.cs @@ -2,7 +2,8 @@ using System; using System.Collections.Generic; using System.Globalization; -using SeqCli.PlainText.LogEvents; +using SeqCli.Data; +using SeqCli.PlainText; using Superpower.Model; using Xunit; diff --git a/test/SeqCli.Tests/PlainText/ExtractionPatternInterpreterTests.cs b/test/SeqCli.Tests/PlainText/ExtractionPatternInterpreterTests.cs index 0993251f..fa251cb9 100644 --- a/test/SeqCli.Tests/PlainText/ExtractionPatternInterpreterTests.cs +++ b/test/SeqCli.Tests/PlainText/ExtractionPatternInterpreterTests.cs @@ -1,7 +1,6 @@ using System; using System.Collections.Generic; using System.Globalization; -using SeqCli.PlainText; using SeqCli.PlainText.Extraction; using SeqCli.PlainText.Patterns; using Xunit; diff --git a/test/SeqCli.Tests/PlainText/ExtractionPatternParserTests.cs b/test/SeqCli.Tests/PlainText/ExtractionPatternParserTests.cs index 171d6aba..fac72167 100644 --- a/test/SeqCli.Tests/PlainText/ExtractionPatternParserTests.cs +++ b/test/SeqCli.Tests/PlainText/ExtractionPatternParserTests.cs @@ -1,5 +1,4 @@ -using System; -using System.Linq; +using System.Linq; using SeqCli.PlainText.Patterns; using Superpower; using Xunit; diff --git a/test/SeqCli.Tests/Syntax/AliasedExpressionParserTests.cs b/test/SeqCli.Tests/Syntax/AliasedExpressionParserTests.cs index 43cceeca..7699f513 100644 --- a/test/SeqCli.Tests/Syntax/AliasedExpressionParserTests.cs +++ b/test/SeqCli.Tests/Syntax/AliasedExpressionParserTests.cs @@ -1,4 +1,3 @@ -using System; using SeqCli.Syntax; using Xunit; From c230f57a21e6878127edb0f28895890c3f327619 Mon Sep 17 00:00:00 2001 From: Nicholas Blumhardt Date: Wed, 2 Sep 2026 11:09:30 +1000 Subject: [PATCH 13/15] More tidy-up --- src/SeqCli/Api/ToSystemTextJson.cs | 2 +- .../{EventJsonDocument.cs => EventJsonFormat.cs} | 14 ++------------ src/SeqCli/Data/ScalarPropertyEnricher.cs | 4 ++-- src/SeqCli/Mapping/EventEntityJson.cs | 2 +- src/SeqCli/PlainText/EventJsonBuilder.cs | 10 ++++++---- src/SeqCli/Sample/Ingestion/MetricsMapping.cs | 2 +- src/SeqCli/Sample/Ingestion/SimulationEvent.cs | 6 +++--- src/SeqCli/SeqCli.csproj | 2 +- 8 files changed, 17 insertions(+), 25 deletions(-) rename src/SeqCli/Data/{EventJsonDocument.cs => EventJsonFormat.cs} (83%) diff --git a/src/SeqCli/Api/ToSystemTextJson.cs b/src/SeqCli/Api/ToSystemTextJson.cs index 52773250..92428c06 100644 --- a/src/SeqCli/Api/ToSystemTextJson.cs +++ b/src/SeqCli/Api/ToSystemTextJson.cs @@ -30,7 +30,7 @@ static class ToSystemTextJson { null => null, JToken token => FromNewtonsoft(token), - _ => EventJsonDocument.CreateScalar(value) + _ => EventJsonFormat.CreateScalar(value) }; } diff --git a/src/SeqCli/Data/EventJsonDocument.cs b/src/SeqCli/Data/EventJsonFormat.cs similarity index 83% rename from src/SeqCli/Data/EventJsonDocument.cs rename to src/SeqCli/Data/EventJsonFormat.cs index d4ac9c08..1f0a83dd 100644 --- a/src/SeqCli/Data/EventJsonDocument.cs +++ b/src/SeqCli/Data/EventJsonFormat.cs @@ -18,23 +18,13 @@ namespace SeqCli.Data; -static class EventJsonDocument +static class EventJsonFormat { - const string InvalidPropertyNameSubstitute = "(unnamed)"; - public static string EscapeUserPropertyName(string name) { - if (string.IsNullOrEmpty(name)) - return InvalidPropertyNameSubstitute; - return name.StartsWith('@') ? $"@{name}" : name; } - - public static void SetUserProperty(JsonObject eventJson, string name, JsonNode? value) - { - eventJson[EscapeUserPropertyName(name)] = value; - } - + public static JsonNode? CreateScalar(object? value) { return value switch diff --git a/src/SeqCli/Data/ScalarPropertyEnricher.cs b/src/SeqCli/Data/ScalarPropertyEnricher.cs index 6e9193d3..0d5f2a4c 100644 --- a/src/SeqCli/Data/ScalarPropertyEnricher.cs +++ b/src/SeqCli/Data/ScalarPropertyEnricher.cs @@ -23,12 +23,12 @@ class ScalarPropertyEnricher : IEventEnricher public ScalarPropertyEnricher(string name, object? scalarValue) { - _name = EventJsonDocument.EscapeUserPropertyName(name); + _name = EventJsonFormat.EscapeUserPropertyName(name); _scalarValue = scalarValue; } public void Enrich(JsonObject eventJson) { - eventJson[_name] = EventJsonDocument.CreateScalar(_scalarValue); + eventJson[_name] = EventJsonFormat.CreateScalar(_scalarValue); } } diff --git a/src/SeqCli/Mapping/EventEntityJson.cs b/src/SeqCli/Mapping/EventEntityJson.cs index 169d7a49..ac8fc6e4 100644 --- a/src/SeqCli/Mapping/EventEntityJson.cs +++ b/src/SeqCli/Mapping/EventEntityJson.cs @@ -75,7 +75,7 @@ public static JsonObject ToEventJson(EventEntity evt) if (evt.Properties != null) { foreach (var property in evt.Properties) - EventJsonDocument.SetUserProperty(eventJson, property.Name, ToSystemTextJson.FromApiValue(property.Value)); + eventJson[EventJsonFormat.EscapeUserPropertyName(property.Name)] = ToSystemTextJson.FromApiValue(property.Value); } return eventJson; diff --git a/src/SeqCli/PlainText/EventJsonBuilder.cs b/src/SeqCli/PlainText/EventJsonBuilder.cs index bff33836..7acddcb2 100644 --- a/src/SeqCli/PlainText/EventJsonBuilder.cs +++ b/src/SeqCli/PlainText/EventJsonBuilder.cs @@ -55,20 +55,22 @@ public static JsonObject FromProperties(IDictionary properties, foreach (var (name, value) in properties) { if (!ReifiedProperties.IsReifiedProperty(name)) - EventJsonDocument.SetUserProperty(eventJson, name, CreateValue(value)); + eventJson[EventJsonFormat.EscapeUserPropertyName(name)] = UnwrapTextSpans(value); } if (remainder != null) - EventJsonDocument.SetUserProperty(eventJson, "@unmatched", remainder); + eventJson[EventJsonFormat.EscapeUserPropertyName("@unmatched")] = UnwrapTextSpans(remainder); return eventJson; } - static JsonNode? CreateValue(object? value) + static JsonNode? UnwrapTextSpans(object? value) { + // We should consider whether text spans might also end up in extracted dictionary or array elements, though + // I don't think they will, currently. return value is TextSpan span ? JsonValue.Create(span.ToStringValue()) - : EventJsonDocument.CreateScalar(value); + : EventJsonFormat.CreateScalar(value); } static bool TryGetText(IDictionary properties, string name, out string text) diff --git a/src/SeqCli/Sample/Ingestion/MetricsMapping.cs b/src/SeqCli/Sample/Ingestion/MetricsMapping.cs index 6782e6c5..f50da4d7 100644 --- a/src/SeqCli/Sample/Ingestion/MetricsMapping.cs +++ b/src/SeqCli/Sample/Ingestion/MetricsMapping.cs @@ -50,7 +50,7 @@ public static bool TryGetMetricSampleJson(LogEvent logEvent, [NotNullWhen(true)] foreach (var (name, value) in logEvent.Properties) { if (name != SurrogateDefinitionsProperty) - EventJsonDocument.SetUserProperty(sample, name, SimulationEvent.ToJsonNode(value)); + sample[EventJsonFormat.EscapeUserPropertyName(name)] = SimulationEvent.ToJsonNode(value); } return true; diff --git a/src/SeqCli/Sample/Ingestion/SimulationEvent.cs b/src/SeqCli/Sample/Ingestion/SimulationEvent.cs index 89bc110c..988b4952 100644 --- a/src/SeqCli/Sample/Ingestion/SimulationEvent.cs +++ b/src/SeqCli/Sample/Ingestion/SimulationEvent.cs @@ -47,7 +47,7 @@ public static JsonObject ToJsonObject(LogEvent logEvent) eventJson["@sp"] = spanId.ToHexString(); foreach (var (name, value) in logEvent.Properties) - EventJsonDocument.SetUserProperty(eventJson, name, ToJsonNode(value)); + eventJson[EventJsonFormat.EscapeUserPropertyName(name)] = ToJsonNode(value); LiftSpanProperties(eventJson); @@ -59,7 +59,7 @@ public static JsonObject ToJsonObject(LogEvent logEvent) switch (value) { case ScalarValue scalar: - return EventJsonDocument.CreateScalar(scalar.Value); + return EventJsonFormat.CreateScalar(scalar.Value); case SequenceValue sequence: return new JsonArray(sequence.Elements.Select(ToJsonNode).ToArray()); @@ -83,7 +83,7 @@ public static JsonObject ToJsonObject(LogEvent logEvent) } default: - return EventJsonDocument.CreateScalar(value.ToString()); + return EventJsonFormat.CreateScalar(value.ToString()); } } diff --git a/src/SeqCli/SeqCli.csproj b/src/SeqCli/SeqCli.csproj index 0e97213a..16f10fb8 100644 --- a/src/SeqCli/SeqCli.csproj +++ b/src/SeqCli/SeqCli.csproj @@ -42,7 +42,7 @@ - + From edf85055c72daee3c5ac8cc06724d6ab10d2b7d2 Mon Sep 17 00:00:00 2001 From: Nicholas Blumhardt Date: Wed, 2 Sep 2026 11:24:26 +1000 Subject: [PATCH 14/15] Add coverage for `search --filter`. Assisted-by: Claude:claude-fable-5-1 --- .../Search/SearchWithFilterTestCase.cs | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 test/SeqCli.EndToEnd/Search/SearchWithFilterTestCase.cs diff --git a/test/SeqCli.EndToEnd/Search/SearchWithFilterTestCase.cs b/test/SeqCli.EndToEnd/Search/SearchWithFilterTestCase.cs new file mode 100644 index 00000000..0e3a2f1d --- /dev/null +++ b/test/SeqCli.EndToEnd/Search/SearchWithFilterTestCase.cs @@ -0,0 +1,35 @@ +using System; +using System.Linq; +using System.Threading.Tasks; +using Newtonsoft.Json.Linq; +using Seq.Api; +using SeqCli.EndToEnd.Support; +using Serilog; +using Xunit; + +namespace SeqCli.EndToEnd.Search; + +public class SearchWithFilterTestCase : ICliTestCase +{ + public async Task ExecuteAsync( + SeqConnection connection, + ILogger logger, + CliCommandRunner runner) + { + await DirectIngestion.IngestClef(connection, "'@mt': 'Event {N}', 'N': 1, 'Host': 'xmpweb-01.example.com'"); + await DirectIngestion.IngestClef(connection, "'@mt': 'Event {N}', 'N': 2, 'Host': 'xmpweb-02.example.com'"); + await DirectIngestion.IngestClef(connection, "'@mt': 'Event {N}', 'N': 3, 'Host': 'xmpweb-02.example.com'"); + + var exit = runner.Exec("search", "--filter=\"Host = 'xmpweb-02.example.com' and N > 2\" --count=10 --json"); + Assert.Equal(0, exit); + + var results = runner.LastRunProcess!.Output + .Split(['\r', '\n'], StringSplitOptions.RemoveEmptyEntries) + .Select(JObject.Parse) + .ToList(); + + var evt = Assert.Single(results); + Assert.Equal(3, evt["N"]!.Value()); + Assert.Equal("xmpweb-02.example.com", evt["Host"]!.Value()); + } +} From 2675a1d0334b7c119e47ea77c884b07b5b8f49ca Mon Sep 17 00:00:00 2001 From: Nicholas Blumhardt Date: Wed, 2 Sep 2026 17:08:15 +1000 Subject: [PATCH 15/15] Feedback --- .../{Mapping => Api}/EventEntityJson.cs | 3 +- src/SeqCli/{Mapping => Api}/LevelMapping.cs | 2 +- src/SeqCli/Apps/Hosting/AppContainer.cs | 2 +- .../Cli/Commands/Alert/CreateCommand.cs | 1 - .../Cli/Commands/ApiKey/CreateCommand.cs | 1 - src/SeqCli/Data/EventJsonFormat.cs | 10 +++-- src/SeqCli/Ingestion/JsonEventReader.cs | 3 +- src/SeqCli/Ingestion/LogShipper.cs | 10 ++--- src/SeqCli/Ingestion/ReadResult.cs | 18 +++++++-- src/SeqCli/Mcp/Tools/Search/SearchTools.cs | 2 +- src/SeqCli/Output/OutputFormat.cs | 1 - src/SeqCli/Output/TraceFormatter.cs | 3 +- src/SeqCli/PlainText/Extraction/Matchers.cs | 2 +- test/SeqCli.Tests/Output/OutputFormatTests.cs | 2 +- .../Output/TextFormattersTests.cs | 2 +- .../Sample/SimulationEventTests.cs | 2 +- .../Traces/StructuredMessageTests.cs | 38 +++++++++---------- 17 files changed, 55 insertions(+), 47 deletions(-) rename src/SeqCli/{Mapping => Api}/EventEntityJson.cs (98%) rename src/SeqCli/{Mapping => Api}/LevelMapping.cs (99%) diff --git a/src/SeqCli/Mapping/EventEntityJson.cs b/src/SeqCli/Api/EventEntityJson.cs similarity index 98% rename from src/SeqCli/Mapping/EventEntityJson.cs rename to src/SeqCli/Api/EventEntityJson.cs index ac8fc6e4..308c0735 100644 --- a/src/SeqCli/Mapping/EventEntityJson.cs +++ b/src/SeqCli/Api/EventEntityJson.cs @@ -19,11 +19,10 @@ using System.Text.Json.Nodes; using Seq.Api.Model.Events; using Seq.Api.Model.Shared; -using SeqCli.Api; using SeqCli.Data; using SeqCli.Output; -namespace SeqCli.Mapping; +namespace SeqCli.Api; /// /// Converts event entities into compact JSON format for further processing. This class is only necessary because diff --git a/src/SeqCli/Mapping/LevelMapping.cs b/src/SeqCli/Api/LevelMapping.cs similarity index 99% rename from src/SeqCli/Mapping/LevelMapping.cs rename to src/SeqCli/Api/LevelMapping.cs index 33639c3a..bc987977 100644 --- a/src/SeqCli/Mapping/LevelMapping.cs +++ b/src/SeqCli/Api/LevelMapping.cs @@ -16,7 +16,7 @@ using System.Collections.Generic; using Seq.Api.Model.LogEvents; -namespace SeqCli.Mapping; +namespace SeqCli.Api; public static class LevelMapping { diff --git a/src/SeqCli/Apps/Hosting/AppContainer.cs b/src/SeqCli/Apps/Hosting/AppContainer.cs index b65029f5..6949921d 100644 --- a/src/SeqCli/Apps/Hosting/AppContainer.cs +++ b/src/SeqCli/Apps/Hosting/AppContainer.cs @@ -21,7 +21,7 @@ using Newtonsoft.Json.Linq; using Seq.Apps; using Seq.Apps.LogEvents; -using SeqCli.Mapping; +using SeqCli.Api; using Serilog; using Serilog.Events; using Serilog.Formatting.Compact.Reader; diff --git a/src/SeqCli/Cli/Commands/Alert/CreateCommand.cs b/src/SeqCli/Cli/Commands/Alert/CreateCommand.cs index e74d833a..d8a2182d 100644 --- a/src/SeqCli/Cli/Commands/Alert/CreateCommand.cs +++ b/src/SeqCli/Cli/Commands/Alert/CreateCommand.cs @@ -21,7 +21,6 @@ using SeqCli.Api; using SeqCli.Cli.Features; using SeqCli.Config; -using SeqCli.Mapping; using SeqCli.Signals; using SeqCli.Syntax; using SeqCli.Util; diff --git a/src/SeqCli/Cli/Commands/ApiKey/CreateCommand.cs b/src/SeqCli/Cli/Commands/ApiKey/CreateCommand.cs index cf928d8c..25375b1e 100644 --- a/src/SeqCli/Cli/Commands/ApiKey/CreateCommand.cs +++ b/src/SeqCli/Cli/Commands/ApiKey/CreateCommand.cs @@ -21,7 +21,6 @@ using SeqCli.Api; using SeqCli.Cli.Features; using SeqCli.Config; -using SeqCli.Mapping; using SeqCli.Util; using Serilog; diff --git a/src/SeqCli/Data/EventJsonFormat.cs b/src/SeqCli/Data/EventJsonFormat.cs index 1f0a83dd..a862e4ff 100644 --- a/src/SeqCli/Data/EventJsonFormat.cs +++ b/src/SeqCli/Data/EventJsonFormat.cs @@ -13,7 +13,6 @@ // limitations under the License. using System; -using System.Globalization; using System.Text.Json.Nodes; namespace SeqCli.Data; @@ -25,6 +24,10 @@ public static string EscapeUserPropertyName(string name) return name.StartsWith('@') ? $"@{name}" : name; } + /// + /// Use this function when converting a value of uncertain or non-primitive type into a . It's + /// okay to use for strongly-typed primitives. + /// public static JsonNode? CreateScalar(object? value) { return value switch @@ -43,8 +46,9 @@ public static string EscapeUserPropertyName(string name) float n => JsonValue.Create(n), double n => JsonValue.Create(n), decimal n => JsonValue.Create(n), - DateTime dt => JsonValue.Create(dt.ToString("o", CultureInfo.InvariantCulture)), - DateTimeOffset dto => JsonValue.Create(dto.ToString("o", CultureInfo.InvariantCulture)), + TimeSpan ts => JsonValue.Create(ts.ToString("c")), + DateTime dt => JsonValue.Create(dt), + DateTimeOffset dto => JsonValue.Create(dto), _ => JsonValue.Create(value.ToString()) }; } diff --git a/src/SeqCli/Ingestion/JsonEventReader.cs b/src/SeqCli/Ingestion/JsonEventReader.cs index a8d4be8c..689710bd 100644 --- a/src/SeqCli/Ingestion/JsonEventReader.cs +++ b/src/SeqCli/Ingestion/JsonEventReader.cs @@ -13,7 +13,6 @@ // limitations under the License. using System; -using System.Globalization; using System.IO; using System.Text.Json.Nodes; using System.Threading.Tasks; @@ -55,7 +54,7 @@ static JsonObject ReadFromJson(string json) throw new InvalidDataException($"The line is not a JSON object: `{json.Trim()}`."); if (!eventJson.ContainsKey("@t")) - eventJson["@t"] = DateTime.UtcNow.ToString("O", CultureInfo.InvariantCulture); + eventJson["@t"] = DateTime.UtcNow; return eventJson; } diff --git a/src/SeqCli/Ingestion/LogShipper.cs b/src/SeqCli/Ingestion/LogShipper.cs index 68674fee..313d6ceb 100644 --- a/src/SeqCli/Ingestion/LogShipper.cs +++ b/src/SeqCli/Ingestion/LogShipper.cs @@ -22,7 +22,6 @@ using System.Text.Json.Nodes; using System.Threading; using System.Threading.Tasks; -using Newtonsoft.Json; using Seq.Api; using SeqCli.Api; using Serilog; @@ -132,7 +131,7 @@ public static async Task ShipEventsAsync( if (sendFailureHandling == SendFailureHandling.Retry) { var millisecondsDelay = (int)Math.Min(Math.Pow(2, retries) * 2000, 60000); - await Task.Delay(millisecondsDelay); + await Task.Delay(millisecondsDelay, cancellationToken); retries += 1; continue; } @@ -191,7 +190,7 @@ static async Task ReadBatchAsync( } catch (Exception ex) { - if (ex is System.Text.Json.JsonException || ex is InvalidDataException) + if (ex is System.Text.Json.JsonException or InvalidDataException) { if (invalidDataHandling == InvalidDataHandling.Ignore) continue; @@ -200,7 +199,7 @@ static async Task ReadBatchAsync( throw; } - return new BatchResult(batch.ToArray(), isLast); + return new BatchResult([.. batch], isLast); } while (true); } @@ -219,6 +218,7 @@ static async Task SendBatchAsync( using (var builder = new StringWriter()) { foreach (var evt in batch) + // ReSharper disable once MethodHasAsyncOverload builder.WriteLine(evt.ToJsonString()); content = new StringContent(builder.ToString(), Encoding.UTF8, ApiConstants.ClefMediaType); @@ -243,7 +243,7 @@ static async Task SendAsync(SeqConnection connection, string? ap { try { - var error = JsonConvert.DeserializeObject(resultJson)!; + var error = Newtonsoft.Json.JsonConvert.DeserializeObject(resultJson)!; sendFailureLog.Error("Shipping failed with status code {StatusCode}: {ErrorMessage}", result.StatusCode, diff --git a/src/SeqCli/Ingestion/ReadResult.cs b/src/SeqCli/Ingestion/ReadResult.cs index a44d30f4..0e074de5 100644 --- a/src/SeqCli/Ingestion/ReadResult.cs +++ b/src/SeqCli/Ingestion/ReadResult.cs @@ -1,13 +1,23 @@ +// Copyright © Datalust and contributors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + using System.Text.Json.Nodes; namespace SeqCli.Ingestion; readonly struct ReadResult { - /// - /// The event, as a JSON document in Seq's emission schema, or null if no event - /// is available. - /// public JsonObject? Document { get; } public bool IsAtEnd { get; } diff --git a/src/SeqCli/Mcp/Tools/Search/SearchTools.cs b/src/SeqCli/Mcp/Tools/Search/SearchTools.cs index 9ceda5e5..7d62d9ef 100644 --- a/src/SeqCli/Mcp/Tools/Search/SearchTools.cs +++ b/src/SeqCli/Mcp/Tools/Search/SearchTools.cs @@ -27,7 +27,7 @@ using Seq.Api.Model.Events; using Seq.Api.Model.Signals; using Seq.Syntax.Templates; -using SeqCli.Mapping; +using SeqCli.Api; using SeqCli.Signals; using SeqCli.Syntax; using Serilog; diff --git a/src/SeqCli/Output/OutputFormat.cs b/src/SeqCli/Output/OutputFormat.cs index 9efb3b97..ae2492fb 100644 --- a/src/SeqCli/Output/OutputFormat.cs +++ b/src/SeqCli/Output/OutputFormat.cs @@ -29,7 +29,6 @@ using SeqCli.Api; using SeqCli.Config; using SeqCli.Csv; -using SeqCli.Mapping; namespace SeqCli.Output; diff --git a/src/SeqCli/Output/TraceFormatter.cs b/src/SeqCli/Output/TraceFormatter.cs index c0a575ef..a69a1324 100644 --- a/src/SeqCli/Output/TraceFormatter.cs +++ b/src/SeqCli/Output/TraceFormatter.cs @@ -18,6 +18,7 @@ using System.Text; using System.Text.Json.Nodes; using SeqCli.Api; +using SeqCli.Data; using SeqCli.Traces; namespace SeqCli.Output; @@ -100,7 +101,7 @@ static JsonObject ToEventJson(TraceTreeNode treeNode, string treePrefix) eventJson[name] = value?.DeepClone(); if (evt.Elapsed is { } elapsed) - eventJson[ElapsedProperty] = elapsed.ToString("c", CultureInfo.InvariantCulture); + eventJson[ElapsedProperty] = EventJsonFormat.CreateScalar(elapsed); for (var i = 0; i < evt.Columns.Count; ++i) { diff --git a/src/SeqCli/PlainText/Extraction/Matchers.cs b/src/SeqCli/PlainText/Extraction/Matchers.cs index 5ae49e6e..fef3c2cc 100644 --- a/src/SeqCli/PlainText/Extraction/Matchers.cs +++ b/src/SeqCli/PlainText/Extraction/Matchers.cs @@ -3,7 +3,7 @@ using System.Globalization; using System.Linq; using System.Reflection; -using SeqCli.Mapping; +using SeqCli.Api; using SeqCli.PlainText.Parsers; using Superpower; using Superpower.Model; diff --git a/test/SeqCli.Tests/Output/OutputFormatTests.cs b/test/SeqCli.Tests/Output/OutputFormatTests.cs index a6777bef..e323aa60 100644 --- a/test/SeqCli.Tests/Output/OutputFormatTests.cs +++ b/test/SeqCli.Tests/Output/OutputFormatTests.cs @@ -1,8 +1,8 @@ using System.IO; using Newtonsoft.Json.Linq; using Seq.Api.Model.Events; +using SeqCli.Api; using SeqCli.Config; -using SeqCli.Mapping; using SeqCli.Output; using SeqCli.Tests.Support; using Xunit; diff --git a/test/SeqCli.Tests/Output/TextFormattersTests.cs b/test/SeqCli.Tests/Output/TextFormattersTests.cs index 7675926e..340b18fc 100644 --- a/test/SeqCli.Tests/Output/TextFormattersTests.cs +++ b/test/SeqCli.Tests/Output/TextFormattersTests.cs @@ -3,7 +3,7 @@ using System.IO; using System.Text.Json.Nodes; using Seq.Syntax.Templates.Themes; -using SeqCli.Mapping; +using SeqCli.Api; using SeqCli.Output; using SeqCli.Tests.Support; using Xunit; diff --git a/test/SeqCli.Tests/Sample/SimulationEventTests.cs b/test/SeqCli.Tests/Sample/SimulationEventTests.cs index 0a71da14..0c69a6e4 100644 --- a/test/SeqCli.Tests/Sample/SimulationEventTests.cs +++ b/test/SeqCli.Tests/Sample/SimulationEventTests.cs @@ -69,7 +69,7 @@ public void SerilogTracingSpanPropertiesAreLifted() .Information("GET /orders")); var eventJson = SimulationEvent.ToJsonObject(evt); - Assert.Equal(start.ToString("o"), (string?)eventJson["@st"]); + Assert.Equal(start, eventJson["@st"]!.GetValue()); Assert.Equal("8899aabbccddeeff", (string?)eventJson["@ps"]); Assert.False(eventJson.ContainsKey("SpanStartTimestamp")); Assert.False(eventJson.ContainsKey("ParentSpanId")); diff --git a/test/SeqCli.Tests/Traces/StructuredMessageTests.cs b/test/SeqCli.Tests/Traces/StructuredMessageTests.cs index 4180126e..a89ed699 100644 --- a/test/SeqCli.Tests/Traces/StructuredMessageTests.cs +++ b/test/SeqCli.Tests/Traces/StructuredMessageTests.cs @@ -22,8 +22,8 @@ public void MissingStructuredMessagesReadAsEmpty() { foreach (var cell in new object?[] { null, JValue.CreateNull() }) { - var (message, properties) = StructuredMessage.Read(cell); - Assert.Equal("", message); + var (mt, properties) = StructuredMessage.Read(cell); + Assert.Equal("", mt); Assert.Empty(properties); } } @@ -31,27 +31,26 @@ public void MissingStructuredMessagesReadAsEmpty() [Fact] public void TextTokensAreRead() { - var (message, properties) = StructuredMessage.Read(new JArray("Hello", ", ", "world")); + var (mt, properties) = StructuredMessage.Read(new JArray("Hello", ", ", "world")); - Assert.Equal("Hello, world", message); + Assert.Equal("Hello, world", mt); Assert.Empty(properties); } [Fact] public void LiteralBracesAreEscapedInTemplateText() { - var (message, _) = StructuredMessage.Read(new JArray("a {not-a-hole} b")); - - Assert.Equal("a {{not-a-hole}} b", message); + var (mt, _) = StructuredMessage.Read(new JArray("a {not-a-hole} b")); + Assert.Equal("a {{not-a-hole}} b", mt); } [Fact] public void HolesCarryRawTextAndValues() { - var (message, properties) = StructuredMessage.Read(new JArray( + var (mt, properties) = StructuredMessage.Read(new JArray( "Hello, ", Hole("Name", "{Name:x}", "World"), "!")); - Assert.Equal("Hello, {Name:x}!", message); + Assert.Equal("Hello, {Name:x}!", mt); var property = Assert.Single(properties); Assert.Equal("Name", property.Key); @@ -61,9 +60,9 @@ public void HolesCarryRawTextAndValues() [Fact] public void HolesWithoutValuesContributeNoProperties() { - var (message, properties) = StructuredMessage.Read(new JArray(Hole("Name"))); + var (mt, properties) = StructuredMessage.Read(new JArray(Hole("Name"))); - Assert.Equal("{Name}", message); + Assert.Equal("{Name}", mt); Assert.Empty(properties); } @@ -97,10 +96,10 @@ public void StructuredHoleValuesBecomeObjects() [Fact] public void DottedHoleNamesBecomeNestedObjects() { - var (message, properties) = StructuredMessage.Read(new JArray( + var (mt, properties) = StructuredMessage.Read(new JArray( Hole("user.name", value: "Barney"))); - Assert.Equal("{user.name}", message); + Assert.Equal("{user.name}", mt); var user = Assert.IsType(properties["user"]); Assert.Equal("Barney", (string?)user["name"]); } @@ -108,25 +107,24 @@ public void DottedHoleNamesBecomeNestedObjects() [Fact] public void TrailingWhitespaceIsTrimmed() { - var (message, _) = StructuredMessage.Read(new JArray("Hi ", "}", " \n")); + var (mt, _) = StructuredMessage.Read(new JArray("Hi ", "}", " \n")); - Assert.Equal("Hi }}", message); + Assert.Equal("Hi }}", mt); } [Fact] public void WhitespaceOnlyMessagesReadAsEmpty() { - var (message, _) = StructuredMessage.Read(new JArray(" ")); + var (mt, _) = StructuredMessage.Read(new JArray(" ")); - Assert.Equal("", message); + Assert.Equal("", mt); } [Fact] public void TrailingHolesAreNotTrimmed() { - var (message, _) = StructuredMessage.Read(new JArray("Took ", Hole("Elapsed"))); - - Assert.Equal("Took {Elapsed}", message); + var (mt, _) = StructuredMessage.Read(new JArray("Took ", Hole("Elapsed"))); + Assert.Equal("Took {Elapsed}", mt); } [Fact]