diff --git a/src/Roastery/Data/Database.cs b/src/Roastery/Data/Database.cs index 9234f1ce..251b67f1 100644 --- a/src/Roastery/Data/Database.cs +++ b/src/Roastery/Data/Database.cs @@ -1,6 +1,5 @@ using System; using System.Collections.Generic; -using System.Diagnostics; using System.Globalization; using System.Linq; using System.Reflection; diff --git a/src/Roastery/Util/Distribution.cs b/src/Roastery/Util/Distribution.cs index b20ffb49..5264cd14 100644 --- a/src/Roastery/Util/Distribution.cs +++ b/src/Roastery/Util/Distribution.cs @@ -1,7 +1,6 @@ using System; using System.Collections.Generic; using System.Runtime.CompilerServices; -using System.Threading; namespace Roastery.Util; diff --git a/src/Roastery/Web/RequestLoggingMiddleware.cs b/src/Roastery/Web/RequestLoggingMiddleware.cs index c75a5002..92d50bf3 100644 --- a/src/Roastery/Web/RequestLoggingMiddleware.cs +++ b/src/Roastery/Web/RequestLoggingMiddleware.cs @@ -1,6 +1,5 @@ using System; using System.Diagnostics; -using System.Diagnostics.Metrics; using System.Net; using System.Threading.Tasks; using Roastery.Metrics; diff --git a/src/SeqCli/Api/EventEntityJson.cs b/src/SeqCli/Api/EventEntityJson.cs new file mode 100644 index 00000000..308c0735 --- /dev/null +++ b/src/SeqCli/Api/EventEntityJson.cs @@ -0,0 +1,104 @@ +// Copyright © Datalust Pty Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Text; +using System.Text.Json.Nodes; +using Seq.Api.Model.Events; +using Seq.Api.Model.Shared; +using SeqCli.Data; +using SeqCli.Output; + +namespace SeqCli.Api; + +/// +/// Converts event entities into compact JSON format for further processing. This class is only necessary because +/// Seq.Api doesn't yet provide a simple compact-JSON based result format for searches. Once we've filled +/// that gap, this class, and can be removed. +/// +static class EventEntityJson +{ + public static JsonObject ToEventJson(EventEntity evt) + { + var eventJson = new JsonObject + { + // Earlier versions relied on Serilog output formatting to show timestamps in local time; we'll need + // to consider adding some compensating mechanism to `Seq.Syntax`. + ["@t"] = DateTimeOffset.ParseExact(evt.Timestamp, "o", CultureInfo.InvariantCulture) + .ToLocalTime().ToString("o", CultureInfo.InvariantCulture) + }; + + if (evt.MessageTemplateTokens != null) + eventJson["@mt"] = ToMessageTemplateText(evt.MessageTemplateTokens); + + if (!string.IsNullOrWhiteSpace(evt.Level) && evt.Level != "Information") + eventJson["@l"] = evt.Level; + + if (!string.IsNullOrWhiteSpace(evt.Exception)) + eventJson["@x"] = evt.Exception; + + if (!string.IsNullOrWhiteSpace(evt.TraceId)) + eventJson["@tr"] = evt.TraceId; + + if (!string.IsNullOrWhiteSpace(evt.SpanId)) + eventJson["@sp"] = evt.SpanId; + + if (!string.IsNullOrWhiteSpace(evt.ParentId)) + eventJson["@ps"] = evt.ParentId; + + if (!string.IsNullOrWhiteSpace(evt.Start)) + eventJson["@st"] = evt.Start; + + if (!string.IsNullOrWhiteSpace(evt.SpanKind)) + eventJson["@sk"] = evt.SpanKind; + + if (evt.Resource?.Count > 0) + eventJson["@ra"] = ToPropertiesObject(evt.Resource); + + if (evt.Scope?.Count > 0) + eventJson["@sa"] = ToPropertiesObject(evt.Scope); + + if (evt.Properties != null) + { + foreach (var property in evt.Properties) + eventJson[EventJsonFormat.EscapeUserPropertyName(property.Name)] = ToSystemTextJson.FromApiValue(property.Value); + } + + return eventJson; + } + + static string ToMessageTemplateText(List tokens) + { + var text = new StringBuilder(); + foreach (var token in tokens) + { + if (token.Text != null) + text.Append(token.Text.Replace("{", "{{").Replace("}", "}}")); + else + text.Append(token.RawText ?? $"{{{token.PropertyName}}}"); + } + + return text.ToString(); + } + + static JsonObject ToPropertiesObject(List properties) + { + var result = new JsonObject(); + foreach (var property in properties) + result[property.Name] = ToSystemTextJson.FromApiValue(property.Value); + return result; + } +} diff --git a/src/SeqCli/Api/LevelMapping.cs b/src/SeqCli/Api/LevelMapping.cs new file mode 100644 index 00000000..bc987977 --- /dev/null +++ b/src/SeqCli/Api/LevelMapping.cs @@ -0,0 +1,99 @@ +// Copyright © Datalust and contributors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Collections.Generic; +using Seq.Api.Model.LogEvents; + +namespace SeqCli.Api; + +public static class LevelMapping +{ + static readonly Dictionary LevelsByName = + new(StringComparer.OrdinalIgnoreCase) + { + ["t"] = "Trace", + ["tr"] = "Trace", + ["trc"] = "Trace", + ["trce"] = "Trace", + ["trace"] = "Trace", + ["v"] = "Verbose", + ["ver"] = "Verbose", + ["vrb"] = "Verbose", + ["verb"] = "Verbose", + ["verbose"] = "Verbose", + ["d"] = "Debug", + ["de"] = "Debug", + ["dbg"] = "Debug", + ["deb"] = "Debug", + ["dbug"] = "Debug", + ["debu"] = "Debug", + ["debug"] = "Debug", + ["i"] = "Information", + ["in"] = "Information", + ["inf"] = "Information", + ["info"] = "Information", + ["information"] = "Information", + ["notice"] = "Notice", + ["w"] = "Warning", + ["wa"] = "Warning", + ["war"] = "Warning", + ["wrn"] = "Warning", + ["warn"] = "Warning", + ["warning"] = "Warning", + ["e"] = "Error", + ["er"] = "Error", + ["err"] = "Error", + ["erro"] = "Error", + ["eror"] = "Error", + ["error"] = "Error", + ["f"] = "Fatal", + ["fa"] = "Fatal", + ["ftl"] = "Fatal", + ["fat"] = "Fatal", + ["fatl"] = "Fatal", + ["fatal"] = "Fatal", + ["c"] = "Critical", + ["cr"] = "Critical", + ["crt"] = "Critical", + ["cri"] = "Critical", + ["crit"] = "Critical", + ["critical"] = "Critical", + ["emerg"] = "Emergency", + ["alert"] = "Alert", + ["panic"] = "Panic" + }; + + // Intended only for use by ingest extraction patterns. + public static string ToFullLevelName(string level) + { + return LevelsByName.TryGetValue(level, out var m) ? m : level; + } + + public static LogEventLevel ToSeqApiLogEventLevel(string level) + { + if (string.IsNullOrEmpty(level)) + return LogEventLevel.Information; + + return ToFullLevelName(level) switch + { + "Trace" or "Verbose" => LogEventLevel.Verbose, + "Debug" => LogEventLevel.Debug, + "Warning" => LogEventLevel.Warning, + "Error" => LogEventLevel.Error, + "Fatal" or "Critical" or "Emergency" or "Alert" or "Panic" => LogEventLevel.Fatal, + _ => LogEventLevel.Information + }; + } +} diff --git a/src/SeqCli/Api/ToSystemTextJson.cs b/src/SeqCli/Api/ToSystemTextJson.cs new file mode 100644 index 00000000..92428c06 --- /dev/null +++ b/src/SeqCli/Api/ToSystemTextJson.cs @@ -0,0 +1,45 @@ +// Copyright © Datalust Pty Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System.Text.Json.Nodes; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; +using SeqCli.Data; + +namespace SeqCli.Api; + +static class ToSystemTextJson +{ + /// + /// Convert a value deserialized by the Seq API client into its `System.Text.Json` equivalent. + /// + public static JsonNode? FromApiValue(object? value) + { + return value switch + { + null => null, + JToken token => FromNewtonsoft(token), + _ => EventJsonFormat.CreateScalar(value) + }; + } + + /// Conversion helper for values retrieved through the Seq API client. + public static JsonNode? FromNewtonsoft(JToken token) + { + if (token is JValue { Value: null }) + return null; + + return JsonNode.Parse(token.ToString(Formatting.None)); + } +} diff --git a/src/SeqCli/Apps/AppLoader.cs b/src/SeqCli/Apps/AppLoader.cs index c0a03ff5..e46cdfed 100644 --- a/src/SeqCli/Apps/AppLoader.cs +++ b/src/SeqCli/Apps/AppLoader.cs @@ -34,7 +34,8 @@ class AppLoader : IDisposable [ typeof(SeqApp).Assembly, typeof(Log).Assembly, - typeof(SerilogExpression).Assembly + // Seq.Syntax uses version-specific assembly names to improve our chances of successful loading. + typeof(SeqExpression).Assembly ]; public AppLoader(string packageBinaryPath) diff --git a/src/SeqCli/Apps/Hosting/AppContainer.cs b/src/SeqCli/Apps/Hosting/AppContainer.cs index 8a58c2bd..6949921d 100644 --- a/src/SeqCli/Apps/Hosting/AppContainer.cs +++ b/src/SeqCli/Apps/Hosting/AppContainer.cs @@ -21,7 +21,7 @@ using Newtonsoft.Json.Linq; using Seq.Apps; using Seq.Apps.LogEvents; -using SeqCli.Mapping; +using SeqCli.Api; using Serilog; using Serilog.Events; using Serilog.Formatting.Compact.Reader; @@ -109,11 +109,11 @@ async Task SendTypedEventAsync(string clef) { if (_seqApp is ISubscribeTo led) { - led.On(EventFormat.FromRaw(eventId, eventType, serilogEvent)); + led.On(EventFormat.FromSerilogLogEvent(eventId, eventType, serilogEvent)); } else if (_seqApp is ISubscribeToAsync leda) { - await leda.OnAsync(EventFormat.FromRaw(eventId, eventType, serilogEvent)); + await leda.OnAsync(EventFormat.FromSerilogLogEvent(eventId, eventType, serilogEvent)); } else if (_seqApp is ISubscribeTo sled) { @@ -143,7 +143,8 @@ LogEvent ReadSerilogEvent(string clef, out string eventId, out uint eventType) if (jobject.TryGetValue("@l", out var levelToken)) { jobject.Remove("@l"); - jobject.Add("@l", new JValue(LevelMapping.ToSerilogLevel(levelToken.Value()!).ToString())); + // The Seq.Api `LogEventLevel` enum intentionally matches the Serilog one. + jobject.Add("@l", new JValue(LevelMapping.ToSeqApiLogEventLevel(levelToken.Value()!).ToString())); } SanitizeTraceIdentifiers(jobject); diff --git a/src/SeqCli/Apps/Hosting/EventFormat.cs b/src/SeqCli/Apps/Hosting/EventFormat.cs index 1ff23253..67a37b4d 100644 --- a/src/SeqCli/Apps/Hosting/EventFormat.cs +++ b/src/SeqCli/Apps/Hosting/EventFormat.cs @@ -24,7 +24,7 @@ namespace SeqCli.Apps.Hosting; static class EventFormat { - public static Event FromRaw(string eventId, uint eventType, LogEvent raw) + public static Event FromSerilogLogEvent(string eventId, uint eventType, LogEvent raw) { var properties = new Dictionary(); foreach (var prop in raw.Properties) diff --git a/src/SeqCli/Cli/Commands/Alert/CreateCommand.cs b/src/SeqCli/Cli/Commands/Alert/CreateCommand.cs index 49ceba1d..d8a2182d 100644 --- a/src/SeqCli/Cli/Commands/Alert/CreateCommand.cs +++ b/src/SeqCli/Cli/Commands/Alert/CreateCommand.cs @@ -17,12 +17,10 @@ using System.Linq; using System.Threading.Tasks; using Seq.Api.Model.Alerting; -using Seq.Api.Model.LogEvents; using Seq.Api.Model.Shared; using SeqCli.Api; using SeqCli.Cli.Features; using SeqCli.Config; -using SeqCli.Mapping; using SeqCli.Signals; using SeqCli.Syntax; using SeqCli.Util; @@ -178,7 +176,7 @@ protected override async Task Run() alert.Having = _having; if (_notificationLevel != null) - alert.NotificationLevel = Enum.Parse(LevelMapping.ToFullLevelName(_notificationLevel)); + alert.NotificationLevel = LevelMapping.ToSeqApiLogEventLevel(_notificationLevel); if (_suppressionTime != null) alert.SuppressionTime = DurationMoniker.ToTimeSpan(_suppressionTime); diff --git a/src/SeqCli/Cli/Commands/ApiKey/CreateCommand.cs b/src/SeqCli/Cli/Commands/ApiKey/CreateCommand.cs index fe514434..25375b1e 100644 --- a/src/SeqCli/Cli/Commands/ApiKey/CreateCommand.cs +++ b/src/SeqCli/Cli/Commands/ApiKey/CreateCommand.cs @@ -16,13 +16,11 @@ using System.Linq; using System.Threading.Tasks; using Seq.Api; -using Seq.Api.Model.LogEvents; using Seq.Api.Model.Security; using Seq.Api.Model.Shared; using SeqCli.Api; using SeqCli.Cli.Features; using SeqCli.Config; -using SeqCli.Mapping; using SeqCli.Util; using Serilog; @@ -125,7 +123,7 @@ protected override async Task Run() if (_level != null) { - apiKey.InputSettings.MinimumLevel = Enum.Parse(LevelMapping.ToFullLevelName(_level)); + apiKey.InputSettings.MinimumLevel = LevelMapping.ToSeqApiLogEventLevel(_level); } apiKey.AssignedPermissions.Clear(); diff --git a/src/SeqCli/Cli/Commands/IngestCommand.cs b/src/SeqCli/Cli/Commands/IngestCommand.cs index b965ddb3..e0ad35a6 100644 --- a/src/SeqCli/Cli/Commands/IngestCommand.cs +++ b/src/SeqCli/Cli/Commands/IngestCommand.cs @@ -14,18 +14,17 @@ using System; using System.Collections.Generic; +using System.Text.Json.Nodes; using System.Threading; using System.Threading.Tasks; using SeqCli.Api; using SeqCli.Cli.Features; using SeqCli.Config; +using SeqCli.Data; using SeqCli.Ingestion; -using SeqCli.Mapping; using SeqCli.PlainText; using SeqCli.Syntax; using Serilog; -using Serilog.Core; -using Serilog.Events; namespace SeqCli.Cli.Commands; @@ -84,19 +83,19 @@ protected override async Task Run() { try { - var enrichers = new List(); - + var enrichers = new List(); + if (_level != null) - enrichers.Add(new ScalarPropertyEnricher(LevelMapping.SurrogateLevelProperty, _level)); - + enrichers.Add(new LevelEnricher(_level)); + foreach (var (name, value) in _properties.FlatProperties) enrichers.Add(new ScalarPropertyEnricher(name, value)); - Func? filter = null; + Func? filter = null; if (_filter != null) { var eval = SeqSyntax.CompileExpression(_filter); - filter = evt => Seq.Syntax.Expressions.ExpressionResult.IsTrue(eval(evt)); + filter = evt => eval(evt).IsTrue(); } var config = RuntimeConfigurationLoader.Load(_storagePath); @@ -112,9 +111,9 @@ protected override async Task Run() { using (input) { - ILogEventReader reader = _json - ? new JsonLogEventReader(input) - : new PlainTextLogEventReader(input, _pattern); + IEventReader reader = _json + ? new JsonEventReader(input) + : new PlainTextEventReader(input, _pattern); reader = new EnrichingReader(reader, enrichers); diff --git a/src/SeqCli/Cli/Commands/PrintCommand.cs b/src/SeqCli/Cli/Commands/PrintCommand.cs index 2740297f..0bc67f59 100644 --- a/src/SeqCli/Cli/Commands/PrintCommand.cs +++ b/src/SeqCli/Cli/Commands/PrintCommand.cs @@ -14,16 +14,16 @@ using System; using System.IO; +using System.Text.Json; +using System.Text.Json.Nodes; using System.Threading.Tasks; -using Newtonsoft.Json; -using Seq.Syntax.Expressions; using SeqCli.Cli.Features; using SeqCli.Config; using SeqCli.Ingestion; using SeqCli.Output; +using SeqCli.Syntax; using SeqCli.Util; using Serilog; -using Serilog.Events; namespace SeqCli.Cli.Commands; @@ -61,16 +61,16 @@ protected override async Task Run() { var config = RuntimeConfigurationLoader.Load(_storage); - Func? filter = null; + Func? filter = null; if (_filter != null) { - if (!SerilogExpression.TryCompile(_filter, out var compiled, out var error)) + if (!SeqSyntax.TryCompileExpression(_filter, out var compiled, out var error)) { Log.Error("The specified filter could not be compiled: {Error}", error); return 1; } - filter = evt => ExpressionResult.IsTrue(compiled(evt)); + filter = evt => compiled(evt).IsTrue(); } var template = _template == null ? null : PrintTemplate.InterpretEscapeChars(_template); @@ -80,7 +80,7 @@ protected override async Task Run() { using (input) { - var reader = new JsonLogEventReader(input); + var reader = new JsonEventReader(input); var isAtEnd = false; do @@ -90,12 +90,12 @@ protected override async Task Run() var result = await reader.TryReadAsync(); isAtEnd = result.IsAtEnd; - if (result.LogEvent != null && (filter == null || filter(result.LogEvent))) - output.WriteLogEvent(result.LogEvent); + if (result.Document != null && (filter == null || filter(result.Document))) + output.WriteEvent(result.Document); } catch (Exception ex) { - if (ex is not JsonReaderException && ex is not InvalidDataException || + if (ex is not JsonException && ex is not InvalidDataException || _invalidDataHandlingFeature.InvalidDataHandling != InvalidDataHandling.Ignore) throw; } diff --git a/src/SeqCli/Cli/Commands/TailCommand.cs b/src/SeqCli/Cli/Commands/TailCommand.cs index 9d4d4957..291433ba 100644 --- a/src/SeqCli/Cli/Commands/TailCommand.cs +++ b/src/SeqCli/Cli/Commands/TailCommand.cs @@ -13,6 +13,8 @@ // limitations under the License. using System; +using System.IO; +using System.Text.Json.Nodes; using System.Threading; using System.Threading.Tasks; using SeqCli.Api; @@ -63,13 +65,15 @@ protected override async Task Run() try { - await foreach (var evt in connection.Events.StreamAsync( + await foreach (var evt in connection.Events.StreamDocumentsAsync( filter: strict, signal: _signal.Signal, render: true, + clef: true, cancellationToken: cancel.Token)) { - output.WriteEventEntity(evt); + var eventJson = JsonNode.Parse(evt)?.AsObject() ?? throw new InvalidDataException("Non-JSON document received."); + output.WriteEvent(eventJson); } } catch (OperationCanceledException) diff --git a/src/SeqCli/Cli/Commands/TraceCommand.cs b/src/SeqCli/Cli/Commands/TraceCommand.cs index 6e255069..67541f3b 100644 --- a/src/SeqCli/Cli/Commands/TraceCommand.cs +++ b/src/SeqCli/Cli/Commands/TraceCommand.cs @@ -151,8 +151,8 @@ protected override async Task Run() } else { - foreach (var logEvent in TraceFormatter.ToLogEvents(subtreeRoot != null ? [subtreeRoot] : roots)) - output.WriteLogEvent(logEvent); + foreach (var eventJson in TraceFormatter.ToEventJson(subtreeRoot != null ? [subtreeRoot] : roots)) + output.WriteEvent(eventJson); } return 0; diff --git a/src/SeqCli/Csv/CsvWriter.cs b/src/SeqCli/Csv/CsvWriter.cs index 75f6553a..f87ae2a3 100644 --- a/src/SeqCli/Csv/CsvWriter.cs +++ b/src/SeqCli/Csv/CsvWriter.cs @@ -1,22 +1,34 @@ using System; -using System.Collections.Generic; using System.IO; using Seq.Api.Model.Data; +using Seq.Syntax.Templates.Themes; using SeqCli.Mcp.Data; -using SeqCli.Output; -using Serilog.Templates.Themes; namespace SeqCli.Csv; static class CsvWriter { + // Delimited output is written directly rather than rendered through a template, so styled + // runs are opened and closed here. + static void SetStyle(TextWriter output, TemplateTheme? theme, TemplateThemeStyle style) + { + if (theme?.Open(style) is { } open) + output.Write(open); + } + + static void ResetStyle(TextWriter output, TemplateTheme? theme, TemplateThemeStyle style) + { + if (theme?.Close(style) is { } close) + output.Write(close); + } + public static void WriteQueryResult(QueryResultPart result, Func stringify, TemplateTheme? theme, TextWriter output) { if (!string.IsNullOrWhiteSpace(result.Error)) { - theme?.Set(output, TemplateThemeStyle.Text); + SetStyle(output, theme, TemplateThemeStyle.Text); QueryResultHelper.WriteErrorResult(output, result); - theme?.Reset(output); + ResetStyle(output, theme, TemplateThemeStyle.Text); } var first = true; @@ -40,39 +52,39 @@ static void WriteCell(TextWriter output, TemplateTheme? theme, object? value, Fu } else { - theme?.Set(output, TemplateThemeStyle.TertiaryText); + SetStyle(output, theme, TemplateThemeStyle.TertiaryText); output.Write(','); - theme?.Reset(output); + ResetStyle(output, theme, TemplateThemeStyle.TertiaryText); } - - theme?.Set(output, TemplateThemeStyle.TertiaryText); + + SetStyle(output, theme, TemplateThemeStyle.TertiaryText); output.Write('"'); - theme?.Reset(output); + ResetStyle(output, theme, TemplateThemeStyle.TertiaryText); var valueAsString = stringify(value); - + var dataStyle = isHeadingRow ? TemplateThemeStyle.Name : TemplateThemeStyle.Text; var doubleQuote = valueAsString.IndexOf('"'); while (doubleQuote != -1) { - theme?.Set(output, dataStyle); + SetStyle(output, theme, dataStyle); output.Write(valueAsString[..doubleQuote]); - theme?.Reset(output); - - theme?.Set(output, TemplateThemeStyle.Scalar); + ResetStyle(output, theme, dataStyle); + + SetStyle(output, theme, TemplateThemeStyle.Scalar); output.Write("\"\""); - theme?.Reset(output); + ResetStyle(output, theme, TemplateThemeStyle.Scalar); valueAsString = valueAsString[(doubleQuote + 1)..]; doubleQuote = valueAsString.IndexOf('"'); } - - theme?.Set(output, dataStyle); + + SetStyle(output, theme, dataStyle); output.Write(valueAsString); - theme?.Reset(output); - - theme?.Set(output, TemplateThemeStyle.TertiaryText); + ResetStyle(output, theme, dataStyle); + + SetStyle(output, theme, TemplateThemeStyle.TertiaryText); output.Write('"'); - theme?.Reset(output); + ResetStyle(output, theme, TemplateThemeStyle.TertiaryText); } } \ No newline at end of file diff --git a/src/SeqCli/Data/EventJsonFormat.cs b/src/SeqCli/Data/EventJsonFormat.cs new file mode 100644 index 00000000..a862e4ff --- /dev/null +++ b/src/SeqCli/Data/EventJsonFormat.cs @@ -0,0 +1,55 @@ +// Copyright © Datalust and contributors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Text.Json.Nodes; + +namespace SeqCli.Data; + +static class EventJsonFormat +{ + public static string EscapeUserPropertyName(string name) + { + return name.StartsWith('@') ? $"@{name}" : name; + } + + /// + /// Use this function when converting a value of uncertain or non-primitive type into a . It's + /// okay to use for strongly-typed primitives. + /// + public static JsonNode? CreateScalar(object? value) + { + return value switch + { + null => null, + string s => JsonValue.Create(s), + bool b => JsonValue.Create(b), + byte n => JsonValue.Create(n), + sbyte n => JsonValue.Create(n), + short n => JsonValue.Create(n), + ushort n => JsonValue.Create(n), + int n => JsonValue.Create(n), + uint n => JsonValue.Create(n), + long n => JsonValue.Create(n), + ulong n => JsonValue.Create(n), + float n => JsonValue.Create(n), + double n => JsonValue.Create(n), + decimal n => JsonValue.Create(n), + TimeSpan ts => JsonValue.Create(ts.ToString("c")), + DateTime dt => JsonValue.Create(dt), + DateTimeOffset dto => JsonValue.Create(dto), + _ => JsonValue.Create(value.ToString()) + }; + } +} diff --git a/src/SeqCli/Util/TextException.cs b/src/SeqCli/Data/IEventEnricher.cs similarity index 60% rename from src/SeqCli/Util/TextException.cs rename to src/SeqCli/Data/IEventEnricher.cs index 2017129d..55c8584b 100644 --- a/src/SeqCli/Util/TextException.cs +++ b/src/SeqCli/Data/IEventEnricher.cs @@ -1,4 +1,4 @@ -// Copyright 2013-2015 Serilog Contributors +// Copyright © Datalust and contributors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,22 +12,15 @@ // See the License for the specific language governing permissions and // limitations under the License. -using System; +using System.Text.Json.Nodes; -namespace SeqCli.Util; +namespace SeqCli.Data; -class TextException : Exception +/// +/// Adds or updates fields on an event JSON document; the equivalent, in Seq's data model, of a +/// Serilog enricher. +/// +interface IEventEnricher { - readonly string _text; - - public TextException(string text) - : base("This exception type provides ToString() access to details only.") - { - _text = text; - } - - public override string ToString() - { - return _text; - } -} \ No newline at end of file + void Enrich(JsonObject eventJson); +} diff --git a/src/SeqCli/Output/RedundantEventTypeRemovalEnricher.cs b/src/SeqCli/Data/LevelEnricher.cs similarity index 64% rename from src/SeqCli/Output/RedundantEventTypeRemovalEnricher.cs rename to src/SeqCli/Data/LevelEnricher.cs index d32e6666..b50df51c 100644 --- a/src/SeqCli/Output/RedundantEventTypeRemovalEnricher.cs +++ b/src/SeqCli/Data/LevelEnricher.cs @@ -1,4 +1,4 @@ -// Copyright © Datalust and contributors. +// Copyright © Datalust and contributors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,15 +12,17 @@ // See the License for the specific language governing permissions and // limitations under the License. -using Serilog.Core; -using Serilog.Events; +using System.Text.Json.Nodes; -namespace SeqCli.Output; +namespace SeqCli.Data; -public class RedundantEventTypeRemovalEnricher : ILogEventEnricher +/// +/// Overrides the event's @l level with a fixed value. +/// +class LevelEnricher(string level) : IEventEnricher { - public void Enrich(LogEvent logEvent, ILogEventPropertyFactory propertyFactory) + public void Enrich(JsonObject eventJson) { - logEvent.RemovePropertyIfPresent("@i"); + eventJson["@l"] = level; } -} \ No newline at end of file +} diff --git a/src/SeqCli/Ingestion/ScalarPropertyEnricher.cs b/src/SeqCli/Data/ScalarPropertyEnricher.cs similarity index 59% rename from src/SeqCli/Ingestion/ScalarPropertyEnricher.cs rename to src/SeqCli/Data/ScalarPropertyEnricher.cs index 7146c7e7..0d5f2a4c 100644 --- a/src/SeqCli/Ingestion/ScalarPropertyEnricher.cs +++ b/src/SeqCli/Data/ScalarPropertyEnricher.cs @@ -1,4 +1,4 @@ -// Copyright © Datalust and contributors. +// Copyright © Datalust and contributors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,23 +12,23 @@ // See the License for the specific language governing permissions and // limitations under the License. -using SeqCli.Util; -using Serilog.Core; -using Serilog.Events; +using System.Text.Json.Nodes; -namespace SeqCli.Ingestion; +namespace SeqCli.Data; -class ScalarPropertyEnricher : ILogEventEnricher +class ScalarPropertyEnricher : IEventEnricher { - readonly LogEventProperty _property; + readonly string _name; + readonly object? _scalarValue; public ScalarPropertyEnricher(string name, object? scalarValue) { - _property = LogEventPropertyFactory.SafeCreate(name, new ScalarValue(scalarValue)); + _name = EventJsonFormat.EscapeUserPropertyName(name); + _scalarValue = scalarValue; } - public void Enrich(LogEvent logEvent, ILogEventPropertyFactory propertyFactory) + public void Enrich(JsonObject eventJson) { - logEvent.AddOrUpdateProperty(_property); + eventJson[_name] = EventJsonFormat.CreateScalar(_scalarValue); } -} \ No newline at end of file +} diff --git a/src/SeqCli/Forwarder/ForwarderModule.cs b/src/SeqCli/Forwarder/ForwarderModule.cs index 6bb7ef67..9ed614b4 100644 --- a/src/SeqCli/Forwarder/ForwarderModule.cs +++ b/src/SeqCli/Forwarder/ForwarderModule.cs @@ -22,8 +22,6 @@ using SeqCli.Forwarder.Web.Api; using SeqCli.Forwarder.Web.Host; using Serilog; -using Serilog.Formatting; -using Serilog.Templates; namespace SeqCli.Forwarder; @@ -66,25 +64,17 @@ protected override void Load(ContainerBuilder builder) if (_config.Forwarder.Diagnostics.ExposeIngestionLog) { Log.ForContext().Warning("Configured to expose ingestion log via HTTP API"); - builder.RegisterType().As(); - - var ingestionLogTemplate = $"[{{@t:o}} {{@l:u3}}] {{@m}}{Environment.NewLine}"; if (_config.Forwarder.Diagnostics.IngestionLogShowDetail) { Log.ForContext().Warning("Including full client, payload, and error detail in the ingestion log"); - ingestionLogTemplate += - $"{{#if ClientHostIP is not null}}Client IP address: {{ClientHostIP}}{Environment.NewLine}{{#end}}" + - $"{{#if DocumentStart is not null}}First {{StartToLog}} characters of payload: {{DocumentStart:l}}{Environment.NewLine}{{#end}}" + - "{@x}"; } - - builder.Register(_ => new ExpressionTemplate(ingestionLogTemplate)).As(); + + builder.Register(_ => new IngestionLogEndpoints(_config.Forwarder.Diagnostics.IngestionLogShowDetail)).As(); } - builder.Register(c => + builder.Register(_ => { - var config = c.Resolve(); - var baseUri = config.Connection.ServerUrl; + var baseUri = _config.Connection.ServerUrl; if (string.IsNullOrWhiteSpace(baseUri)) throw new ArgumentException("The destination Seq server URL must be configured in `SeqCli.json`."); @@ -95,13 +85,13 @@ protected override void Load(ContainerBuilder builder) // this expression, using an "or" operator. var hasSocketHandlerOption = - config.Connection.PooledConnectionLifetimeMilliseconds.HasValue; + _config.Connection.PooledConnectionLifetimeMilliseconds.HasValue; if (hasSocketHandlerOption) { var httpMessageHandler = new SocketsHttpHandler { - PooledConnectionLifetime = config.Connection.PooledConnectionLifetimeMilliseconds.HasValue ? TimeSpan.FromMilliseconds(config.Connection.PooledConnectionLifetimeMilliseconds.Value) : Timeout.InfiniteTimeSpan, + PooledConnectionLifetime = _config.Connection.PooledConnectionLifetimeMilliseconds.HasValue ? TimeSpan.FromMilliseconds(_config.Connection.PooledConnectionLifetimeMilliseconds.Value) : Timeout.InfiniteTimeSpan, }; return new HttpClient(httpMessageHandler) { BaseAddress = new Uri(baseUri) }; diff --git a/src/SeqCli/Forwarder/Web/Api/IngestionLogEndpoints.cs b/src/SeqCli/Forwarder/Web/Api/IngestionLogEndpoints.cs index cf30acfb..eba1c743 100644 --- a/src/SeqCli/Forwarder/Web/Api/IngestionLogEndpoints.cs +++ b/src/SeqCli/Forwarder/Web/Api/IngestionLogEndpoints.cs @@ -12,23 +12,25 @@ // See the License for the specific language governing permissions and // limitations under the License. +using System; +using System.Globalization; using System.IO; using System.Text; using Microsoft.AspNetCore.Builder; using Microsoft.AspNetCore.Http; using SeqCli.Forwarder.Diagnostics; -using Serilog.Formatting; +using Serilog.Events; namespace SeqCli.Forwarder.Web.Api; class IngestionLogEndpoints : IMapEndpoints { - readonly ITextFormatter _formatter; + readonly bool _showDetail; readonly Encoding _utf8 = new UTF8Encoding(false); - public IngestionLogEndpoints(ITextFormatter formatter) + public IngestionLogEndpoints(bool showDetail) { - _formatter = formatter; + _showDetail = showDetail; } public void MapEndpoints(WebApplication app) @@ -45,10 +47,55 @@ public void MapEndpoints(WebApplication app) using var log = new StringWriter(); foreach (var logEvent in events) { - _formatter.Format(logEvent, log); + Format(logEvent, log); } return Results.Content(log.ToString(), "text/plain", _utf8); }); } + + void Format(LogEvent logEvent, TextWriter log) + { + log.Write($"[{logEvent.Timestamp:o} {Abbreviate(logEvent.Level)}] "); + + static string Abbreviate(LogEventLevel logEventLevel) + { + // Here because we don't want Serilog level conversion routines, or any other Serilog model conversion + // routines, to propagate. + return logEventLevel switch + { + LogEventLevel.Verbose => "VRB", + LogEventLevel.Debug => "DBG", + LogEventLevel.Information => "INF", + LogEventLevel.Warning => "WAR", + LogEventLevel.Error => "ERR", + LogEventLevel.Fatal => "FTL", + _ => throw new ArgumentOutOfRangeException(nameof(logEventLevel), logEventLevel, null) + }; + } + + logEvent.RenderMessage(log, CultureInfo.InvariantCulture); + log.WriteLine(); + if (_showDetail) + { + if (logEvent.Properties.TryGetValue("ClientHostIP", out var clientHostIPProperty) && + clientHostIPProperty is ScalarValue { Value: string clientHostIP}) + { + log.WriteLine($"Client IP address: {clientHostIP}"); + } + + if (logEvent.Properties.TryGetValue("DocumentStart", out var documentStartProperty) && + documentStartProperty is ScalarValue { Value: string documentStart} && + logEvent.Properties.TryGetValue("StartToLog", out var startToLogProperty) && + startToLogProperty is ScalarValue { Value: {} startToLog }) + { + log.WriteLine($"First {startToLog} characters of payload: {documentStart}"); + } + + if (logEvent.Exception is { } exception) + { + log.WriteLine(exception); + } + } + } } diff --git a/src/SeqCli/Ingestion/BatchResult.cs b/src/SeqCli/Ingestion/BatchResult.cs index 0c4b52ec..daef0697 100644 --- a/src/SeqCli/Ingestion/BatchResult.cs +++ b/src/SeqCli/Ingestion/BatchResult.cs @@ -1,15 +1,15 @@ -using Serilog.Events; +using System.Text.Json.Nodes; namespace SeqCli.Ingestion; struct BatchResult { - public LogEvent[] LogEvents { get; } + public JsonObject[] Documents { get; } public bool IsLast { get; } - public BatchResult(LogEvent[] logEvents, bool isLast) + public BatchResult(JsonObject[] documents, bool isLast) { - LogEvents = logEvents; + Documents = documents; IsLast = isLast; } -} \ No newline at end of file +} diff --git a/src/SeqCli/Ingestion/EnrichingReader.cs b/src/SeqCli/Ingestion/EnrichingReader.cs index 198ab234..6207bf4c 100644 --- a/src/SeqCli/Ingestion/EnrichingReader.cs +++ b/src/SeqCli/Ingestion/EnrichingReader.cs @@ -1,18 +1,18 @@ using System; using System.Collections.Generic; using System.Threading.Tasks; -using Serilog.Core; +using SeqCli.Data; namespace SeqCli.Ingestion; -class EnrichingReader : ILogEventReader +class EnrichingReader : IEventReader { - readonly ILogEventReader _inner; - readonly IReadOnlyCollection _enrichers; + readonly IEventReader _inner; + readonly IReadOnlyCollection _enrichers; public EnrichingReader( - ILogEventReader inner, - IReadOnlyCollection enrichers) + IEventReader inner, + IReadOnlyCollection enrichers) { _inner = inner ?? throw new ArgumentNullException(nameof(inner)); _enrichers = enrichers ?? throw new ArgumentNullException(nameof(enrichers)); @@ -22,13 +22,12 @@ public async Task TryReadAsync() { var result = await _inner.TryReadAsync(); - if (result.LogEvent != null) + if (result.Document != null) { foreach (var enricher in _enrichers) - // We're breaking the nullability contract of `ILogEventEnricher.Enrich()`, here. - enricher.Enrich(result.LogEvent, null!); + enricher.Enrich(result.Document); } return result; } -} \ No newline at end of file +} diff --git a/src/SeqCli/Ingestion/ILogEventReader.cs b/src/SeqCli/Ingestion/IEventReader.cs similarity index 79% rename from src/SeqCli/Ingestion/ILogEventReader.cs rename to src/SeqCli/Ingestion/IEventReader.cs index a92b09b9..0ca24530 100644 --- a/src/SeqCli/Ingestion/ILogEventReader.cs +++ b/src/SeqCli/Ingestion/IEventReader.cs @@ -2,7 +2,7 @@ namespace SeqCli.Ingestion; -interface ILogEventReader +interface IEventReader { Task TryReadAsync(); } \ No newline at end of file diff --git a/src/SeqCli/Ingestion/JsonEventReader.cs b/src/SeqCli/Ingestion/JsonEventReader.cs new file mode 100644 index 00000000..689710bd --- /dev/null +++ b/src/SeqCli/Ingestion/JsonEventReader.cs @@ -0,0 +1,61 @@ +// Copyright © Datalust and contributors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.IO; +using System.Text.Json.Nodes; +using System.Threading.Tasks; +using SeqCli.PlainText.Framing; +using Superpower; +using Superpower.Model; + +namespace SeqCli.Ingestion; + +class JsonEventReader : IEventReader +{ + static readonly TimeSpan TrailingLineArrivalDeadline = TimeSpan.FromMilliseconds(10); + + readonly FrameReader _reader; + + public JsonEventReader(TextReader input) + { + _reader = new FrameReader( + input ?? throw new ArgumentNullException(nameof(input)), + Parse.Return(TextSpan.None), + TrailingLineArrivalDeadline); + } + + public async Task TryReadAsync() + { + var frame = await _reader.TryReadAsync(); + if (!frame.HasValue) + return new ReadResult(null, frame.IsAtEnd); + + if (frame.IsOrphan) + throw new InvalidDataException($"A line arrived late or could not be parsed: `{frame.Value.Trim()}`."); + + return new ReadResult(ReadFromJson(frame.Value), frame.IsAtEnd); + } + + static JsonObject ReadFromJson(string json) + { + if (JsonNode.Parse(json) is not JsonObject eventJson) + throw new InvalidDataException($"The line is not a JSON object: `{json.Trim()}`."); + + if (!eventJson.ContainsKey("@t")) + eventJson["@t"] = DateTime.UtcNow; + + return eventJson; + } +} diff --git a/src/SeqCli/Ingestion/JsonLogEventReader.cs b/src/SeqCli/Ingestion/JsonLogEventReader.cs deleted file mode 100644 index 719da10c..00000000 --- a/src/SeqCli/Ingestion/JsonLogEventReader.cs +++ /dev/null @@ -1,94 +0,0 @@ -// Copyright © Datalust and contributors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -using System; -using System.Globalization; -using System.IO; -using System.Threading.Tasks; -using Newtonsoft.Json; -using Newtonsoft.Json.Linq; -using SeqCli.Mapping; -using SeqCli.PlainText.Framing; -using Serilog.Events; -using Serilog.Formatting.Compact.Reader; -using Superpower; -using Superpower.Model; - -namespace SeqCli.Ingestion; - -class JsonLogEventReader : ILogEventReader -{ - static readonly TimeSpan TrailingLineArrivalDeadline = TimeSpan.FromMilliseconds(10); - static readonly JsonSerializer _serializer = JsonSerializer.Create(new JsonSerializerSettings - { - DateParseHandling = DateParseHandling.None, - Culture = CultureInfo.InvariantCulture - }); - - readonly FrameReader _reader; - - public JsonLogEventReader(TextReader input) - { - _reader = new FrameReader( - input ?? throw new ArgumentNullException(nameof(input)), - Parse.Return(TextSpan.None), - TrailingLineArrivalDeadline); - } - - public async Task TryReadAsync() - { - var frame = await _reader.TryReadAsync(); - if (!frame.HasValue) - return new ReadResult(null, frame.IsAtEnd); - - if (frame.IsOrphan) - throw new InvalidDataException($"A line arrived late or could not be parsed: `{frame.Value.Trim()}`."); - - var frameValue = new JsonTextReader(new StringReader(frame.Value)); - if (!(_serializer.Deserialize(frameValue) is JObject jobject)) - throw new InvalidDataException($"The line is not a JSON object: `{frame.Value.Trim()}`."); - - var evt = ReadFromJObject(jobject); - return new ReadResult(evt, frame.IsAtEnd); - } - - public static LogEvent ReadFromJson(string json) - { - var frameValue = new JsonTextReader(new StringReader(json)); - if (_serializer.Deserialize(frameValue) is not JObject jObject) - throw new InvalidDataException($"The line is not a JSON object: `{json.Trim()}`."); - - return ReadFromJObject(jObject); - } - - static LogEvent ReadFromJObject(JObject jObject) - { - if (!jObject.TryGetValue("@t", out _)) - jObject.Add("@t", new JValue(DateTime.UtcNow.ToString("O"))); - - if (jObject.TryGetValue("@l", out var levelToken)) - { - var originalLevel = levelToken.Value()!; - jObject.Remove("@l"); - - var serilogLevel = LevelMapping.ToSerilogLevel(originalLevel); - if (serilogLevel != LogEventLevel.Information) - jObject.Add("@l", new JValue(serilogLevel.ToString())); - - jObject.Add(LevelMapping.SurrogateLevelProperty, originalLevel); - } - - return LogEventReader.ReadFromJObject(jObject); - } -} \ No newline at end of file diff --git a/src/SeqCli/Ingestion/LogShipper.cs b/src/SeqCli/Ingestion/LogShipper.cs index f0a19741..313d6ceb 100644 --- a/src/SeqCli/Ingestion/LogShipper.cs +++ b/src/SeqCli/Ingestion/LogShipper.cs @@ -19,22 +19,17 @@ using System.Net.Http; using System.Net.Http.Headers; using System.Text; +using System.Text.Json.Nodes; using System.Threading; using System.Threading.Tasks; -using Newtonsoft.Json; using Seq.Api; using SeqCli.Api; -using SeqCli.Output; using Serilog; -using Serilog.Events; -using Serilog.Formatting; namespace SeqCli.Ingestion; static class LogShipper { - static readonly ITextFormatter JsonFormatter = TextFormatters.Json(null); - public static async Task ShipBufferAsync( SeqConnection connection, string? apiKey, @@ -49,7 +44,7 @@ public static async Task ShipBufferAsync( ContentType = new MediaTypeHeaderValue(ApiConstants.ClefMediaType, "utf-8") } }; - + var retries = 0; while (true) { @@ -87,22 +82,22 @@ public static async Task ShipBufferAsync( { sendFailureLog.Error(ex, "Failed to ship a batch"); } - + var millisecondsDelay = (int)Math.Min(Math.Pow(2, retries) * 2000, 60000); sendFailureLog.Information("Backing off connection schedule; will retry in {MillisecondsDelay}", millisecondsDelay); await Task.Delay(millisecondsDelay, cancellationToken); retries += 1; } } - + public static async Task ShipEventsAsync( SeqConnection connection, string? apiKey, - ILogEventReader reader, + IEventReader reader, InvalidDataHandling invalidDataHandling, SendFailureHandling sendFailureHandling, int batchSize, - Func? filter, + Func? filter, CancellationToken cancellationToken) { const int maxEmptyBatchWaitMS = 2000; @@ -116,10 +111,10 @@ public static async Task ShipEventsAsync( var statusCode = await SendBatchAsync( connection, apiKey, - batch.LogEvents, + batch.Documents, sendFailureHandling != SendFailureHandling.Ignore ? Log.Logger : null, cancellationToken); - + sendSucceeded = (int)statusCode is >= 200 and < 300; } catch (Exception ex) @@ -136,7 +131,7 @@ public static async Task ShipEventsAsync( if (sendFailureHandling == SendFailureHandling.Retry) { var millisecondsDelay = (int)Math.Min(Math.Pow(2, retries) * 2000, 60000); - await Task.Delay(millisecondsDelay); + await Task.Delay(millisecondsDelay, cancellationToken); retries += 1; continue; } @@ -146,7 +141,7 @@ public static async Task ShipEventsAsync( if (batch.IsLast) break; - + batch = await ReadBatchAsync(reader, filter, batchSize, invalidDataHandling, maxEmptyBatchWaitMS); } @@ -154,15 +149,15 @@ public static async Task ShipEventsAsync( } static async Task ReadBatchAsync( - ILogEventReader reader, - Func? filter, + IEventReader reader, + Func? filter, int count, InvalidDataHandling invalidDataHandling, int maxWaitMS) { - var batch = new List(); + var batch = new List(); var isLast = false; - + // Avoid consuming stacks of CPU unnecessarily when there's no work to do. We do eventually yield // an empty batch, because level switching relies on this. var totalWaitMS = 0; @@ -175,7 +170,7 @@ static async Task ReadBatchAsync( { var rr = await reader.TryReadAsync(); isLast = rr.IsAtEnd; - var evt = rr.LogEvent; + var evt = rr.Document; if (evt == null) { if (isLast || batch.Count != 0 || totalWaitMS > maxWaitMS) @@ -195,7 +190,7 @@ static async Task ReadBatchAsync( } catch (Exception ex) { - if (ex is JsonReaderException || ex is InvalidDataException) + if (ex is System.Text.Json.JsonException or InvalidDataException) { if (invalidDataHandling == InvalidDataHandling.Ignore) continue; @@ -204,14 +199,14 @@ static async Task ReadBatchAsync( throw; } - return new BatchResult(batch.ToArray(), isLast); + return new BatchResult([.. batch], isLast); } while (true); } static async Task SendBatchAsync( SeqConnection connection, string? apiKey, - IReadOnlyCollection batch, + IReadOnlyCollection batch, ILogger? sendFailureLog, CancellationToken cancellationToken) { @@ -223,7 +218,8 @@ static async Task SendBatchAsync( using (var builder = new StringWriter()) { foreach (var evt in batch) - JsonFormatter.Format(evt, builder); + // ReSharper disable once MethodHasAsyncOverload + builder.WriteLine(evt.ToJsonString()); content = new StringContent(builder.ToString(), Encoding.UTF8, ApiConstants.ClefMediaType); } @@ -247,7 +243,7 @@ static async Task SendAsync(SeqConnection connection, string? ap { try { - var error = JsonConvert.DeserializeObject(resultJson)!; + var error = Newtonsoft.Json.JsonConvert.DeserializeObject(resultJson)!; sendFailureLog.Error("Shipping failed with status code {StatusCode}: {ErrorMessage}", result.StatusCode, @@ -264,4 +260,4 @@ static async Task SendAsync(SeqConnection connection, string? ap sendFailureLog.Error("Shipping failed with status code {StatusCode} ({ReasonPhrase})", result.StatusCode, result.ReasonPhrase); return result.StatusCode; } -} \ No newline at end of file +} diff --git a/src/SeqCli/Ingestion/ReadResult.cs b/src/SeqCli/Ingestion/ReadResult.cs index 87e10076..0e074de5 100644 --- a/src/SeqCli/Ingestion/ReadResult.cs +++ b/src/SeqCli/Ingestion/ReadResult.cs @@ -1,15 +1,30 @@ -using Serilog.Events; +// Copyright © Datalust and contributors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System.Text.Json.Nodes; namespace SeqCli.Ingestion; readonly struct ReadResult { - public LogEvent? LogEvent { get; } + public JsonObject? Document { get; } + public bool IsAtEnd { get; } - public ReadResult(LogEvent? logEvent, bool isAtEnd) + public ReadResult(JsonObject? document, bool isAtEnd) { - LogEvent = logEvent; + Document = document; IsAtEnd = isAtEnd; } -} \ No newline at end of file +} diff --git a/src/SeqCli/Ingestion/StaticMessageTemplateReader.cs b/src/SeqCli/Ingestion/StaticMessageTemplateReader.cs index 973bff60..d5d62591 100644 --- a/src/SeqCli/Ingestion/StaticMessageTemplateReader.cs +++ b/src/SeqCli/Ingestion/StaticMessageTemplateReader.cs @@ -1,37 +1,29 @@ using System; -using System.Linq; using System.Threading.Tasks; -using SeqCli.Util; -using Serilog.Events; -using Serilog.Parsing; namespace SeqCli.Ingestion; -class StaticMessageTemplateReader : ILogEventReader +class StaticMessageTemplateReader : IEventReader { - readonly ILogEventReader _inner; - readonly MessageTemplate _messageTemplate; + readonly IEventReader _inner; + readonly string _messageTemplate; - public StaticMessageTemplateReader(ILogEventReader inner, string messageTemplate) + public StaticMessageTemplateReader(IEventReader inner, string messageTemplate) { _inner = inner ?? throw new ArgumentNullException(nameof(inner)); - _messageTemplate = new MessageTemplateParser().Parse(messageTemplate); + _messageTemplate = messageTemplate ?? throw new ArgumentNullException(nameof(messageTemplate)); } public async Task TryReadAsync() { var result = await _inner.TryReadAsync(); - if (result.LogEvent == null) - return result; + if (result.Document != null) + { + result.Document.Remove("@m"); + result.Document["@mt"] = _messageTemplate; + } - var evt = new LogEvent( - result.LogEvent.Timestamp, - result.LogEvent.Level, - result.LogEvent.Exception, - _messageTemplate, - result.LogEvent.Properties.Select(kv => LogEventPropertyFactory.SafeCreate(kv.Key, kv.Value))); - - return new ReadResult(evt, result.IsAtEnd); + return result; } -} \ No newline at end of file +} diff --git a/src/SeqCli/Ingestion/TraceConstants.cs b/src/SeqCli/Ingestion/TraceConstants.cs deleted file mode 100644 index 55fe7f34..00000000 --- a/src/SeqCli/Ingestion/TraceConstants.cs +++ /dev/null @@ -1,8 +0,0 @@ -namespace SeqCli.Ingestion; - -static class TraceConstants -{ - internal const string ParentSpanIdProperty = "ParentSpanId"; - - internal const string SpanStartTimestampProperty = "SpanStartTimestamp"; -} diff --git a/src/SeqCli/Mapping/LevelMapping.cs b/src/SeqCli/Mapping/LevelMapping.cs deleted file mode 100644 index ff79087b..00000000 --- a/src/SeqCli/Mapping/LevelMapping.cs +++ /dev/null @@ -1,93 +0,0 @@ -// Copyright © Datalust and contributors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -using System; -using System.Collections.Generic; -using Serilog.Events; - -namespace SeqCli.Mapping; - -public static class LevelMapping -{ - // Use a "hygienic" name for the original level value to avoid collisions - internal static readonly string SurrogateLevelProperty = $"_SeqcliOriginalLevel_{Guid.NewGuid():N}"; - - static readonly Dictionary LevelsByName = - new(StringComparer.OrdinalIgnoreCase) - { - ["t"] = ("Trace", LogEventLevel.Verbose), - ["tr"] = ("Trace", LogEventLevel.Verbose), - ["trc"] = ("Trace", LogEventLevel.Verbose), - ["trce"] = ("Trace", LogEventLevel.Verbose), - ["trace"] = ("Trace", LogEventLevel.Verbose), - ["v"] = ("Verbose", LogEventLevel.Verbose), - ["ver"] = ("Verbose", LogEventLevel.Verbose), - ["vrb"] = ("Verbose", LogEventLevel.Verbose), - ["verb"] = ("Verbose", LogEventLevel.Verbose), - ["verbose"] = ("Verbose", LogEventLevel.Verbose), - ["d"] = ("Debug", LogEventLevel.Debug), - ["de"] = ("Debug", LogEventLevel.Debug), - ["dbg"] = ("Debug", LogEventLevel.Debug), - ["deb"] = ("Debug", LogEventLevel.Debug), - ["dbug"] = ("Debug", LogEventLevel.Debug), - ["debu"] = ("Debug", LogEventLevel.Debug), - ["debug"] = ("Debug", LogEventLevel.Debug), - ["i"] = ("Information", LogEventLevel.Information), - ["in"] = ("Information", LogEventLevel.Information), - ["inf"] = ("Information", LogEventLevel.Information), - ["info"] = ("Information", LogEventLevel.Information), - ["information"] = ("Information", LogEventLevel.Information), - ["notice"] = ("Notice", LogEventLevel.Information), - ["w"] = ("Warning", LogEventLevel.Warning), - ["wa"] = ("Warning", LogEventLevel.Warning), - ["war"] = ("Warning", LogEventLevel.Warning), - ["wrn"] = ("Warning", LogEventLevel.Warning), - ["warn"] = ("Warning", LogEventLevel.Warning), - ["warning"] = ("Warning", LogEventLevel.Warning), - ["e"] = ("Error", LogEventLevel.Error), - ["er"] = ("Error", LogEventLevel.Error), - ["err"] = ("Error", LogEventLevel.Error), - ["erro"] = ("Error", LogEventLevel.Error), - ["eror"] = ("Error", LogEventLevel.Error), - ["error"] = ("Error", LogEventLevel.Error), - ["f"] = ("Fatal", LogEventLevel.Fatal), - ["fa"] = ("Fatal", LogEventLevel.Fatal), - ["ftl"] = ("Fatal", LogEventLevel.Fatal), - ["fat"] = ("Fatal", LogEventLevel.Fatal), - ["fatl"] = ("Fatal", LogEventLevel.Fatal), - ["fatal"] = ("Fatal", LogEventLevel.Fatal), - ["c"] = ("Critical", LogEventLevel.Fatal), - ["cr"] = ("Critical", LogEventLevel.Fatal), - ["crt"] = ("Critical", LogEventLevel.Fatal), - ["cri"] = ("Critical", LogEventLevel.Fatal), - ["crit"] = ("Critical", LogEventLevel.Fatal), - ["critical"] = ("Critical", LogEventLevel.Fatal), - ["emerg"] = ("Emergency", LogEventLevel.Fatal), - ["alert"] = ("Alert", LogEventLevel.Fatal), - ["panic"] = ("Panic", LogEventLevel.Fatal) - }; - - public static LogEventLevel ToSerilogLevel(string level) - { - if (string.IsNullOrEmpty(level)) - return LogEventLevel.Information; - - return LevelsByName.TryGetValue(level, out var m) ? m.Item2 : LogEventLevel.Information; - } - - public static string ToFullLevelName(string level) - { - return LevelsByName.TryGetValue(level, out var m) ? m.Item1 : level; - } -} \ No newline at end of file diff --git a/src/SeqCli/Mapping/MetricsMapping.cs b/src/SeqCli/Mapping/MetricsMapping.cs deleted file mode 100644 index fe104666..00000000 --- a/src/SeqCli/Mapping/MetricsMapping.cs +++ /dev/null @@ -1,8 +0,0 @@ -using System; - -namespace SeqCli.Mapping; - -public static class MetricsMapping -{ - internal static readonly string SurrogateDefinitionsProperty = $"_SeqcliMetricDefinitions_{Guid.NewGuid():N}"; -} diff --git a/src/SeqCli/Mcp/Tools/Search/SearchTools.cs b/src/SeqCli/Mcp/Tools/Search/SearchTools.cs index 7dc9671d..7d62d9ef 100644 --- a/src/SeqCli/Mcp/Tools/Search/SearchTools.cs +++ b/src/SeqCli/Mcp/Tools/Search/SearchTools.cs @@ -27,11 +27,10 @@ using Seq.Api.Model.Events; using Seq.Api.Model.Signals; using Seq.Syntax.Templates; -using SeqCli.Mapping; -using SeqCli.Output; +using SeqCli.Api; using SeqCli.Signals; +using SeqCli.Syntax; using Serilog; -using Serilog.Events; using NativeFormatter = SeqCli.Output.NativeFormatter; // ReSharper disable UnusedMember.Global @@ -42,8 +41,9 @@ namespace SeqCli.Mcp.Tools.Search; class SearchTools(McpSession session, SeqConnection connection) { const string ResultIdPropertyName = "__seqcli_ResultId"; - static readonly ExpressionTemplate SearchResultFormatter = new ( - $"{{{ResultIdPropertyName}}} [{{UtcDateTime(@t)}} {{{LevelMapping.SurrogateLevelProperty}}}] {{@m}}{Environment.NewLine}{{#if @x is not null}}{{Substring(ToString(@x), 0, 512)}}...{Environment.NewLine}{{#end}}" + static readonly ExpressionTemplate SearchResultFormatter = SeqSyntax.ParseTemplate( + $"{{{ResultIdPropertyName}}} [{{UtcDateTime(@Timestamp)}} {{@Level}}] {{@Message}}{Environment.NewLine}" + + $"{{#if @Exception is not null}}{{Substring(ToString(@Exception), 0, 512)}}...{Environment.NewLine}{{#end}}" ); [McpServerTool(Name = "seq_new_session", ReadOnly = true, Title = "Begin a new Search/Query Session")] @@ -181,12 +181,10 @@ public async Task SearchEventsAsync( foreach (var result in takenResults) { var resultId = session.ImportSearchResult(result); - - var serilogEvent = OutputFormat.ToSerilogEvent(result); - OutputFormat.FlattenPropertiesUsedWithDottedNames(result, serilogEvent); - serilogEvent.AddOrUpdateProperty(new LogEventProperty(ResultIdPropertyName, new ScalarValue(resultId))); - serilogEvent.AddOrUpdateProperty(new LogEventProperty(LevelMapping.SurrogateLevelProperty, new ScalarValue(result.Level ?? "Information"))); - SearchResultFormatter.Format(serilogEvent, responseText); + + var eventJson = EventEntityJson.ToEventJson(result); + eventJson[ResultIdPropertyName] = resultId; + SearchResultFormatter.Format(eventJson, responseText); } return new CallToolResult diff --git a/src/SeqCli/Output/FlareTheme.cs b/src/SeqCli/Output/FlareTheme.cs index 38d1e578..a1026fee 100644 --- a/src/SeqCli/Output/FlareTheme.cs +++ b/src/SeqCli/Output/FlareTheme.cs @@ -13,13 +13,12 @@ // limitations under the License. using System.Collections.Generic; -using System.IO; -using Serilog.Templates.Themes; +using Seq.Syntax.Templates.Themes; namespace SeqCli.Output; /// -/// Flare is Seq's embedded stream/columnar database. This theme is derived from one build originally +/// Flare is Seq's embedded stream/columnar database. This theme is derived from one built originally /// for the flaretl command-line tooling used there. /// static class FlareTheme @@ -45,26 +44,5 @@ static class FlareTheme [TemplateThemeStyle.LevelFatal] = "\e[38;5;0197m\e[48;5;0238m" }; - public static readonly TemplateTheme SeqCli = new(FlareThemeStyles); - - // `CsvWriter` implements its own theming behavior because the required APIs are not public in Serilog.Expressions. - // The best way forward for this is likely to be porting theming to Seq.Syntax, and exposing the required APIs there. - - const string AnsiStyleResetSequence = "\e[0m"; - - // The passed-in theme is ignored because SerilogExpressions themes are opaque. All formatting uses the SeqCli theme. - // ReSharper disable once UnusedParameter.Global - extension(TemplateTheme theme) - { - public void Set(TextWriter output, TemplateThemeStyle style) - { - if (FlareThemeStyles.TryGetValue(style, out var styleSequence)) - output.Write(styleSequence); - } - - public void Reset(TextWriter output) - { - output.Write(AnsiStyleResetSequence); - } - } -} \ No newline at end of file + public static readonly TemplateTheme SeqCli = new AnsiTheme(FlareThemeStyles); +} diff --git a/src/SeqCli/Output/OutputFormat.cs b/src/SeqCli/Output/OutputFormat.cs index 7b9c5ea7..ae2492fb 100644 --- a/src/SeqCli/Output/OutputFormat.cs +++ b/src/SeqCli/Output/OutputFormat.cs @@ -15,24 +15,20 @@ using System; using System.Collections; using System.Collections.Generic; -using System.Diagnostics; using System.Globalization; -using System.Linq; +using System.Text.Json.Nodes; using Newtonsoft.Json; using Newtonsoft.Json.Converters; using Newtonsoft.Json.Linq; using Seq.Api.Model; using Seq.Api.Model.Data; using Seq.Api.Model.Events; +using Seq.Syntax.Templates; +using Seq.Syntax.Templates.Encoding; +using Seq.Syntax.Templates.Themes; +using SeqCli.Api; using SeqCli.Config; using SeqCli.Csv; -using SeqCli.Mapping; -using SeqCli.Util; -using Serilog; -using Serilog.Core; -using Serilog.Events; -using Serilog.Parsing; -using Serilog.Templates.Themes; namespace SeqCli.Output; @@ -40,10 +36,10 @@ sealed class OutputFormat { // See https://no-color.org for semantics. const string NoColorEnvironmentVariable = "NO_COLOR"; - + readonly OutputSyntax _syntax; - readonly string? _plainTextTemplate; - readonly Logger _formatter; + readonly ExpressionTemplate? _eventFormatter; + readonly ExpressionTemplate _jsonValueFormatter; readonly JsonSerializer _serializer = JsonSerializer.CreateDefault(new JsonSerializerSettings { @@ -92,7 +88,6 @@ internal OutputFormat( bool allowAnsiEscapes) { _syntax = syntax; - _plainTextTemplate = plainTextTemplate; var resolvedNoColor = ResolveNoColor(noColor, forceColor, outputConfig, noColorSetInEnvironment, allowAnsiEscapes); var applyThemeToRedirectedOutput = !resolvedNoColor && (forceColor ?? outputConfig.ForceColor); @@ -102,12 +97,20 @@ internal OutputFormat( ? FlareTheme.SeqCli : null; - _formatter = CreateOutputLogger(); + _eventFormatter = Json + ? TextFormatters.Json(TemplateTheme) + : Text + ? TextFormatters.Plain(TemplateTheme, plainTextTemplate) + : null; + + _jsonValueFormatter = new ExpressionTemplate( + "{Value}" + Environment.NewLine, + encoder: TemplateTheme != null ? TemplateOutputEncoder.Ansi(TemplateTheme) : null); } static bool NoColorSetInEnvironment() => !string.IsNullOrEmpty(Environment.GetEnvironmentVariable(NoColorEnvironmentVariable)); - + internal static bool ResolveNoColor( bool? noColorFlag, bool? forceColorFlag, @@ -135,27 +138,6 @@ internal static bool ResolveNoColor( public bool RequiresRender => Native; - Logger CreateOutputLogger() - { - var outputConfiguration = new LoggerConfiguration() - .MinimumLevel.Is(LevelAlias.Minimum) - .Enrich.With(); - - if (Json) - { - outputConfiguration.WriteTo.Console(TextFormatters.Json(TemplateTheme)); - } - else if (Text) - { - outputConfiguration.WriteTo.Console(TextFormatters.Plain(TemplateTheme, _plainTextTemplate)); - } - - // The logger is not configured for Native output, which avoids it. Ideally we'll shift away from using - // Serilog here, and move Text/Json over to EventEntity-driven formatters, too. - - return outputConfiguration.CreateLogger(); - } - public void WriteEntity(Entity entity) { if (entity == null) throw new ArgumentNullException(nameof(entity)); @@ -163,18 +145,11 @@ public void WriteEntity(Entity entity) var jo = JObject.FromObject( entity, _serializer); - + if (Json) { jo.Remove("Links"); - - var writer = new LoggerConfiguration() - .Destructure.With() - .Destructure.ToMaximumDepth(10000) - .Enrich.With() - .WriteTo.Console(TextFormatters.Plain(TemplateTheme, "{@m}" + Environment.NewLine)) - .CreateLogger(); - writer.Information("{@Entity}", jo); + WriteJsonValue(ToSystemTextJson.FromNewtonsoft(jo)); } else if (Text) { @@ -190,22 +165,14 @@ public void WriteEntity(Entity entity) public void WriteObject(object value) { if (value == null) throw new ArgumentNullException(nameof(value)); - + if (Json) { var jo = value is ICollection and not (IDictionary or JToken) ? (JToken)JArray.FromObject(value, _serializer) : JObject.FromObject(value, _serializer); - // Using the same method of JSON colorization as above - - var writer = new LoggerConfiguration() - .Destructure.With() - .Destructure.ToMaximumDepth(10000) - .Enrich.With() - .WriteTo.Console(TextFormatters.Plain(TemplateTheme, "{@m}" + Environment.NewLine)) - .CreateLogger(); - writer.Information("{@Entity}", jo); + WriteJsonValue(ToSystemTextJson.FromNewtonsoft(jo)); } else if (Text) { @@ -218,6 +185,11 @@ public void WriteObject(object value) } } + void WriteJsonValue(JsonNode? value) + { + _jsonValueFormatter.Format(new JsonObject { ["Value"] = value }, Console.Out); + } + public void ListEntities(IEnumerable list) { foreach (var entity in list) @@ -225,7 +197,7 @@ public void ListEntities(IEnumerable list) WriteEntity(entity); } } - + // ReSharper disable once MemberCanBeMadeStatic.Global #pragma warning disable CA1822 public void WriteText(string? text) @@ -259,125 +231,15 @@ public void WriteEventEntity(EventEntity evt) } else { - var serilogEvent = ToSerilogEvent(evt); - - if (Text) - { - // Add flattened versions of structured properties that are referenced using dotted-name syntax in - // message templates, e.g. {user.name}. Serilog.Expressions template rendering doesn't otherwise - // support these. In text output mode, these aren't usually observable, though - // seqcli print --template="{@p}" will make them visible. - FlattenPropertiesUsedWithDottedNames(evt, serilogEvent); - } - - WriteLogEvent(serilogEvent); - } - } - - public void WriteLogEvent(LogEvent logEvent) - { - _formatter.Write(logEvent); - } - - public static LogEvent ToSerilogEvent(EventEntity evt) - { - ActivityTraceId traceId = default; - if (!string.IsNullOrWhiteSpace(evt.TraceId)) - traceId = ActivityTraceId.CreateFromString(evt.TraceId); - - ActivitySpanId spanId = default; - if (!string.IsNullOrWhiteSpace(evt.SpanId)) - spanId = ActivitySpanId.CreateFromString(evt.SpanId); - - var serilogEvent = new LogEvent( - DateTimeOffset.ParseExact(evt.Timestamp, "o", CultureInfo.InvariantCulture).ToLocalTime(), - LevelMapping.ToSerilogLevel(evt.Level), - string.IsNullOrWhiteSpace(evt.Exception) ? null : new TextException(evt.Exception), - new MessageTemplate(evt.MessageTemplateTokens.Select(ToMessageTemplateToken)), - evt.Properties - .Select(p => CreateProperty(p.Name, p.Value)), - traceId, - spanId - ); - - if (evt.Scope?.Count > 0) - serilogEvent.AddOrUpdateProperty(new("@sa", new StructureValue(evt.Scope.Select(p => CreateProperty(p.Name, p.Value))))); - - if (evt.Resource?.Count > 0) - serilogEvent.AddOrUpdateProperty(new("@ra", new StructureValue(evt.Resource.Select(p => CreateProperty(p.Name, p.Value))))); - - if (!string.IsNullOrWhiteSpace(evt.ParentId)) - serilogEvent.AddOrUpdateProperty(new("@ps", new ScalarValue(evt.ParentId))); - - if (!string.IsNullOrWhiteSpace(evt.Start)) - serilogEvent.AddOrUpdateProperty(new("@st", new ScalarValue(evt.Start))); - - if (!string.IsNullOrWhiteSpace(evt.SpanKind)) - serilogEvent.AddOrUpdateProperty(new("@sk", new ScalarValue(evt.SpanKind))); - - return serilogEvent; - } - - public static void FlattenPropertiesUsedWithDottedNames(EventEntity evt, LogEvent serilogEvent) - { - foreach (var token in evt.MessageTemplateTokens) - { - if (token.Text != null || token.PropertyName is not { } name || !name.Contains('.') || - serilogEvent.Properties.ContainsKey(name)) - { - continue; - } - - var steps = name.Split('.'); - var value = evt.Properties.FirstOrDefault(p => p.Name == steps[0])?.Value; - for (var i = 1; i < steps.Length; ++i) - { - value = (value as JObject)?.GetValue(steps[i]); - } - - if (value is JToken resolved) - { - // Existing flat-named properties, where present, win. - serilogEvent.AddPropertyIfAbsent(LogEventPropertyFactory.SafeCreate( - name, resolved is JValue scalar ? new ScalarValue(scalar.Value) : CreatePropertyValue(resolved))); - } + WriteEvent(EventEntityJson.ToEventJson(evt)); } } - static MessageTemplateToken ToMessageTemplateToken(MessageTemplateTokenPart token) - { - // Not ideal, we lose renderings, alignment etc. here. - - if (token.Text != null) - return new TextToken(token.Text); - return new PropertyToken(token.PropertyName, token.RawText ?? $"{{{token.PropertyName}}}"); - } - - static LogEventProperty CreateProperty(string name, object value) + public void WriteEvent(JsonObject eventJson) { - return LogEventPropertyFactory.SafeCreate(name, CreatePropertyValue(value)); + _eventFormatter?.Format(eventJson, Console.Out); } - internal static LogEventPropertyValue CreatePropertyValue(object value) - { - switch (value) - { - case JObject jo: - jo.TryGetValue("$typeTag", out var tt); - return new StructureValue( - jo.Properties() - .Where(kvp => kvp.Name != "$typeTag") - .Select(kvp => CreateProperty(kvp.Name, kvp.Value)), - (tt as JValue)?.Value as string); - - case JArray ja: - return new SequenceValue(ja.Select(CreatePropertyValue)); - - default: - return new ScalarValue(value); - } - } - static string Stringify(object? value) { return value switch diff --git a/src/SeqCli/Output/StripStructureTypeEnricher.cs b/src/SeqCli/Output/StripStructureTypeEnricher.cs deleted file mode 100644 index 352cd1bf..00000000 --- a/src/SeqCli/Output/StripStructureTypeEnricher.cs +++ /dev/null @@ -1,25 +0,0 @@ -using System.Linq; -using SeqCli.Util; -using Serilog.Core; -using Serilog.Data; -using Serilog.Events; - -namespace SeqCli.Output; - -public class StripStructureTypeEnricher : LogEventPropertyValueRewriter, ILogEventEnricher -{ - public void Enrich(LogEvent logEvent, ILogEventPropertyFactory propertyFactory) - { - foreach (var property in logEvent.Properties) - { - var updated = LogEventPropertyFactory.SafeCreate(property.Key, Visit(null, property.Value)); - logEvent.AddOrUpdateProperty(updated); - } - } - - protected override LogEventPropertyValue VisitStructureValue(object? state, StructureValue structure) - { - return new StructureValue(structure.Properties.Select(p => - LogEventPropertyFactory.SafeCreate(p.Name, Visit(null, p.Value)))); - } -} \ No newline at end of file diff --git a/src/SeqCli/Output/TextFormatters.cs b/src/SeqCli/Output/TextFormatters.cs index 86cfbee7..88025fdf 100644 --- a/src/SeqCli/Output/TextFormatters.cs +++ b/src/SeqCli/Output/TextFormatters.cs @@ -13,42 +13,32 @@ // limitations under the License. using System; -using SeqCli.Ingestion; -using SeqCli.Mapping; -using Serilog.Expressions; -using Serilog.Formatting; -using Serilog.Templates; -using Serilog.Templates.Themes; +using Seq.Syntax.Templates; +using Seq.Syntax.Templates.Encoding; +using Seq.Syntax.Templates.Themes; +using SeqCli.Syntax; namespace SeqCli.Output; -// This is the only usage of Serilog.Expressions remaining in seqcli; the upstream Seq.Syntax doesn't yet support -// tracing properties or theming. static class TextFormatters { - public static ITextFormatter Json(TemplateTheme? theme) => new ExpressionTemplate( - $"{{ " + - $"if {MetricsMapping.SurrogateDefinitionsProperty} is not null then " + - // Emit a metric sample - $"{{@t, @l: undefined(), @d: {MetricsMapping.SurrogateDefinitionsProperty}, ..rest()}} " + - $"else " + - // Emit a log or span - $"{{@t, @mt, @l: coalesce({LevelMapping.SurrogateLevelProperty}, if @l = 'Information' then undefined() else @l), @x, @sp, @tr, @ps: coalesce({TraceConstants.ParentSpanIdProperty}, @ps), @st: coalesce({TraceConstants.SpanStartTimestampProperty}, @st), ..rest()}} " + - $"}}" + - Environment.NewLine, - theme: theme, - // The `OutputFormat` constructor has already decided whether to colorize. - applyThemeWhenOutputIsRedirected: true - ); + /// + /// Newline-delimited CLEF output: the event JSON document is written verbatim, with theming + /// when a theme is supplied. + /// + public static ExpressionTemplate Json(TemplateTheme? theme) => new( + "{@Data}" + Environment.NewLine, + encoder: Encoder(theme)); + // Guarding on `@Elapsed` rather than the built-in `IsSpan()` shows elapsed time for any + // event carrying a span start timestamp, whether or not trace and span ids accompany it. static readonly string DefaultPlainTextOutputTemplate = - "[{@t:o} {@l:u3}] {@m}{#if IsSpan()} ({Milliseconds(Elapsed()):0.###} ms){#end}" + Environment.NewLine + "{@x}"; + "[{@Timestamp:o} {@Level:u3}] {@Message}{#if @Elapsed is not null} ({TotalMilliseconds(@Elapsed):0.###} ms){#end}" + + Environment.NewLine + "{@Exception}"; - public static ITextFormatter Plain(TemplateTheme? theme, string? outputTemplate) => new ExpressionTemplate( - outputTemplate ?? DefaultPlainTextOutputTemplate, - theme: theme, - nameResolver: new StaticMemberNameResolver(typeof(TracingFunctions)), - // The `OutputFormat` constructor has already decided whether to colorize. - applyThemeWhenOutputIsRedirected: true - ); -} \ No newline at end of file + public static ExpressionTemplate Plain(TemplateTheme? theme, string? outputTemplate) => + SeqSyntax.ParseTemplate(outputTemplate ?? DefaultPlainTextOutputTemplate, Encoder(theme)); + + static TemplateOutputEncoder? Encoder(TemplateTheme? theme) => + theme != null ? TemplateOutputEncoder.Ansi(theme) : null; +} diff --git a/src/SeqCli/Output/TraceFormatter.cs b/src/SeqCli/Output/TraceFormatter.cs index d2256e3d..a69a1324 100644 --- a/src/SeqCli/Output/TraceFormatter.cs +++ b/src/SeqCli/Output/TraceFormatter.cs @@ -14,11 +14,12 @@ using System; using System.Collections.Generic; +using System.Globalization; using System.Text; -using SeqCli.Mapping; +using System.Text.Json.Nodes; +using SeqCli.Api; +using SeqCli.Data; using SeqCli.Traces; -using SeqCli.Util; -using Serilog.Events; namespace SeqCli.Output; @@ -35,7 +36,7 @@ static class TraceFormatter public static string OutputTemplate(int columnCount) { - var template = new StringBuilder($"[{{@t:o}} {{@l:u3}}] {{{TreePrefixProperty}}}"); + var template = new StringBuilder($"[{{@Timestamp:o}} {{@Level:u3}}] {{{TreePrefixProperty}}}"); // `<> ''` is undefined, and hence falsy, when the property is missing; the guard thus // drops the column, and its trailing space, for both missing and empty values. @@ -45,22 +46,22 @@ public static string OutputTemplate(int columnCount) template.Append($"{{#if {column} <> ''}}{{{column}}} {{#end}}"); } - template.Append($"{{@m}}{{#if {ElapsedProperty} is not null}} ({{Milliseconds({ElapsedProperty}):0.###}} ms){{#end}}"); - template.Append(Environment.NewLine).Append("{@x}"); + template.Append($"{{@Message}}{{#if {ElapsedProperty} is not null}} ({{TotalMilliseconds({ElapsedProperty}):0.###}} ms){{#end}}"); + template.Append(Environment.NewLine).Append("{@Exception}"); return template.ToString(); } - public static IEnumerable ToLogEvents(IReadOnlyList roots) + public static IEnumerable ToEventJson(IReadOnlyList roots) { foreach (var root in roots) { - yield return ToLogEvent(root, root.Element.IsSpan ? "" : LogConnector); + yield return ToEventJson(root, root.Element.IsSpan ? "" : LogConnector); foreach (var descendant in WalkChildren(root, "")) yield return descendant; } } - static IEnumerable WalkChildren(TraceTreeNode parent, string indent) + static IEnumerable WalkChildren(TraceTreeNode parent, string indent) { for (var i = 0; i < parent.Children.Count; ++i) { @@ -71,40 +72,43 @@ static IEnumerable WalkChildren(TraceTreeNode parent, string indent) isLast ? LastSpanConnector : SpanConnector : LogConnector; - yield return ToLogEvent(child, indent + connector); + yield return ToEventJson(child, indent + connector); foreach (var descendant in WalkChildren(child, indent + (isLast ? Gap : Continuation))) yield return descendant; } } - static LogEvent ToLogEvent(TraceTreeNode treeNode, string treePrefix) + static JsonObject ToEventJson(TraceTreeNode treeNode, string treePrefix) { var evt = treeNode.Element; - var properties = new List + // Spans are positioned and shown at their start time. + var eventJson = new JsonObject { - new(TreePrefixProperty, new ScalarValue(treePrefix)) + ["@t"] = evt.SortKey.ToLocalTime().ToString("o", CultureInfo.InvariantCulture), + ["@mt"] = evt.MessageTemplate, + [TreePrefixProperty] = treePrefix }; - properties.AddRange(evt.TemplateProperties); + if (!string.IsNullOrEmpty(evt.Level)) + eventJson["@l"] = evt.Level; + + if (!string.IsNullOrWhiteSpace(evt.Exception)) + eventJson["@x"] = evt.Exception; + + foreach (var (name, value) in evt.TemplateProperties) + eventJson[name] = value?.DeepClone(); if (evt.Elapsed is { } elapsed) - properties.Add(new(ElapsedProperty, new ScalarValue(elapsed))); + eventJson[ElapsedProperty] = EventJsonFormat.CreateScalar(elapsed); for (var i = 0; i < evt.Columns.Count; ++i) { if (evt.Columns[i] is { } value) - properties.Add(LogEventPropertyFactory.SafeCreate( - ColumnPropertyName(i), OutputFormat.CreatePropertyValue(value))); + eventJson[ColumnPropertyName(i)] = ToSystemTextJson.FromApiValue(value); } - // Spans are positioned and shown at their start time. - return new LogEvent( - evt.SortKey.ToLocalTime(), - LevelMapping.ToSerilogLevel(evt.Level ?? ""), - string.IsNullOrWhiteSpace(evt.Exception) ? null : new TextException(evt.Exception), - evt.MessageTemplate, - properties); + return eventJson; } } diff --git a/src/SeqCli/Output/TracingFunctions.cs b/src/SeqCli/Output/TracingFunctions.cs deleted file mode 100644 index 5e2ba112..00000000 --- a/src/SeqCli/Output/TracingFunctions.cs +++ /dev/null @@ -1,55 +0,0 @@ -// Copyright © Datalust Pty Ltd -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -using System; -using System.Globalization; -using SeqCli.Ingestion; -using Serilog.Events; - -namespace SeqCli.Output; - -static class TracingFunctions -{ - public static LogEventPropertyValue? Elapsed(LogEvent logEvent) - { - if (logEvent.Properties.TryGetValue(TraceConstants.SpanStartTimestampProperty, out var sst) && - sst is ScalarValue { Value: DateTime spanStart }) - { - return new ScalarValue(logEvent.Timestamp - spanStart); - } - - if (logEvent.Properties.TryGetValue("@st", out var st) && - st is ScalarValue { Value: string spanStartIso } && - DateTimeOffset.TryParse(spanStartIso, CultureInfo.InvariantCulture, out var spanStartDto)) - { - return new ScalarValue(logEvent.Timestamp - spanStartDto); - } - - return null; - } - - public static LogEventPropertyValue? IsSpan(LogEvent logEvent) - { - return new ScalarValue(Elapsed(logEvent) != null); - } - - public static LogEventPropertyValue? Milliseconds(LogEventPropertyValue? timeSpan) - { - // Truncates instead of rounding. - if (timeSpan is ScalarValue { Value: TimeSpan ts }) - return new ScalarValue((decimal)ts.Ticks / TimeSpan.TicksPerMillisecond); - - return null; - } -} \ No newline at end of file diff --git a/src/SeqCli/PlainText/EventJsonBuilder.cs b/src/SeqCli/PlainText/EventJsonBuilder.cs new file mode 100644 index 00000000..7acddcb2 --- /dev/null +++ b/src/SeqCli/PlainText/EventJsonBuilder.cs @@ -0,0 +1,102 @@ +// Copyright © Datalust and contributors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Text.Json.Nodes; +using SeqCli.Data; +using Superpower.Model; + +namespace SeqCli.PlainText; + +/// +/// Assembles the values captured by a plain-text extraction pattern into an event JSON +/// document in Seq's emission schema. +/// +static class EventJsonBuilder +{ + public static JsonObject FromProperties(IDictionary properties, string? remainder) + { + var eventJson = new JsonObject + { + ["@t"] = GetTimestamp(properties).ToString("o", CultureInfo.InvariantCulture) + }; + + if (TryGetText(properties, ReifiedProperties.Level, out var level)) + eventJson["@l"] = level; + + if (TryGetText(properties, ReifiedProperties.Message, out var message)) + eventJson["@m"] = message; + + if (TryGetText(properties, ReifiedProperties.Exception, out var exception)) + eventJson["@x"] = exception; + + if (TryGetText(properties, ReifiedProperties.TraceId, out var traceId)) + eventJson["@tr"] = traceId; + + if (TryGetText(properties, ReifiedProperties.SpanId, out var spanId)) + eventJson["@sp"] = spanId; + + if (TryGetText(properties, ReifiedProperties.StartTimestamp, out var start)) + eventJson["@st"] = start; + + foreach (var (name, value) in properties) + { + if (!ReifiedProperties.IsReifiedProperty(name)) + eventJson[EventJsonFormat.EscapeUserPropertyName(name)] = UnwrapTextSpans(value); + } + + if (remainder != null) + eventJson[EventJsonFormat.EscapeUserPropertyName("@unmatched")] = UnwrapTextSpans(remainder); + + return eventJson; + } + + static JsonNode? UnwrapTextSpans(object? value) + { + // We should consider whether text spans might also end up in extracted dictionary or array elements, though + // I don't think they will, currently. + return value is TextSpan span + ? JsonValue.Create(span.ToStringValue()) + : EventJsonFormat.CreateScalar(value); + } + + static bool TryGetText(IDictionary properties, string name, out string text) + { + if (properties.TryGetValue(name, out var value) && value is TextSpan span) + { + text = span.ToStringValue(); + return true; + } + + text = ""; + return false; + } + + static DateTimeOffset GetTimestamp(IDictionary properties) + { + if (properties.TryGetValue(ReifiedProperties.Timestamp, out var t)) + { + if (t is TextSpan span && DateTimeOffset.TryParse(span.ToStringValue(), + CultureInfo.InvariantCulture, DateTimeStyles.AssumeLocal, out var ts)) + return ts; + + if (t is DateTimeOffset dto) + return dto; + } + + return DateTimeOffset.Now; + } +} diff --git a/src/SeqCli/PlainText/Extraction/Matchers.cs b/src/SeqCli/PlainText/Extraction/Matchers.cs index c2326f01..fef3c2cc 100644 --- a/src/SeqCli/PlainText/Extraction/Matchers.cs +++ b/src/SeqCli/PlainText/Extraction/Matchers.cs @@ -3,11 +3,12 @@ using System.Globalization; using System.Linq; using System.Reflection; -using SeqCli.Mapping; +using SeqCli.Api; using SeqCli.PlainText.Parsers; using Superpower; using Superpower.Model; using Superpower.Parsers; +// ReSharper disable MemberCanBePrivate.Global namespace SeqCli.PlainText.Extraction; @@ -122,7 +123,7 @@ static class Matchers // Equivalent to :* at end-of-pattern public static TextParser MultiLineContent { get; } = - Span.WithAll(ch => true) + Span.WithAll(_ => true) .Select(span => (object?)span); [Matcher("n")] diff --git a/src/SeqCli/PlainText/LogEvents/LogEventBuilder.cs b/src/SeqCli/PlainText/LogEvents/LogEventBuilder.cs deleted file mode 100644 index 1362716f..00000000 --- a/src/SeqCli/PlainText/LogEvents/LogEventBuilder.cs +++ /dev/null @@ -1,131 +0,0 @@ -// Copyright © Datalust and contributors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -using System; -using System.Collections.Generic; -using System.Diagnostics; -using System.Globalization; -using System.Linq; -using SeqCli.Mapping; -using SeqCli.Util; -using Serilog.Events; -using Serilog.Parsing; -using Superpower.Model; - -namespace SeqCli.PlainText.LogEvents; - -static class LogEventBuilder -{ - public static LogEvent FromProperties(IDictionary properties, string? remainder) - { - var timestamp = GetTimestamp(properties); - var level = GetLevel(properties); - var exception = TryGetException(properties); - var messageTemplate = GetMessageTemplate(properties); - var traceId = GetTraceId(properties); - var spanId = GetSpanId(properties); - var props = GetLogEventProperties(properties, remainder); - - var fallbackMappedLevel = level != null ? LevelMapping.ToSerilogLevel(level) : LogEventLevel.Information; - properties[LevelMapping.SurrogateLevelProperty] = level; - - return new LogEvent( - timestamp, - fallbackMappedLevel, - exception, - messageTemplate, - props, - traceId ?? default, - spanId ?? default - ); - } - - static readonly MessageTemplate NoMessage = new MessageTemplateParser().Parse(""); - - static MessageTemplate GetMessageTemplate(IDictionary properties) - { - if (properties.TryGetValue(ReifiedProperties.Message, out var m) && - m is TextSpan ts) - { - var text = ts.ToStringValue(); - return new MessageTemplate([new TextToken(text)]); - } - - return NoMessage; - } - - static string? GetLevel(IDictionary properties) - { - if (properties.TryGetValue(ReifiedProperties.Level, out var l) && - l is TextSpan ts) - return ts.ToStringValue(); - - return null; - } - - static ActivityTraceId? GetTraceId(IDictionary properties) - { - if (properties.TryGetValue(ReifiedProperties.TraceId, out var tr) && - tr is TextSpan ts) - return ActivityTraceId.CreateFromString(ts.ToStringValue()); - - return null; - } - - static ActivitySpanId? GetSpanId(IDictionary properties) - { - if (properties.TryGetValue(ReifiedProperties.SpanId, out var sp) && - sp is TextSpan ts) - return ActivitySpanId.CreateFromString(ts.ToStringValue()); - - return null; - } - - static Exception? TryGetException(IDictionary properties) - { - if (properties.TryGetValue(ReifiedProperties.Exception, out var x) && - x is TextSpan ts) - return new TextOnlyException(ts.ToStringValue()); - return null; - } - - static IEnumerable GetLogEventProperties(IDictionary properties, string? remainder) - { - var payload = properties - .Where(p => !ReifiedProperties.IsReifiedProperty(p.Key)) - .Select(p => LogEventPropertyFactory.SafeCreate(p.Key, new ScalarValue(p.Value))); - - if (remainder != null) - payload = payload.Concat(new[] - { - LogEventPropertyFactory.SafeCreate("@unmatched", new ScalarValue(remainder)) - }); - return payload; - } - - static DateTimeOffset GetTimestamp(IDictionary properties) - { - if (properties.TryGetValue(ReifiedProperties.Timestamp, out var t)) - { - if (t is TextSpan span && DateTimeOffset.TryParse(span.ToStringValue(), - CultureInfo.InvariantCulture, DateTimeStyles.AssumeLocal, out var ts)) - return ts; - - if (t is DateTimeOffset dto) - return dto; - } - - return DateTimeOffset.Now; - } -} \ No newline at end of file diff --git a/src/SeqCli/PlainText/LogEvents/TextOnlyException.cs b/src/SeqCli/PlainText/LogEvents/TextOnlyException.cs deleted file mode 100644 index 614c927f..00000000 --- a/src/SeqCli/PlainText/LogEvents/TextOnlyException.cs +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright © Datalust and contributors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -using System; - -namespace SeqCli.PlainText.LogEvents; - -class TextOnlyException : Exception -{ - readonly string _toStringValue; - - public TextOnlyException(string toStringValue) - { - _toStringValue = toStringValue ?? throw new ArgumentNullException(nameof(toStringValue)); - } - - public override string ToString() - { - return _toStringValue; - } -} \ No newline at end of file diff --git a/src/SeqCli/PlainText/PlainTextLogEventReader.cs b/src/SeqCli/PlainText/PlainTextEventReader.cs similarity index 84% rename from src/SeqCli/PlainText/PlainTextLogEventReader.cs rename to src/SeqCli/PlainText/PlainTextEventReader.cs index fae2df86..21da4e89 100644 --- a/src/SeqCli/PlainText/PlainTextLogEventReader.cs +++ b/src/SeqCli/PlainText/PlainTextEventReader.cs @@ -1,23 +1,23 @@ using System; using System.IO; using System.Threading.Tasks; +using SeqCli.Data; using SeqCli.Ingestion; using SeqCli.PlainText.Extraction; using SeqCli.PlainText.Framing; -using SeqCli.PlainText.LogEvents; using SeqCli.PlainText.Parsers; using SeqCli.PlainText.Patterns; namespace SeqCli.PlainText; -class PlainTextLogEventReader : ILogEventReader +class PlainTextEventReader : IEventReader { static readonly TimeSpan TrailingLineArrivalDeadline = TimeSpan.FromMilliseconds(10); readonly NameValueExtractor _nameValueExtractor; readonly FrameReader _reader; - public PlainTextLogEventReader(TextReader input, string extractionPattern) + public PlainTextEventReader(TextReader input, string extractionPattern) { if (extractionPattern == null) throw new ArgumentNullException(nameof(extractionPattern)); _nameValueExtractor = ExtractionPatternInterpreter.CreateNameValueExtractor(ExtractionPatternParser.Parse(extractionPattern)); @@ -36,7 +36,7 @@ public async Task TryReadAsync() var (properties, remainder) = _nameValueExtractor.ExtractValues(frame.Value); - var evt = LogEventBuilder.FromProperties(properties, remainder); + var evt = EventJsonBuilder.FromProperties(properties, remainder); return new ReadResult(evt, frame.IsAtEnd); } } \ No newline at end of file diff --git a/src/SeqCli/Ingestion/BufferingSink.cs b/src/SeqCli/Sample/Ingestion/BufferingSink.cs similarity index 50% rename from src/SeqCli/Ingestion/BufferingSink.cs rename to src/SeqCli/Sample/Ingestion/BufferingSink.cs index 879b930c..59b0225d 100644 --- a/src/SeqCli/Ingestion/BufferingSink.cs +++ b/src/SeqCli/Sample/Ingestion/BufferingSink.cs @@ -1,31 +1,41 @@ -using System; +using System; using System.Collections.Concurrent; +using System.Text.Json.Nodes; using System.Threading.Tasks; +using SeqCli.Ingestion; using Serilog.Core; using Serilog.Events; -namespace SeqCli.Ingestion; +namespace SeqCli.Sample.Ingestion; -class BufferingSink: ILogEventSink, ILogEventReader, IDisposable +/// +/// Bridges the sample simulation's Serilog-based event generation into the +/// JSON-document-based shipping pipeline. +/// +class BufferingSink: ILogEventSink, IEventReader, IDisposable { - readonly ConcurrentQueue _queue = new(); + readonly ConcurrentQueue _queue = new(); const int QueueCapacity = 10000; volatile bool _disposed; - + public void Emit(LogEvent logEvent) { // No problem if this is racy - we can afford a bit of extra queue space. if (_disposed || _queue.Count > QueueCapacity) return; - - _queue.Enqueue(logEvent); + + var document = MetricsMapping.TryGetMetricSampleJson(logEvent, out var sample) + ? sample + : SimulationEvent.ToJsonObject(logEvent); + + _queue.Enqueue(document); } public Task TryReadAsync() { - if (!_queue.TryDequeue(out var logEvent)) + if (!_queue.TryDequeue(out var document)) return Task.FromResult(new ReadResult(null, _disposed)); - return Task.FromResult(new ReadResult(logEvent, _disposed)); + return Task.FromResult(new ReadResult(document, _disposed)); } public void Dispose() @@ -34,4 +44,4 @@ public void Dispose() _disposed = true; _queue.Clear(); } -} \ No newline at end of file +} diff --git a/src/SeqCli/Sample/Ingestion/MetricsMapping.cs b/src/SeqCli/Sample/Ingestion/MetricsMapping.cs new file mode 100644 index 00000000..f50da4d7 --- /dev/null +++ b/src/SeqCli/Sample/Ingestion/MetricsMapping.cs @@ -0,0 +1,58 @@ +// Copyright © Datalust and contributors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Diagnostics.CodeAnalysis; +using System.Globalization; +using System.Text.Json.Nodes; +using SeqCli.Data; +using Serilog.Events; + +namespace SeqCli.Sample.Ingestion; + +/// +/// The sample simulation generates metric samples as Serilog events carrying their metric +/// definitions in a surrogate property, because Serilog's data model has no @d +/// equivalent. Events marked this way ship as metric samples rather than logs. +/// +static class MetricsMapping +{ + // Use a "hygienic" name for the definitions property to avoid collisions. + internal static readonly string SurrogateDefinitionsProperty = $"_SeqcliMetricDefinitions_{Guid.NewGuid():N}"; + + public static bool TryGetMetricSampleJson(LogEvent logEvent, [NotNullWhen(true)] out JsonObject? sample) + { + if (!logEvent.Properties.TryGetValue(SurrogateDefinitionsProperty, out var definitions)) + { + sample = null; + return false; + } + + // Metric samples carry only a timestamp, definitions, and their dimension/value + // properties; no message or level. + sample = new JsonObject + { + ["@t"] = logEvent.Timestamp.ToString("o", CultureInfo.InvariantCulture), + ["@d"] = SimulationEvent.ToJsonNode(definitions) + }; + + foreach (var (name, value) in logEvent.Properties) + { + if (name != SurrogateDefinitionsProperty) + sample[EventJsonFormat.EscapeUserPropertyName(name)] = SimulationEvent.ToJsonNode(value); + } + + return true; + } +} diff --git a/src/SeqCli/Sample/Ingestion/SimulationEvent.cs b/src/SeqCli/Sample/Ingestion/SimulationEvent.cs new file mode 100644 index 00000000..988b4952 --- /dev/null +++ b/src/SeqCli/Sample/Ingestion/SimulationEvent.cs @@ -0,0 +1,105 @@ +// Copyright © Datalust and contributors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System.Globalization; +using System.Linq; +using System.Text.Json.Nodes; +using SeqCli.Data; +using Serilog.Events; + +namespace SeqCli.Sample.Ingestion; + +/// Used only in the Roastery simulation; no other event data should ever be processed using this type. +static class SimulationEvent +{ + const string ParentSpanIdProperty = "ParentSpanId", + SpanStartTimestampProperty = "SpanStartTimestamp"; + + public static JsonObject ToJsonObject(LogEvent logEvent) + { + var eventJson = new JsonObject + { + ["@t"] = logEvent.Timestamp.ToString("o", CultureInfo.InvariantCulture), + ["@mt"] = logEvent.MessageTemplate.Text + }; + + if (logEvent.Level != LogEventLevel.Information) + eventJson["@l"] = logEvent.Level.ToString(); + + if (logEvent.Exception != null) + eventJson["@x"] = logEvent.Exception.ToString(); + + if (logEvent.TraceId is { } traceId) + eventJson["@tr"] = traceId.ToHexString(); + + if (logEvent.SpanId is { } spanId) + eventJson["@sp"] = spanId.ToHexString(); + + foreach (var (name, value) in logEvent.Properties) + eventJson[EventJsonFormat.EscapeUserPropertyName(name)] = ToJsonNode(value); + + LiftSpanProperties(eventJson); + + return eventJson; + } + + public static JsonNode? ToJsonNode(LogEventPropertyValue value) + { + switch (value) + { + case ScalarValue scalar: + return EventJsonFormat.CreateScalar(scalar.Value); + + case SequenceValue sequence: + return new JsonArray(sequence.Elements.Select(ToJsonNode).ToArray()); + + case StructureValue structure: + { + var result = new JsonObject(); + foreach (var property in structure.Properties) + result[property.Name] = ToJsonNode(property.Value); + if (structure.TypeTag != null) + result["$type"] = structure.TypeTag; + return result; + } + + case DictionaryValue dictionary: + { + var result = new JsonObject(); + foreach (var (key, element) in dictionary.Elements) + result[key.Value?.ToString() ?? "null"] = ToJsonNode(element); + return result; + } + + default: + return EventJsonFormat.CreateScalar(value.ToString()); + } + } + + static void LiftSpanProperties(JsonObject eventJson) + { + LiftProperty(eventJson, SpanStartTimestampProperty, "@st"); + LiftProperty(eventJson, ParentSpanIdProperty, "@ps"); + } + + static void LiftProperty(JsonObject eventJson, string propertyName, string reifiedName) + { + if (eventJson.TryGetPropertyValue(propertyName, out var value)) + { + eventJson.Remove(propertyName); + if (!eventJson.ContainsKey(reifiedName)) + eventJson[reifiedName] = value; + } + } +} diff --git a/src/SeqCli/Sample/Loader/Simulation.cs b/src/SeqCli/Sample/Loader/Simulation.cs index a54f0a28..23828632 100644 --- a/src/SeqCli/Sample/Loader/Simulation.cs +++ b/src/SeqCli/Sample/Loader/Simulation.cs @@ -17,7 +17,7 @@ using Roastery.Metrics; using Seq.Api; using SeqCli.Ingestion; -using SeqCli.Mapping; +using SeqCli.Sample.Ingestion; using Serilog; namespace SeqCli.Sample.Loader; diff --git a/src/SeqCli/SeqCli.csproj b/src/SeqCli/SeqCli.csproj index f666ccdd..16f10fb8 100644 --- a/src/SeqCli/SeqCli.csproj +++ b/src/SeqCli/SeqCli.csproj @@ -31,7 +31,6 @@ - @@ -43,13 +42,10 @@ - + - - - diff --git a/src/SeqCli/Syntax/SeqCliNameResolver.cs b/src/SeqCli/Syntax/SeqCliNameResolver.cs deleted file mode 100644 index 91b4abab..00000000 --- a/src/SeqCli/Syntax/SeqCliNameResolver.cs +++ /dev/null @@ -1,20 +0,0 @@ -using System.Diagnostics.CodeAnalysis; -using Seq.Syntax.Expressions; - -namespace SeqCli.Syntax; - -class SeqCliNameResolver: NameResolver -{ - public override bool TryResolveBuiltInPropertyName(string alias, [MaybeNullWhen(false)] out string target) - { - switch (alias) - { - case "@l": - target = "coalesce(SeqCliOriginalLevel, @l)"; - return true; - default: - target = null; - return false; - } - } -} diff --git a/src/SeqCli/Syntax/SeqSyntax.cs b/src/SeqCli/Syntax/SeqSyntax.cs index 3974b4ad..4d05d078 100644 --- a/src/SeqCli/Syntax/SeqSyntax.cs +++ b/src/SeqCli/Syntax/SeqSyntax.cs @@ -1,11 +1,54 @@ -using Seq.Syntax.Expressions; +// Copyright © Datalust and contributors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Diagnostics.CodeAnalysis; +using Seq.Syntax.Expressions; +using Seq.Syntax.Templates; +using Seq.Syntax.Templates.Encoding; +using Seq.Syntax.Compatibility; namespace SeqCli.Syntax; +/// +/// Compiles the expressions and templates accepted on the command line. Uses the Seq.Syntax v1 +/// compatibility shim so that established seqcli syntax — abbreviated built-in names like +/// @l, and the Elapsed()/Milliseconds() functions — keeps working. +/// static class SeqSyntax { public static CompiledExpression CompileExpression(string expression) { - return SerilogExpression.Compile(expression, nameResolver: new SeqCliNameResolver()); + if (!TryCompileExpression(expression, out var compiled, out var error)) + throw new ArgumentException(error); + + return compiled; + } + + public static bool TryCompileExpression( + string expression, + [MaybeNullWhen(false)] out CompiledExpression result, + [MaybeNullWhen(true)] out string error) + { + return V1.TryCompileExpression(expression, formatProvider: null, null, out result, out error); + } + + public static ExpressionTemplate ParseTemplate(string template, TemplateOutputEncoder? encoder = null) + { + if (!V1.TryParseTemplate(template, culture: null, null, encoder, out var parsed, out var error)) + throw new ArgumentException(error); + + return parsed; } -} \ No newline at end of file +} diff --git a/src/SeqCli/Traces/StructuredMessage.cs b/src/SeqCli/Traces/StructuredMessage.cs index 73654f04..702c9006 100644 --- a/src/SeqCli/Traces/StructuredMessage.cs +++ b/src/SeqCli/Traces/StructuredMessage.cs @@ -14,30 +14,30 @@ using System.Collections.Generic; using System.IO; +using System.Linq; +using System.Text.Json.Nodes; using Newtonsoft.Json.Linq; -using SeqCli.Output; -using SeqCli.Util; -using Serilog.Events; -using Serilog.Parsing; +using SeqCli.Api; namespace SeqCli.Traces; static class StructuredMessage { /// - /// Reads the token array produced by the Seq `@StructuredMessage` property - /// into a Serilog message template, along with the property values needed to render it. + /// Reads the token array produced by the Seq `@StructuredMessage` property into message + /// template text, along with the property values needed to render it. Dotted hole names + /// are stored as nested structures, matching how message rendering resolves them. /// - public static (MessageTemplate Message, IReadOnlyList Properties) Read(object? structuredMessage) + public static (string MessageTemplate, JsonObject Properties) Read(object? structuredMessage) { if (structuredMessage is null or JValue { Type: JTokenType.Null }) - return (new MessageTemplate([]), []); + return ("", new JsonObject()); if (structuredMessage is not JArray tokens) throw new InvalidDataException($"Expected a structured message but found `{structuredMessage}`."); - var templateTokens = new List(); - var properties = new List(); + var templateTokens = new List<(bool IsText, string Text)>(); + var properties = new JsonObject(); var propertyNames = new HashSet(); foreach (var token in tokens) @@ -48,14 +48,14 @@ public static (MessageTemplate Message, IReadOnlyList Properti throw new InvalidDataException("A message template hole is missing its `name`."); // Currently ignores `formatted`. - templateTokens.Add(new PropertyToken(name, (hole["raw"] as JValue)?.Value as string ?? $"{{{name}}}")); + templateTokens.Add((false, (hole["raw"] as JValue)?.Value as string ?? $"{{{name}}}")); if (hole.TryGetValue("value", out var value) && propertyNames.Add(name)) - properties.Add(LogEventPropertyFactory.SafeCreate(name, CreatePropertyValue(value))); + SetPathProperty(properties, name, ToSystemTextJson.FromNewtonsoft(value)); } else if (token is JValue { Type: JTokenType.String } text) { - templateTokens.Add(new TextToken((string)text.Value!)); + templateTokens.Add((true, (string)text.Value!)); } else { @@ -65,27 +65,53 @@ public static (MessageTemplate Message, IReadOnlyList Properti TrimEnd(templateTokens); - return (new MessageTemplate(templateTokens), properties); + var templateText = string.Concat(templateTokens.Select(t => + t.IsText ? t.Text.Replace("{", "{{").Replace("}", "}}") : t.Text)); + + return (templateText, properties); + } + + // Message rendering resolves dotted hole names as paths into nested objects, so `a.b` + // becomes member `b` of object `a`. If placing a value along the path would collide with a + // non-object value, the hole is left unresolvable and renders as raw text. + static void SetPathProperty(JsonObject properties, string name, JsonNode? value) + { + var steps = name.Split('.'); + var target = properties; + for (var i = 0; i < steps.Length - 1; ++i) + { + if (target.TryGetPropertyValue(steps[i], out var next)) + { + if (next is not JsonObject nextObject) + return; + + target = nextObject; + } + else + { + var nextObject = new JsonObject(); + target[steps[i]] = nextObject; + target = nextObject; + } + } + + target[steps[^1]] = value; } - static void TrimEnd(List templateTokens) + static void TrimEnd(List<(bool IsText, string Text)> templateTokens) { - while (templateTokens.Count > 0 && templateTokens[^1] is TextToken text) + while (templateTokens.Count > 0 && templateTokens[^1] is (true, var text)) { - var trimmed = text.Text.TrimEnd(); - if (trimmed.Length == text.Text.Length) + var trimmed = text.TrimEnd(); + if (trimmed.Length == text.Length) break; templateTokens.RemoveAt(templateTokens.Count - 1); if (trimmed.Length > 0) { - templateTokens.Add(new TextToken(trimmed)); + templateTokens.Add((true, trimmed)); break; } } } - - static LogEventPropertyValue CreatePropertyValue(JToken value) => value is JValue scalar ? - new ScalarValue(scalar.Value) : - OutputFormat.CreatePropertyValue(value); } diff --git a/src/SeqCli/Traces/TraceTreeElement.cs b/src/SeqCli/Traces/TraceTreeElement.cs index 52a25a8f..df4a6756 100644 --- a/src/SeqCli/Traces/TraceTreeElement.cs +++ b/src/SeqCli/Traces/TraceTreeElement.cs @@ -14,7 +14,7 @@ using System; using System.Collections.Generic; -using Serilog.Events; +using System.Text.Json.Nodes; namespace SeqCli.Traces; @@ -22,8 +22,8 @@ record TraceTreeElement( string Id, DateTimeOffset Timestamp, string? Level, - MessageTemplate MessageTemplate, - IReadOnlyList TemplateProperties, + string MessageTemplate, + JsonObject TemplateProperties, string? Exception, string? SpanId, string? ParentId, diff --git a/src/SeqCli/Traces/TraceTreeJObjectConverter.cs b/src/SeqCli/Traces/TraceTreeJObjectConverter.cs index 4399c15b..5d170f83 100644 --- a/src/SeqCli/Traces/TraceTreeJObjectConverter.cs +++ b/src/SeqCli/Traces/TraceTreeJObjectConverter.cs @@ -15,17 +15,16 @@ using System.Collections.Generic; using System.Globalization; using System.IO; +using System.Text.Json.Nodes; using Newtonsoft.Json.Linq; -using SeqCli.Mapping; +using Seq.Syntax.Templates; using SeqCli.Output; -using Serilog.Events; -using Serilog.Formatting; namespace SeqCli.Traces; static class TraceTreeJObjectConverter { - static readonly ITextFormatter MessageFormatter = TextFormatters.Plain(theme: null, "{@m}"); + static readonly ExpressionTemplate MessageFormatter = TextFormatters.Plain(theme: null, "{@Message}"); public static JObject FromRoots(string traceId, IReadOnlyList roots, bool complete, bool includeTypeMarker, IReadOnlyList columns) { @@ -82,7 +81,7 @@ static JObject ToJson(TraceTreeNode node, bool includeTypeMarker, IReadOnlyList< json["parentSpanId"] = evt.ParentId; if (!string.IsNullOrEmpty(evt.Level)) - json["level"] = LevelMapping.ToFullLevelName(evt.Level); + json["level"] = evt.Level; if (evt.IsSpan) { @@ -131,15 +130,16 @@ static JObject ToJson(TraceTreeNode node, bool includeTypeMarker, IReadOnlyList< static string RenderMessage(TraceTreeElement evt) { - var logEvent = new LogEvent( - evt.SortKey, - LevelMapping.ToSerilogLevel(evt.Level ?? ""), - exception: null, - evt.MessageTemplate, - evt.TemplateProperties); + var eventJson = new JsonObject + { + ["@mt"] = evt.MessageTemplate + }; + + foreach (var (name, value) in evt.TemplateProperties) + eventJson[name] = value?.DeepClone(); var message = new StringWriter(); - MessageFormatter.Format(logEvent, message); + MessageFormatter.Format(eventJson, message); return message.ToString(); } } diff --git a/src/SeqCli/Util/JsonNetDestructuringPolicy.cs b/src/SeqCli/Util/JsonNetDestructuringPolicy.cs deleted file mode 100644 index 8d9bf7bc..00000000 --- a/src/SeqCli/Util/JsonNetDestructuringPolicy.cs +++ /dev/null @@ -1,91 +0,0 @@ -// Copyright 2015 Destructurama Contributors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -using System.Collections.Generic; -using System.Diagnostics.CodeAnalysis; -using System.Linq; -using Newtonsoft.Json.Linq; -using Serilog.Core; -using Serilog.Events; - -namespace SeqCli.Util; - -sealed class JsonNetDestructuringPolicy : IDestructuringPolicy -{ - public bool TryDestructure(object value, ILogEventPropertyValueFactory propertyValueFactory, [NotNullWhen(true)] out LogEventPropertyValue? result) - { - switch (value) - { - case JObject jo: - result = Destructure(jo, propertyValueFactory); - return true; - case JArray ja: - result = Destructure(ja, propertyValueFactory); - return true; - case JValue jv: - result = Destructure(jv, propertyValueFactory); - return true; - } - - result = null; - return false; - } - - static LogEventPropertyValue Destructure(JValue jv, ILogEventPropertyValueFactory propertyValueFactory) - { - return propertyValueFactory.CreatePropertyValue(jv.Value!, destructureObjects: true); - } - - static SequenceValue Destructure(JArray ja, ILogEventPropertyValueFactory propertyValueFactory) - { - var elems = ja.Select(t => propertyValueFactory.CreatePropertyValue(t, destructureObjects: true)); - return new SequenceValue(elems); - } - - static LogEventPropertyValue Destructure(JObject jo, ILogEventPropertyValueFactory propertyValueFactory) - { - string? typeTag = null; - var props = new List(jo.Count); - - foreach (var prop in jo.Properties()) - { - if (prop.Name == "$type") - { - if (prop.Value is JValue typeVal && typeVal.Value is string v) - { - typeTag = v; - continue; - } - } - else if (!LogEventProperty.IsValidName(prop.Name)) - { - return DestructureToDictionaryValue(jo, propertyValueFactory); - } - - props.Add(new LogEventProperty(prop.Name, propertyValueFactory.CreatePropertyValue(prop.Value, destructureObjects: true))); - } - - return new StructureValue(props, typeTag); - } - - static DictionaryValue DestructureToDictionaryValue(JObject jo, ILogEventPropertyValueFactory propertyValueFactory) - { - var elements = jo.Properties().Select( - prop => new KeyValuePair( - new ScalarValue(prop.Name), - propertyValueFactory.CreatePropertyValue(prop.Value, destructureObjects: true)) - ); - return new DictionaryValue(elements); - } -} \ No newline at end of file diff --git a/src/SeqCli/Util/LogEventPropertyFactory.cs b/src/SeqCli/Util/LogEventPropertyFactory.cs deleted file mode 100644 index 89c23987..00000000 --- a/src/SeqCli/Util/LogEventPropertyFactory.cs +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright © Datalust Pty Ltd and Contributors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -using System; -using Serilog.Events; - -namespace SeqCli.Util; - -static class LogEventPropertyFactory -{ - const string InvalidPropertyNameSubstitute = "(unnamed)"; - - public static LogEventProperty SafeCreate(string name, LogEventPropertyValue value) - { - if (value == null) throw new ArgumentNullException(nameof(value)); - - if (!LogEventProperty.IsValidName(name)) - name = InvalidPropertyNameSubstitute; - - return new LogEventProperty(name, value); - } -} \ No newline at end of file diff --git a/test/SeqCli.EndToEnd/Events/EventsDeleteTestCase.cs b/test/SeqCli.EndToEnd/Events/EventsDeleteTestCase.cs index 29b585c9..61d8281b 100644 --- a/test/SeqCli.EndToEnd/Events/EventsDeleteTestCase.cs +++ b/test/SeqCli.EndToEnd/Events/EventsDeleteTestCase.cs @@ -1,4 +1,3 @@ -using System; using System.IO; using System.Threading.Tasks; using Seq.Api; diff --git a/test/SeqCli.EndToEnd/Forwarder/ForwarderSimpleIngestionTestCase.cs b/test/SeqCli.EndToEnd/Forwarder/ForwarderSimpleIngestionTestCase.cs index bb199942..d90065e5 100644 --- a/test/SeqCli.EndToEnd/Forwarder/ForwarderSimpleIngestionTestCase.cs +++ b/test/SeqCli.EndToEnd/Forwarder/ForwarderSimpleIngestionTestCase.cs @@ -1,5 +1,4 @@ using System; -using System.Globalization; using System.Threading.Tasks; using Seq.Api; using SeqCli.EndToEnd.Support; diff --git a/test/SeqCli.EndToEnd/Mcp/McpMetricsBasicsTestCase.cs b/test/SeqCli.EndToEnd/Mcp/McpMetricsBasicsTestCase.cs index 77fc5f69..f1fa88fc 100644 --- a/test/SeqCli.EndToEnd/Mcp/McpMetricsBasicsTestCase.cs +++ b/test/SeqCli.EndToEnd/Mcp/McpMetricsBasicsTestCase.cs @@ -1,6 +1,5 @@ using System; using System.Collections.Generic; -using System.Text.Json.Nodes; using System.Threading.Tasks; using JetBrains.Annotations; using ModelContextProtocol.Client; diff --git a/test/SeqCli.EndToEnd/Search/SearchWithFilterTestCase.cs b/test/SeqCli.EndToEnd/Search/SearchWithFilterTestCase.cs new file mode 100644 index 00000000..0e3a2f1d --- /dev/null +++ b/test/SeqCli.EndToEnd/Search/SearchWithFilterTestCase.cs @@ -0,0 +1,35 @@ +using System; +using System.Linq; +using System.Threading.Tasks; +using Newtonsoft.Json.Linq; +using Seq.Api; +using SeqCli.EndToEnd.Support; +using Serilog; +using Xunit; + +namespace SeqCli.EndToEnd.Search; + +public class SearchWithFilterTestCase : ICliTestCase +{ + public async Task ExecuteAsync( + SeqConnection connection, + ILogger logger, + CliCommandRunner runner) + { + await DirectIngestion.IngestClef(connection, "'@mt': 'Event {N}', 'N': 1, 'Host': 'xmpweb-01.example.com'"); + await DirectIngestion.IngestClef(connection, "'@mt': 'Event {N}', 'N': 2, 'Host': 'xmpweb-02.example.com'"); + await DirectIngestion.IngestClef(connection, "'@mt': 'Event {N}', 'N': 3, 'Host': 'xmpweb-02.example.com'"); + + var exit = runner.Exec("search", "--filter=\"Host = 'xmpweb-02.example.com' and N > 2\" --count=10 --json"); + Assert.Equal(0, exit); + + var results = runner.LastRunProcess!.Output + .Split(['\r', '\n'], StringSplitOptions.RemoveEmptyEntries) + .Select(JObject.Parse) + .ToList(); + + var evt = Assert.Single(results); + Assert.Equal(3, evt["N"]!.Value()); + Assert.Equal("xmpweb-02.example.com", evt["Host"]!.Value()); + } +} diff --git a/test/SeqCli.EndToEnd/Settings/SettingBasicsTestCase.cs b/test/SeqCli.EndToEnd/Settings/SettingBasicsTestCase.cs index f387a400..1de07801 100644 --- a/test/SeqCli.EndToEnd/Settings/SettingBasicsTestCase.cs +++ b/test/SeqCli.EndToEnd/Settings/SettingBasicsTestCase.cs @@ -1,5 +1,4 @@ -using System; -using System.Threading.Tasks; +using System.Threading.Tasks; using Seq.Api; using SeqCli.EndToEnd.Support; using Serilog; diff --git a/test/SeqCli.EndToEnd/Skills/SkillsInstallTestCase.cs b/test/SeqCli.EndToEnd/Skills/SkillsInstallTestCase.cs index 88871bc1..5bca2a32 100644 --- a/test/SeqCli.EndToEnd/Skills/SkillsInstallTestCase.cs +++ b/test/SeqCli.EndToEnd/Skills/SkillsInstallTestCase.cs @@ -1,6 +1,5 @@ using System.IO; using System.Threading.Tasks; -using JetBrains.Annotations; using Seq.Api; using SeqCli.EndToEnd.Support; using Serilog; diff --git a/test/SeqCli.EndToEnd/User/UserCreateRemoveTestCase.cs b/test/SeqCli.EndToEnd/User/UserCreateRemoveTestCase.cs index ddbf95d2..25e3fb99 100644 --- a/test/SeqCli.EndToEnd/User/UserCreateRemoveTestCase.cs +++ b/test/SeqCli.EndToEnd/User/UserCreateRemoveTestCase.cs @@ -4,7 +4,6 @@ using SeqCli.EndToEnd.Support; using Serilog; using Xunit; -using System.IO; using System.Linq; namespace SeqCli.EndToEnd.User; diff --git a/test/SeqCli.Tests/Csv/CsvWriterTests.cs b/test/SeqCli.Tests/Csv/CsvWriterTests.cs index cf4dbeb9..8d46d098 100644 --- a/test/SeqCli.Tests/Csv/CsvWriterTests.cs +++ b/test/SeqCli.Tests/Csv/CsvWriterTests.cs @@ -3,7 +3,7 @@ using System.IO; using Seq.Api.Model.Data; using SeqCli.Csv; -using Serilog.Templates.Themes; +using Seq.Syntax.Templates.Themes; using Xunit; namespace SeqCli.Tests.Csv; @@ -12,8 +12,8 @@ public class CsvWriterTests { const char Escape = '\x1b'; - // `CsvWriter` writes to the console without going through Serilog's console sink, so unlike the other - // output paths it has no opportunity to suppress the theme itself. + // `CsvWriter` writes delimited output directly rather than rendering a template, so unlike the + // other output paths it applies (or omits) the theme itself. [Fact] public void QueryResultsAreNotColorizedWhenOutputIsRedirected() { diff --git a/test/SeqCli.Tests/Forwarder/Storage/BufferTests.cs b/test/SeqCli.Tests/Forwarder/Storage/BufferTests.cs index 60dee141..7e6bf24e 100644 --- a/test/SeqCli.Tests/Forwarder/Storage/BufferTests.cs +++ b/test/SeqCli.Tests/Forwarder/Storage/BufferTests.cs @@ -1,5 +1,4 @@ using System.Linq; -using SeqCli.Forwarder.Filesystem.System; using SeqCli.Forwarder.Storage; using SeqCli.Tests.Forwarder.Filesystem; using Xunit; diff --git a/test/SeqCli.Tests/Output/OutputFormatTests.cs b/test/SeqCli.Tests/Output/OutputFormatTests.cs index 4c08a64e..e323aa60 100644 --- a/test/SeqCli.Tests/Output/OutputFormatTests.cs +++ b/test/SeqCli.Tests/Output/OutputFormatTests.cs @@ -1,10 +1,10 @@ using System.IO; using Newtonsoft.Json.Linq; using Seq.Api.Model.Events; +using SeqCli.Api; using SeqCli.Config; using SeqCli.Output; using SeqCli.Tests.Support; -using Serilog.Events; using Xunit; #nullable enable @@ -128,11 +128,10 @@ static EventEntity MakeDottedHoleEvent(params (string Name, object? Value)[] pro static string RenderMessage(EventEntity evt) { - var serilogEvent = OutputFormat.ToSerilogEvent(evt); - OutputFormat.FlattenPropertiesUsedWithDottedNames(evt, serilogEvent); + var eventJson = EventEntityJson.ToEventJson(evt); var output = new StringWriter(); - TextFormatters.Plain(theme: null, "{@m}").Format(serilogEvent, output); + TextFormatters.Plain(theme: null, "{@m}").Format(eventJson, output); return output.ToString(); } @@ -145,16 +144,6 @@ public void DottedHoleNamesResolveThroughNestedStructures() Assert.Equal("Hello Barney!", RenderMessage(evt)); } - [Fact] - public void FlatPropertiesWinOverStructureTraversal() - { - var evt = MakeDottedHoleEvent( - ("user.greeting.first", "G'day"), - ("user", JObject.Parse("""{"greeting": {"first": "Hello"}, "name": "Barney"}"""))); - - Assert.Equal("G'day Barney!", RenderMessage(evt)); - } - [Fact] public void UnresolvableDottedHolesRenderAsRawText() { @@ -162,20 +151,4 @@ public void UnresolvableDottedHolesRenderAsRawText() Assert.Equal("{user.greeting.first} {user.name}!", RenderMessage(evt)); } - - [Fact] - public void ResolvedScalarsAreUnwrappedFromTheirJsonRepresentation() - { - var evt = Some.MakeEvent(e => - { - e.MessageTemplateTokens = [new MessageTemplateTokenPart { PropertyName = "order.total" }]; - e.Properties = Some.MakeProperties(("order", JObject.Parse("""{"total": 42}"""))); - }); - - var serilogEvent = OutputFormat.ToSerilogEvent(evt); - OutputFormat.FlattenPropertiesUsedWithDottedNames(evt, serilogEvent); - - var scalar = Assert.IsType(serilogEvent.Properties["order.total"]); - Assert.Equal(42L, scalar.Value); - } } diff --git a/test/SeqCli.Tests/Output/TextFormattersTests.cs b/test/SeqCli.Tests/Output/TextFormattersTests.cs index f3548e0d..340b18fc 100644 --- a/test/SeqCli.Tests/Output/TextFormattersTests.cs +++ b/test/SeqCli.Tests/Output/TextFormattersTests.cs @@ -1,11 +1,11 @@ #nullable enable using System; using System.IO; +using System.Text.Json.Nodes; +using Seq.Syntax.Templates.Themes; +using SeqCli.Api; using SeqCli.Output; using SeqCli.Tests.Support; -using Serilog.Events; -using Serilog.Parsing; -using Serilog.Templates.Themes; using Xunit; namespace SeqCli.Tests.Output; @@ -13,7 +13,7 @@ namespace SeqCli.Tests.Output; public class TextFormattersTests { const char Escape = '\x1b'; - static readonly DateTimeOffset FixedTimestamp = new(2024, 1, 1, 10, 0, 1, 250, TimeSpan.Zero); + const string FixedTimestamp = "2024-01-01T10:00:01.2500000+00:00"; [Fact] public void ThemedJsonOutputIsColorizedRegardlessOfRedirection() @@ -27,12 +27,18 @@ public void UnthemedJsonOutputIsNotColorized() Assert.DoesNotContain(Escape, RenderJson(theme: null)); } + [Fact] + public void UnthemedJsonOutputIsTheEventDocumentVerbatim() + { + Assert.Equal( + """{"@t":"2024-01-01T10:00:01.2500000+00:00","@mt":"Hello, {Name}!","Name":"world"}""" + Environment.NewLine, + RenderJson(theme: null, SomeEventJson())); + } + [Fact] public void LogEventsAreFormattedWithTheDefaultTextTemplate() { - var evt = SomeLogEvent( - level: LogEventLevel.Warning, - properties: new LogEventProperty("Name", new ScalarValue("world"))); + var evt = SomeEventJson(level: "Warning"); Assert.Equal( $"[2024-01-01T10:00:01.2500000+00:00 WRN] Hello, world!{Environment.NewLine}", @@ -42,11 +48,7 @@ public void LogEventsAreFormattedWithTheDefaultTextTemplate() [Fact] public void ExceptionsAreIncludedInTextOutput() { - var evt = SomeLogEvent( - FixedTimestamp, - LogEventLevel.Error, - new Exception("Boom!"), - new LogEventProperty("Name", new ScalarValue("world"))); + var evt = SomeEventJson(level: "Error", exception: "System.Exception: Boom!"); Assert.Equal( $"[2024-01-01T10:00:01.2500000+00:00 ERR] Hello, world!{Environment.NewLine}System.Exception: Boom!{Environment.NewLine}", @@ -54,14 +56,10 @@ public void ExceptionsAreIncludedInTextOutput() } [Fact] - public void SpanElapsedTimeIsComputedFromTheStartTimestampProperty() + public void SpanElapsedTimeIsComputedFromTheStartTimestamp() { - // Events retrieved from the Seq API carry span start timestamps in ISO-8601 `@st` properties. - var evt = SomeLogEvent(FixedTimestamp, properties: - [ - new LogEventProperty("Name", new ScalarValue("world")), - new LogEventProperty("@st", new ScalarValue("2024-01-01T10:00:00.0000000Z")) - ]); + var evt = SomeEventJson(); + evt["@st"] = "2024-01-01T10:00:00.0000000Z"; Assert.Equal( $"[2024-01-01T10:00:01.2500000+00:00 INF] Hello, world! (1250 ms){Environment.NewLine}", @@ -69,53 +67,41 @@ public void SpanElapsedTimeIsComputedFromTheStartTimestampProperty() } [Fact] - public void SpanElapsedTimeIsComputedFromTheSurrogateStartTimestampProperty() + public void ACustomOutputTemplateReplacesTheDefault() { - // Ingested spans carry a surrogate `SpanStartTimestamp` property with a `DateTime` value. - var evt = SomeLogEvent(FixedTimestamp, properties: - [ - new LogEventProperty("Name", new ScalarValue("world")), - new LogEventProperty("SpanStartTimestamp", new ScalarValue( - FixedTimestamp.UtcDateTime.AddMilliseconds(-1.5))) - ]); - Assert.Equal( - $"[2024-01-01T10:00:01.2500000+00:00 INF] Hello, world! (1.5 ms){Environment.NewLine}", - RenderText(evt)); + $"INF Hello, world!{Environment.NewLine}", + RenderText(SomeEventJson(), $"{{@l:u3}} {{@m}}{Environment.NewLine}")); } - [Fact] - public void ACustomOutputTemplateReplacesTheDefault() + static JsonObject SomeEventJson(string? level = null, string? exception = null) { - var evt = SomeLogEvent(properties: new LogEventProperty("Name", new ScalarValue("world"))); + var evt = new JsonObject + { + ["@t"] = FixedTimestamp, + ["@mt"] = "Hello, {Name}!", + ["Name"] = "world" + }; - Assert.Equal($"INF Hello, world!{Environment.NewLine}", RenderText(evt, $"{{@l:u3}} {{@m}}{Environment.NewLine}")); - } + if (level != null) + evt["@l"] = level; - static LogEvent SomeLogEvent( - DateTimeOffset? timestamp = null, - LogEventLevel level = LogEventLevel.Information, - Exception? exception = null, - params LogEventProperty[] properties) - { - return new LogEvent( - timestamp ?? FixedTimestamp, - level, - exception, - new MessageTemplateParser().Parse("Hello, {Name}!"), - properties); + if (exception != null) + evt["@x"] = exception; + + return evt; } - static string RenderText(LogEvent evt, string? outputTemplate = null) + static string RenderText(JsonObject evt, string? outputTemplate = null) { var output = new StringWriter(); TextFormatters.Plain(theme: null, outputTemplate).Format(evt, output); return output.ToString(); } - static string RenderJson(TemplateTheme? theme) + static string RenderJson(TemplateTheme? theme, JsonObject? evt = null) { - var evt = OutputFormat.ToSerilogEvent(Some.MakeEvent(e => e.Properties = [])); + evt ??= EventEntityJson.ToEventJson(Some.MakeEvent(e => e.Properties = [])); var output = new StringWriter(); TextFormatters.Json(theme).Format(evt, output); diff --git a/test/SeqCli.Tests/Output/TraceFormatterTests.cs b/test/SeqCli.Tests/Output/TraceFormatterTests.cs index fce99356..9dc3a694 100644 --- a/test/SeqCli.Tests/Output/TraceFormatterTests.cs +++ b/test/SeqCli.Tests/Output/TraceFormatterTests.cs @@ -3,10 +3,9 @@ using System.Collections.Generic; using System.IO; using System.Linq; +using System.Text.Json.Nodes; using SeqCli.Output; using SeqCli.Traces; -using Serilog.Events; -using Serilog.Parsing; using Xunit; namespace SeqCli.Tests.Output; @@ -18,14 +17,14 @@ public class TraceFormatterTests static TraceTreeElement Span(string spanId, string? parentId, double startMs = 0, double elapsedMs = 1, string? message = null, IReadOnlyList? columns = null) => new($"event-span-{spanId}", T0.AddMilliseconds(startMs + elapsedMs), null, - new MessageTemplate([new TextToken(message ?? $"span {spanId}")]), [], + message ?? $"span {spanId}", new JsonObject(), null, spanId, parentId, T0.AddMilliseconds(startMs), TimeSpan.FromMilliseconds(elapsedMs), columns ?? []); static TraceTreeElement Log(string? spanId, double timestampMs, string message = "log", string? level = null, string? exception = null) => new($"event-log-{timestampMs}-{message}", T0.AddMilliseconds(timestampMs), level, - new MessageTemplate([new TextToken(message)]), [], exception, + message, new JsonObject(), exception, spanId, null, null, null, []); static string Render(params TraceTreeElement[] events) @@ -33,8 +32,8 @@ static string Render(params TraceTreeElement[] events) var output = new StringWriter(); var formatter = TextFormatters.Plain(theme: null, TraceFormatter.OutputTemplate(events.Max(e => e.Columns.Count))); - foreach (var logEvent in TraceFormatter.ToLogEvents(TraceTreeBuilder.Build(events))) - formatter.Format(logEvent, output); + foreach (var eventJson in TraceFormatter.ToEventJson(TraceTreeBuilder.Build(events))) + formatter.Format(eventJson, output); return output.ToString(); } @@ -115,12 +114,8 @@ public void MissingAndEmptyColumnValuesLeaveNoRedundantSpace(object? first) public void TemplateHolesAreFilledFromMessageProperties() { var evt = new TraceTreeElement("event-1", T0.AddMilliseconds(1.5), null, - new MessageTemplate([ - new TextToken("GET "), - new PropertyToken("Route", "{Route}"), - new TextToken(" as "), - new PropertyToken("User", "{User}")]), - [new LogEventProperty("Route", new ScalarValue("/orders"))], + "GET {Route} as {User}", + new JsonObject { ["Route"] = "/orders" }, null, "a", null, T0, TimeSpan.FromMilliseconds(1.5), []); Assert.Equal( diff --git a/test/SeqCli.Tests/PlainText/EventJsonBuilderTests.cs b/test/SeqCli.Tests/PlainText/EventJsonBuilderTests.cs new file mode 100644 index 00000000..0e392eb9 --- /dev/null +++ b/test/SeqCli.Tests/PlainText/EventJsonBuilderTests.cs @@ -0,0 +1,61 @@ +#nullable enable +using System; +using System.Collections.Generic; +using System.Globalization; +using SeqCli.Data; +using SeqCli.PlainText; +using Superpower.Model; +using Xunit; + +namespace SeqCli.Tests.PlainText; + +public class EventJsonBuilderTests +{ + [Fact] + public void SuppliedValuesAreUsed() + { + var properties = new Dictionary + { + ["@t"] = new TextSpan("2018-02-01T13:00:00.123Z"), + ["@l"] = new TextSpan("WRN"), + ["@m"] = new TextSpan("Hello, world"), + ["@x"] = new TextSpan("EverythingFailedException"), + ["MachineName"] = new TextSpan("TP"), + ["Count"] = 42 + }; + + var remainder = "rem"; + var evt = EventJsonBuilder.FromProperties(properties, remainder); + + Assert.Equal("2018-02-01T13:00:00.1230000+00:00", + DateTimeOffset.Parse((string)evt["@t"]!, CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind) + .ToUniversalTime().ToString("o")); + Assert.Equal("Hello, world", (string?)evt["@m"]); + Assert.Equal("WRN", (string?)evt["@l"]); + Assert.Equal("EverythingFailedException", (string?)evt["@x"]); + Assert.Equal(42, (int?)evt["Count"]); + Assert.Equal("TP", (string?)evt["MachineName"]); + Assert.Equal("rem", (string?)evt["@@unmatched"]); + } + + [Fact] + public void MissingValuesAreDefaulted() + { + var evt = EventJsonBuilder.FromProperties(new Dictionary(), null); + + var timestamp = DateTimeOffset.Parse((string)evt["@t"]!, CultureInfo.InvariantCulture, + DateTimeStyles.RoundtripKind); + Assert.True(timestamp > DateTimeOffset.Now.AddSeconds(-5)); + Assert.False(evt.ContainsKey("@m")); + Assert.False(evt.ContainsKey("@l")); + Assert.False(evt.ContainsKey("@x")); + } + + [Fact] + public void DateTimeOffsetTimestampsAreAccepted() + { + var then = DateTimeOffset.Now.AddDays(-5); + var evt = EventJsonBuilder.FromProperties(new Dictionary{["@t"] = then}, null); + Assert.Equal(then.ToString("o", CultureInfo.InvariantCulture), (string?)evt["@t"]); + } +} diff --git a/test/SeqCli.Tests/PlainText/ExtractionPatternInterpreterTests.cs b/test/SeqCli.Tests/PlainText/ExtractionPatternInterpreterTests.cs index 0993251f..fa251cb9 100644 --- a/test/SeqCli.Tests/PlainText/ExtractionPatternInterpreterTests.cs +++ b/test/SeqCli.Tests/PlainText/ExtractionPatternInterpreterTests.cs @@ -1,7 +1,6 @@ using System; using System.Collections.Generic; using System.Globalization; -using SeqCli.PlainText; using SeqCli.PlainText.Extraction; using SeqCli.PlainText.Patterns; using Xunit; diff --git a/test/SeqCli.Tests/PlainText/ExtractionPatternParserTests.cs b/test/SeqCli.Tests/PlainText/ExtractionPatternParserTests.cs index 171d6aba..fac72167 100644 --- a/test/SeqCli.Tests/PlainText/ExtractionPatternParserTests.cs +++ b/test/SeqCli.Tests/PlainText/ExtractionPatternParserTests.cs @@ -1,5 +1,4 @@ -using System; -using System.Linq; +using System.Linq; using SeqCli.PlainText.Patterns; using Superpower; using Xunit; diff --git a/test/SeqCli.Tests/PlainText/LogEventBuilderTests.cs b/test/SeqCli.Tests/PlainText/LogEventBuilderTests.cs deleted file mode 100644 index 75eaf8d5..00000000 --- a/test/SeqCli.Tests/PlainText/LogEventBuilderTests.cs +++ /dev/null @@ -1,56 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using SeqCli.PlainText.LogEvents; -using Serilog.Events; -using Superpower.Model; -using Xunit; - -namespace SeqCli.Tests.PlainText; - -public class LogEventBuilderTests -{ - [Fact] - public void SuppliedValuesAreUsed() - { - var properties = new Dictionary - { - ["@t"] = new TextSpan("2018-02-01T13:00:00.123Z"), - ["@l"] = new TextSpan("WRN"), - ["@m"] = new TextSpan("Hello, world"), - ["@x"] = new TextSpan("EverythingFailedException"), - ["MachineName"] = new TextSpan("TP"), - ["Count"] = 42 - }; - - var remainder = "rem"; - var evt = LogEventBuilder.FromProperties(properties, remainder); - - Assert.Equal("2018-02-01T13:00:00.1230000+00:00", evt.Timestamp.ToString("o")); - Assert.Equal("Hello, world", evt.RenderMessage()); - Assert.Equal(LogEventLevel.Warning, evt.Level); - Assert.Equal("EverythingFailedException", evt.Exception?.ToString()); - Assert.Equal(42, ((ScalarValue)evt.Properties["Count"]).Value); - Assert.Equal("TP", ((ScalarValue)evt.Properties["MachineName"]).Value!.ToString()); - Assert.Equal("rem", ((ScalarValue)evt.Properties["@unmatched"]).Value!.ToString()); - } - - [Fact] - public void MissingValuesAreDefaulted() - { - var evt = LogEventBuilder.FromProperties(new Dictionary(), null); - - Assert.True(evt.Timestamp > DateTimeOffset.Now.AddSeconds(-5)); - Assert.Equal("", evt.RenderMessage()); - Assert.Equal(LogEventLevel.Information, evt.Level); - Assert.Null(evt.Exception); - } - - [Fact] - public void DateTimeOffsetTimestampsAreAccepted() - { - var then = DateTimeOffset.Now.AddDays(-5); - var evt = LogEventBuilder.FromProperties(new Dictionary{["@t"] = then}, null); - Assert.Equal(then, evt.Timestamp); - } -} \ No newline at end of file diff --git a/test/SeqCli.Tests/PlainText/StaticMessageTemplateReaderTests.cs b/test/SeqCli.Tests/PlainText/StaticMessageTemplateReaderTests.cs index 29f99471..9c68a1d5 100644 --- a/test/SeqCli.Tests/PlainText/StaticMessageTemplateReaderTests.cs +++ b/test/SeqCli.Tests/PlainText/StaticMessageTemplateReaderTests.cs @@ -1,4 +1,5 @@ -using System.Threading.Tasks; +#nullable enable +using System.Threading.Tasks; using SeqCli.Ingestion; using SeqCli.Tests.Support; using Xunit; @@ -10,11 +11,13 @@ public class StaticMessageTemplateReaderTests [Fact] public async Task ReaderSubstitutesMessageTemplate() { - var evt = Some.LogEvent(); + var evt = Some.EventJson(); + evt["@m"] = "A pre-rendered message"; const string mt = "This is a message template"; - var reader = new FixedLogEventReader(new ReadResult(evt, false)); + var reader = new FixedEventReader(new ReadResult(evt, false)); var wrapper = new StaticMessageTemplateReader(reader, mt); var result = await wrapper.TryReadAsync(); - Assert.Equal(mt, result.LogEvent.MessageTemplate.Text); + Assert.Equal(mt, (string?)result.Document!["@mt"]); + Assert.False(result.Document.ContainsKey("@m")); } -} \ No newline at end of file +} diff --git a/test/SeqCli.Tests/Sample/SimulationEventTests.cs b/test/SeqCli.Tests/Sample/SimulationEventTests.cs new file mode 100644 index 00000000..0c69a6e4 --- /dev/null +++ b/test/SeqCli.Tests/Sample/SimulationEventTests.cs @@ -0,0 +1,108 @@ +#nullable enable +using System; +using System.Linq; +using SeqCli.Sample.Ingestion; +using Serilog; +using Serilog.Events; +using Xunit; + +namespace SeqCli.Tests.Sample; + +public class SimulationEventTests +{ + static LogEvent CaptureEvent(Action log) + { + LogEvent? captured = null; + var logger = new LoggerConfiguration() + .MinimumLevel.Verbose() + .WriteTo.Sink(new CapturingSink(evt => captured = evt)) + .CreateLogger(); + log(logger); + return captured ?? throw new InvalidOperationException("No event was captured."); + } + + class CapturingSink(Action capture) : Serilog.Core.ILogEventSink + { + public void Emit(LogEvent logEvent) => capture(logEvent); + } + + [Fact] + public void EventFieldsMapToTheEmissionSchema() + { + var evt = CaptureEvent(log => log.Warning(new Exception("Boom!"), "Hello, {Name}!", "world")); + var eventJson = SimulationEvent.ToJsonObject(evt); + + Assert.Equal(evt.Timestamp.ToString("o"), (string?)eventJson["@t"]); + Assert.Equal("Hello, {Name}!", (string?)eventJson["@mt"]); + Assert.Equal("Warning", (string?)eventJson["@l"]); + Assert.StartsWith("System.Exception: Boom!", (string?)eventJson["@x"]); + Assert.Equal("world", (string?)eventJson["Name"]); + } + + [Fact] + public void InformationLevelsAreOmitted() + { + var evt = CaptureEvent(log => log.Information("Hello")); + + Assert.False(SimulationEvent.ToJsonObject(evt).ContainsKey("@l")); + } + + [Fact] + public void StructuredValuesSerializeAsJson() + { + var evt = CaptureEvent(log => log.Information("{@Order} {Items}", + new { Id = 7, Total = 4.5 }, new[] { "a", "b" })); + var eventJson = SimulationEvent.ToJsonObject(evt); + + Assert.Equal(7, (int?)eventJson["Order"]!["Id"]); + Assert.Equal(4.5, (double?)eventJson["Order"]!["Total"]); + Assert.Equal(new[] { "a", "b" }, eventJson["Items"]!.AsArray().Select(i => (string?)i).ToArray()); + } + + [Fact] + public void SerilogTracingSpanPropertiesAreLifted() + { + var start = DateTime.UtcNow.AddMilliseconds(-25); + var evt = CaptureEvent(log => log + .ForContext("SpanStartTimestamp", start) + .ForContext("ParentSpanId", "8899aabbccddeeff") + .Information("GET /orders")); + var eventJson = SimulationEvent.ToJsonObject(evt); + + Assert.Equal(start, eventJson["@st"]!.GetValue()); + Assert.Equal("8899aabbccddeeff", (string?)eventJson["@ps"]); + Assert.False(eventJson.ContainsKey("SpanStartTimestamp")); + Assert.False(eventJson.ContainsKey("ParentSpanId")); + } + + [Fact] + public void MetricDefinitionsProduceMetricSamples() + { + var evt = CaptureEvent(log => log + .ForContext(MetricsMapping.SurrogateDefinitionsProperty, new { roasted_kg = new { unit = "kg" } }, destructureObjects: true) + .ForContext("roasted_kg", 42.5) + .Information("Metrics sampled")); + + Assert.True(MetricsMapping.TryGetMetricSampleJson(evt, out var eventJson)); + Assert.Equal("kg", (string?)eventJson["@d"]!["roasted_kg"]!["unit"]); + Assert.Equal(42.5, (double?)eventJson["roasted_kg"]); + Assert.False(eventJson.ContainsKey("@mt")); + Assert.False(eventJson.ContainsKey("@l")); + } + + [Fact] + public void PlainEventsAreNotMetricSamples() + { + var evt = CaptureEvent(log => log.Information("Hello")); + + Assert.False(MetricsMapping.TryGetMetricSampleJson(evt, out _)); + } + + [Fact] + public void PropertyNamesBeginningWithAtAreEscaped() + { + var evt = CaptureEvent(log => log.ForContext("@evil", "value").Information("Hello")); + + Assert.Equal("value", (string?)SimulationEvent.ToJsonObject(evt)["@@evil"]); + } +} diff --git a/test/SeqCli.Tests/Support/FixedLogEventReader.cs b/test/SeqCli.Tests/Support/FixedEventReader.cs similarity index 73% rename from test/SeqCli.Tests/Support/FixedLogEventReader.cs rename to test/SeqCli.Tests/Support/FixedEventReader.cs index 538c5b65..2c29398f 100644 --- a/test/SeqCli.Tests/Support/FixedLogEventReader.cs +++ b/test/SeqCli.Tests/Support/FixedEventReader.cs @@ -3,11 +3,11 @@ namespace SeqCli.Tests.Support; -class FixedLogEventReader : ILogEventReader +class FixedEventReader : IEventReader { readonly ReadResult _result; - public FixedLogEventReader(ReadResult result) + public FixedEventReader(ReadResult result) { _result = result; } diff --git a/test/SeqCli.Tests/Support/Some.cs b/test/SeqCli.Tests/Support/Some.cs index 7ba27d31..0a2aa24e 100644 --- a/test/SeqCli.Tests/Support/Some.cs +++ b/test/SeqCli.Tests/Support/Some.cs @@ -1,11 +1,10 @@ -using System; +using System; using System.Collections.Generic; using System.Linq; using System.Security.Cryptography; +using System.Text.Json.Nodes; using Seq.Api.Model.Events; using Seq.Api.Model.Shared; -using Serilog.Events; -using Serilog.Parsing; namespace SeqCli.Tests.Support; @@ -15,14 +14,13 @@ static class Some { static readonly RandomNumberGenerator Rng = RandomNumberGenerator.Create(); - public static LogEvent LogEvent() + public static JsonObject EventJson() { - return new LogEvent( - DateTimeOffset.UtcNow, - LogEventLevel.Information, - null, - new MessageTemplateParser().Parse("Test"), - Enumerable.Empty()); + return new JsonObject + { + ["@t"] = DateTimeOffset.UtcNow.ToString("o"), + ["@mt"] = "Test" + }; } public static string String() @@ -41,7 +39,7 @@ public static byte[] Bytes(int count) Rng.GetBytes(bytes); return bytes; } - + public static EventEntity MakeEvent(Action? configure = null) { var evt = new EventEntity @@ -58,4 +56,4 @@ public static EventEntity MakeEvent(Action? configure = null) public static List MakeProperties(params (string Name, object? Value)[] items) => items.Select(i => new EventPropertyPart(i.Name, i.Value)).ToList(); -} \ No newline at end of file +} diff --git a/test/SeqCli.Tests/Syntax/AliasedExpressionParserTests.cs b/test/SeqCli.Tests/Syntax/AliasedExpressionParserTests.cs index 43cceeca..7699f513 100644 --- a/test/SeqCli.Tests/Syntax/AliasedExpressionParserTests.cs +++ b/test/SeqCli.Tests/Syntax/AliasedExpressionParserTests.cs @@ -1,4 +1,3 @@ -using System; using SeqCli.Syntax; using Xunit; diff --git a/test/SeqCli.Tests/Traces/StructuredMessageTests.cs b/test/SeqCli.Tests/Traces/StructuredMessageTests.cs index 731a0bb8..a89ed699 100644 --- a/test/SeqCli.Tests/Traces/StructuredMessageTests.cs +++ b/test/SeqCli.Tests/Traces/StructuredMessageTests.cs @@ -1,10 +1,8 @@ #nullable enable using System.IO; -using System.Linq; +using System.Text.Json.Nodes; using Newtonsoft.Json.Linq; using SeqCli.Traces; -using Serilog.Events; -using Serilog.Parsing; using Xunit; namespace SeqCli.Tests.Traces; @@ -24,8 +22,8 @@ public void MissingStructuredMessagesReadAsEmpty() { foreach (var cell in new object?[] { null, JValue.CreateNull() }) { - var (message, properties) = StructuredMessage.Read(cell); - Assert.Empty(message.Tokens); + var (mt, properties) = StructuredMessage.Read(cell); + Assert.Equal("", mt); Assert.Empty(properties); } } @@ -33,34 +31,38 @@ public void MissingStructuredMessagesReadAsEmpty() [Fact] public void TextTokensAreRead() { - var (message, properties) = StructuredMessage.Read(new JArray("Hello", ", ", "world")); + var (mt, properties) = StructuredMessage.Read(new JArray("Hello", ", ", "world")); - Assert.Equal("Hello, world", message.Text); - Assert.All(message.Tokens, token => Assert.IsType(token)); + Assert.Equal("Hello, world", mt); Assert.Empty(properties); } + [Fact] + public void LiteralBracesAreEscapedInTemplateText() + { + var (mt, _) = StructuredMessage.Read(new JArray("a {not-a-hole} b")); + Assert.Equal("a {{not-a-hole}} b", mt); + } + [Fact] public void HolesCarryRawTextAndValues() { - var (message, properties) = StructuredMessage.Read(new JArray( + var (mt, properties) = StructuredMessage.Read(new JArray( "Hello, ", Hole("Name", "{Name:x}", "World"), "!")); - Assert.Equal("Hello, {Name:x}!", message.Text); - var hole = Assert.IsType(message.Tokens.ElementAt(1)); - Assert.Equal("Name", hole.PropertyName); + Assert.Equal("Hello, {Name:x}!", mt); var property = Assert.Single(properties); - Assert.Equal("Name", property.Name); - Assert.Equal(new ScalarValue("World"), property.Value); + Assert.Equal("Name", property.Key); + Assert.Equal("World", (string?)property.Value); } [Fact] public void HolesWithoutValuesContributeNoProperties() { - var (message, properties) = StructuredMessage.Read(new JArray(Hole("Name"))); + var (mt, properties) = StructuredMessage.Read(new JArray(Hole("Name"))); - Assert.Equal("{Name}", message.Text); + Assert.Equal("{Name}", mt); Assert.Empty(properties); } @@ -74,46 +76,55 @@ public void DuplicateHolesContributeASingleProperty() } [Fact] - public void ScalarHoleValuesAreUnwrapped() + public void ScalarHoleValuesAreRead() { var (_, properties) = StructuredMessage.Read(new JArray(Hole("Count", value: 42L))); - var scalar = Assert.IsType(Assert.Single(properties).Value); - Assert.Equal(42L, scalar.Value); + Assert.Equal(42L, (long?)Assert.Single(properties).Value); } [Fact] - public void StructuredHoleValuesBecomeStructures() + public void StructuredHoleValuesBecomeObjects() { var (_, properties) = StructuredMessage.Read(new JArray( Hole("Order", value: new JObject(new JProperty("Id", 7))))); - var structure = Assert.IsType(Assert.Single(properties).Value); - Assert.Equal("Id", Assert.Single(structure.Properties).Name); + var structure = Assert.IsType(Assert.Single(properties).Value); + Assert.Equal(7, (int?)structure["Id"]); + } + + [Fact] + public void DottedHoleNamesBecomeNestedObjects() + { + var (mt, properties) = StructuredMessage.Read(new JArray( + Hole("user.name", value: "Barney"))); + + Assert.Equal("{user.name}", mt); + var user = Assert.IsType(properties["user"]); + Assert.Equal("Barney", (string?)user["name"]); } [Fact] public void TrailingWhitespaceIsTrimmed() { - var (message, _) = StructuredMessage.Read(new JArray("Hi ", "}", " \n")); + var (mt, _) = StructuredMessage.Read(new JArray("Hi ", "}", " \n")); - Assert.Equal("Hi }", message.Text); + Assert.Equal("Hi }}", mt); } [Fact] public void WhitespaceOnlyMessagesReadAsEmpty() { - var (message, _) = StructuredMessage.Read(new JArray(" ")); + var (mt, _) = StructuredMessage.Read(new JArray(" ")); - Assert.Empty(message.Tokens); + Assert.Equal("", mt); } [Fact] public void TrailingHolesAreNotTrimmed() { - var (message, _) = StructuredMessage.Read(new JArray("Took ", Hole("Elapsed"))); - - Assert.Equal("Took {Elapsed}", message.Text); + var (mt, _) = StructuredMessage.Read(new JArray("Took ", Hole("Elapsed"))); + Assert.Equal("Took {Elapsed}", mt); } [Fact] diff --git a/test/SeqCli.Tests/Traces/TraceQueryTests.cs b/test/SeqCli.Tests/Traces/TraceQueryTests.cs index fb282be5..84a5baab 100644 --- a/test/SeqCli.Tests/Traces/TraceQueryTests.cs +++ b/test/SeqCli.Tests/Traces/TraceQueryTests.cs @@ -3,7 +3,6 @@ using Newtonsoft.Json.Linq; using Seq.Api.Model.Data; using SeqCli.Traces; -using Serilog.Events; using Xunit; namespace SeqCli.Tests.Traces; @@ -85,7 +84,7 @@ public void SpanRowsAreRead() Assert.Equal("event-1", evt.Id); Assert.Equal(timestamp, evt.Timestamp); Assert.Equal("INFO", evt.Level); - Assert.Equal("Hello!", evt.MessageTemplate.Text); + Assert.Equal("Hello!", evt.MessageTemplate); Assert.Empty(evt.TemplateProperties); Assert.Null(evt.Exception); Assert.Equal("0011223344556677", evt.SpanId); @@ -157,10 +156,10 @@ public void StructuredMessageHolesBecomeTemplatePropertiesAndValues() var evt = Assert.Single(TraceQuery.ReadEvents(result, includeExceptions: true, [])); - Assert.Equal("Hello, {Name}!", evt.MessageTemplate.Text); + Assert.Equal("Hello, {Name}!", evt.MessageTemplate); var property = Assert.Single(evt.TemplateProperties); - Assert.Equal("Name", property.Name); - Assert.Equal(new ScalarValue("World"), property.Value); + Assert.Equal("Name", property.Key); + Assert.Equal("World", (string?)property.Value); } [Fact] diff --git a/test/SeqCli.Tests/Traces/TraceTreeBuilderTests.cs b/test/SeqCli.Tests/Traces/TraceTreeBuilderTests.cs index 2511109f..2f16c2f9 100644 --- a/test/SeqCli.Tests/Traces/TraceTreeBuilderTests.cs +++ b/test/SeqCli.Tests/Traces/TraceTreeBuilderTests.cs @@ -1,9 +1,8 @@ #nullable enable using System; using System.Linq; +using System.Text.Json.Nodes; using SeqCli.Traces; -using Serilog.Events; -using Serilog.Parsing; using Xunit; namespace SeqCli.Tests.Traces; @@ -14,12 +13,12 @@ public class TraceTreeBuilderTests static TraceTreeElement Span(string spanId, string? parentId, double startMs = 0, double elapsedMs = 1) => new($"event-span-{spanId}", T0.AddMilliseconds(startMs + elapsedMs), null, - new MessageTemplate([new TextToken($"span {spanId}")]), [], null, + $"span {spanId}", new JsonObject(), null, spanId, parentId, T0.AddMilliseconds(startMs), TimeSpan.FromMilliseconds(elapsedMs), []); static TraceTreeElement Log(string? spanId, double timestampMs, string message = "log") => new($"event-log-{timestampMs}-{message}", T0.AddMilliseconds(timestampMs), null, - new MessageTemplate([new TextToken(message)]), [], null, + message, new JsonObject(), null, spanId, null, null, null, []); [Fact] @@ -76,7 +75,7 @@ public void SiblingSpansAndLogsInterleaveChronologically() var root = Assert.Single(roots); Assert.Equal( ["first", "span c", "span b", "last"], - root.Children.Select(c => c.Element.MessageTemplate.Text).ToArray()); + root.Children.Select(c => c.Element.MessageTemplate).ToArray()); } [Fact] @@ -104,7 +103,7 @@ public void OrphanLogsBecomeRootsAlongsideSpans() Assert.Equal( ["span a", "first", "second", "span b"], - roots.Select(r => r.Element.MessageTemplate.Text).ToArray()); + roots.Select(r => r.Element.MessageTemplate).ToArray()); Assert.All(roots, r => Assert.Empty(r.Children)); } @@ -117,7 +116,7 @@ public void OrphanLogsWithNoRootSpanRemainAtRootLevel() ]); Assert.Equal(2, roots.Count); - Assert.Equal(["first", "second"], roots.Select(r => r.Element.MessageTemplate.Text).ToArray()); + Assert.Equal(["first", "second"], roots.Select(r => r.Element.MessageTemplate).ToArray()); } [Fact] diff --git a/test/SeqCli.Tests/Traces/TraceTreeJObjectConverterTests.cs b/test/SeqCli.Tests/Traces/TraceTreeJObjectConverterTests.cs index b0dfa1c9..6eb90582 100644 --- a/test/SeqCli.Tests/Traces/TraceTreeJObjectConverterTests.cs +++ b/test/SeqCli.Tests/Traces/TraceTreeJObjectConverterTests.cs @@ -2,10 +2,9 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Text.Json.Nodes; using Newtonsoft.Json.Linq; using SeqCli.Traces; -using Serilog.Events; -using Serilog.Parsing; using Xunit; namespace SeqCli.Tests.Traces; @@ -20,14 +19,14 @@ static TraceTreeElement Span(string spanId, string? parentId, double startMs = 0 string? message = null, string? level = null, string? exception = null, IReadOnlyList? columns = null) => new($"event-span-{spanId}", T0.AddMilliseconds(startMs + elapsedMs), level, - new MessageTemplate([new TextToken(message ?? $"span {spanId}")]), [], + message ?? $"span {spanId}", new JsonObject(), exception, spanId, parentId, T0.AddMilliseconds(startMs), TimeSpan.FromMilliseconds(elapsedMs), columns ?? []); static TraceTreeElement Log(string? spanId, double timestampMs, string message = "log", string? level = null, string? exception = null, IReadOnlyList? columns = null) => new($"event-log-{timestampMs}-{message}", T0.AddMilliseconds(timestampMs), level, - new MessageTemplate([new TextToken(message)]), [], exception, + message, new JsonObject(), exception, spanId, null, null, null, columns ?? []); static JObject ToJson(params TraceTreeElement[] events) => ToJson([], events); @@ -142,30 +141,13 @@ public void LogsWithNoCapturedEnclosingSpanBecomeOrphans() Assert.Equal("uncaptured", (string?)orphans[0]["spanId"]); Assert.Null(orphans[2]["spanId"]); } - - [Fact] - public void LevelsAreNormalizedToFullNames() - { - var document = ToJson( - Span("a", null), - Log("a", 1, level: "warn"), - Log("a", 2, level: "Nonstandard")); - - var children = (JArray)document["root"]!["children"]!; - Assert.Equal("Warning", (string?)children[0]["level"]); - Assert.Equal("Nonstandard", (string?)children[1]["level"]); - } - + [Fact] public void TemplateHolesAreFilledFromMessageProperties() { var evt = new TraceTreeElement("event-1", T0.AddMilliseconds(1.5), null, - new MessageTemplate([ - new TextToken("GET "), - new PropertyToken("Route", "{Route}"), - new TextToken(" as "), - new PropertyToken("User", "{User}")]), - [new LogEventProperty("Route", new ScalarValue("/orders"))], + "GET {Route} as {User}", + new JsonObject { ["Route"] = "/orders" }, null, "a", null, T0, TimeSpan.FromMilliseconds(1.5), []); var document = ToJson(evt);