diff --git a/Source/Libraries/Adapters/openHistorian.Adapters/LocalOutputAdapter.cs b/Source/Libraries/Adapters/openHistorian.Adapters/LocalOutputAdapter.cs index fae8b16bc4..456fc33911 100755 --- a/Source/Libraries/Adapters/openHistorian.Adapters/LocalOutputAdapter.cs +++ b/Source/Libraries/Adapters/openHistorian.Adapters/LocalOutputAdapter.cs @@ -18,6 +18,9 @@ // ---------------------------------------------------------------------------------------------------- // 07/25/2013 - J. Ritchie Carroll // Generated original version of source code. +// 09/25/2026 - J. Ritchie Carroll +// Updated to support optional point ID / time-bucket Bloom filter indexes on finalized archives +// to support optimized nearest timestamp seek filters for sparse data reads. // //****************************************************************************************************** @@ -60,1337 +63,1433 @@ using GSFDataPublisher = GSF.TimeSeries.Transport.DataPublisher; using STTPDataPublisher = sttp.DataPublisher; -namespace openHistorian.Adapters +namespace openHistorian.Adapters; + +/// +/// Represents an output adapter that archives measurements to a local archive. +/// +[Description("openHistorian 2.0 (Local): archives measurements to a local in-process openHistorian.")] +public class LocalOutputAdapter : OutputAdapterBase { + #region [ Members ] + + // Constants + + /// + /// Defines the default listening port for the historian. + /// + public const int DefaultPort = 38402; + + /// + /// Defines default value for . + /// + public const bool DefaultWatchAttachedPaths = false; + + /// + /// Defines default value for . + /// + public const bool DefaultEnablePointIDIndex = true; + + /// + /// Defines default value for . + /// + public const bool DefaultEnableTimeBucketBloomFilter = false; + + /// + /// Defines default value for . + /// + public const int DefaultBloomFilterMaximumIndexSizeMB = 64; + + /// + /// Defines default value for . + /// + public const int DefaultBloomFilterBucketDurationSeconds = 900; + + /// + /// Defines default value for . + /// + public const double DefaultBloomFilterFalsePositiveProbability = 0.01D; + + /// + /// Defines default value for . + /// + public const string DefaultDataChannel = "port=38402; interface=::0"; + + /// + /// Defines the default value for . + /// + public const double DefaultTargetFileSize = 2.0D; + + /// + /// Defines the default value for . + /// + public const double DefaultDesiredRemainingSpace = 100.0D; + + /// + /// Defines the default value for . + /// + public const int DefaultMaximumArchiveDays = 0; + + /// + /// Defines the default value for . + /// + public const string DefaultDownsamplingIntervals = ""; + + /// + /// Defines the default value for . + /// + public const bool DefaultAutoRemoveOldestFilesBeforeFull = true; + + /// + /// Defines the default value for . + /// + public const bool DefaultEnableTimeReasonabilityCheck = false; + /// - /// Represents an output adapter that archives measurements to a local archive. + /// Defines the default value for . /// - [Description("openHistorian 2.0 (Local): archives measurements to a local in-process openHistorian.")] - public class LocalOutputAdapter : OutputAdapterBase + public const double DefaultPastTimeReasonabilityLimit = 43200.0D; + + /// + /// Defines the default value for . + /// + public const double DefaultFutureTimeReasonabilityLimit = 43200.0D; + + /// + /// Defines the default value for . + /// + public const bool DefaultSwingingDoorCompressionEnabled = false; + + /// + /// Defines the default value for . + /// + public const ArchiveDirectoryMethod DefaultDirectoryNamingMode = ArchiveDirectoryMethod.YearThenMonth; + + /// + /// Defines the default value for . + /// + public const ArchiveDirectoryFillMethod DefaultDirectoryFillMethod = ArchiveDirectoryFillMethod.Sequential; + + /// + /// Defines the default value for . + /// + public const int DefaultArchiveCurtailmentInterval = Time.SecondsPerDay; + + // Fields + private HistorianIArchive m_archive; + private HistorianServerDatabaseConfig m_archiveInfo; + private string m_instanceName; + private string m_workingDirectory; + private string[] m_archiveDirectories; + private string[] m_attachedPaths; + private double m_targetFileSize; + private double m_desiredRemainingSpace; + private long m_pastTimeReasonabilityLimit; + private long m_futureTimeReasonabilityLimit; + private DataServices m_dataServices; + private ReplicationProviders m_replicationProviders; + private long m_archivedMeasurements; + private readonly HistorianKey m_key; + private readonly HistorianValue m_value; + private Dictionary m_measurements; + private Dictionary> m_compressionSettings; + private readonly Dictionary> m_swingingDoorStates; + private SortedList m_downsamplingIntervals; + private Timer m_archiveCurtailmentTimer; + private SafeFileWatcher[] m_attachedPathWatchers; + private bool m_disposed; + + #endregion + + #region [ Constructors ] + + /// + /// Initializes a new instance of the class. + /// + public LocalOutputAdapter() { - #region [ Members ] - - // Constants - - /// - /// Defines the default listening port for the historian. - /// - public const int DefaultPort = 38402; - - /// - /// Defines default value for . - /// - public const bool DefaultWatchAttachedPaths = false; - - /// - /// Defines default value for . - /// - public const string DefaultDataChannel = "port=38402; interface=::0"; - - /// - /// Defines the default value for . - /// - public const double DefaultTargetFileSize = 2.0D; - - /// - /// Defines the default value for . - /// - public const double DefaultDesiredRemainingSpace = 100.0D; - - /// - /// Defines the default value for . - /// - public const int DefaultMaximumArchiveDays = 0; - - /// - /// Defines the default value for . - /// - public const string DefaultDownsamplingIntervals = ""; - - /// - /// Defines the default value for . - /// - public const bool DefaultAutoRemoveOldestFilesBeforeFull = true; - - /// - /// Defines the default value for . - /// - public const bool DefaultEnableTimeReasonabilityCheck = false; - - /// - /// Defines the default value for . - /// - public const double DefaultPastTimeReasonabilityLimit = 43200.0D; - - /// - /// Defines the default value for . - /// - public const double DefaultFutureTimeReasonabilityLimit = 43200.0D; - - /// - /// Defines the default value for . - /// - public const bool DefaultSwingingDoorCompressionEnabled = false; - - /// - /// Defines the default value for . - /// - public const ArchiveDirectoryMethod DefaultDirectoryNamingMode = ArchiveDirectoryMethod.YearThenMonth; - - /// - /// Defines the default value for . - /// - public const ArchiveDirectoryFillMethod DefaultDirectoryFillMethod = ArchiveDirectoryFillMethod.Sequential; - - /// - /// Defines the default value for . - /// - public const int DefaultArchiveCurtailmentInterval = Time.SecondsPerDay; - - // Fields - private HistorianIArchive m_archive; - private HistorianServerDatabaseConfig m_archiveInfo; - private string m_instanceName; - private string m_workingDirectory; - private string[] m_archiveDirectories; - private string[] m_attachedPaths; - private double m_targetFileSize; - private double m_desiredRemainingSpace; - private long m_pastTimeReasonabilityLimit; - private long m_futureTimeReasonabilityLimit; - private DataServices m_dataServices; - private ReplicationProviders m_replicationProviders; - private long m_archivedMeasurements; - private readonly HistorianKey m_key; - private readonly HistorianValue m_value; - private Dictionary m_measurements; - private Dictionary> m_compressionSettings; - private readonly Dictionary> m_swingingDoorStates; - private SortedList m_downsamplingIntervals; - private Timer m_archiveCurtailmentTimer; - private SafeFileWatcher[] m_attachedPathWatchers; - private bool m_disposed; - - #endregion - - #region [ Constructors ] - - /// - /// Initializes a new instance of the class. - /// - public LocalOutputAdapter() - { - m_key = new HistorianKey(); - m_value = new HistorianValue(); - m_swingingDoorStates = new Dictionary>(); - m_downsamplingIntervals = new SortedList(); - } + m_key = new HistorianKey(); + m_value = new HistorianValue(); + m_swingingDoorStates = new Dictionary>(); + m_downsamplingIntervals = new SortedList(); + } - #endregion + #endregion - #region [ Properties ] + #region [ Properties ] - /// - /// Gets or sets instance name defined for this . - /// - [ConnectionStringParameter] - [Description("Define the instance name for the historian. Leave this value blank to default to the adapter name.")] - [DefaultValue("")] - public string InstanceName - { - get => string.IsNullOrEmpty(m_instanceName) ? Name.ToLower() : m_instanceName; - set => m_instanceName = value; - } + /// + /// Controls exact per-file point ID index generation on newly finalized archives. + /// + [ConnectionStringParameter] + [DefaultValue(DefaultEnablePointIDIndex)] + [Description("Generate exact point ID presence sidecars during rollover. Existing indexes remain readable when disabled.")] + public bool EnablePointIDIndex { get; set; } = DefaultEnablePointIDIndex; + + /// + /// Enables time-bucket Bloom indexes on newly finalized archives. + /// + [ConnectionStringParameter] + [DefaultValue(DefaultEnableTimeBucketBloomFilter)] + [Description("Generate optional time-bucket PointID Bloom sidecars during rollover. Existing indexes remain readable when disabled.")] + public bool EnableTimeBucketBloomFilter { get; set; } = DefaultEnableTimeBucketBloomFilter; + + /// + /// Maximum uncompressed Bloom index payload per archive in MB. + /// + [ConnectionStringParameter] + [DefaultValue(DefaultBloomFilterMaximumIndexSizeMB)] + [Description("Maximum uncompressed Bloom index payload per archive, 1 to 64 MB. If exceeded, omit the entire Bloom index and use ordinary reads.")] + public int BloomFilterMaximumIndexSizeMB { get; set; } = DefaultBloomFilterMaximumIndexSizeMB; + + /// + /// Gets or sets Bloom bucket width in seconds (1 through 86400). + /// + [ConnectionStringParameter] + [DefaultValue(DefaultBloomFilterBucketDurationSeconds)] + [Description("Bloom bucket duration in seconds. Smaller buckets improve gap pruning but increase index size. Range: 1 to 86400.")] + public int BloomFilterBucketDurationSeconds { get; set; } = DefaultBloomFilterBucketDurationSeconds; + + /// + /// Gets or sets the target per-bucket false-positive probability. + /// + [ConnectionStringParameter] + [DefaultValue(DefaultBloomFilterFalsePositiveProbability)] + [Description("Target per-bucket Bloom false-positive probability, e.g., 0.01 for 1%. Range: 0.0001 to 0.25. Lower values increase size and hashing work.")] + public double BloomFilterFalsePositiveProbability { get; set; } = DefaultBloomFilterFalsePositiveProbability; + + /// + /// Gets or sets instance name defined for this . + /// + [ConnectionStringParameter] + [Description("Define the instance name for the historian. Leave this value blank to default to the adapter name.")] + [DefaultValue("")] + public string InstanceName + { + get => string.IsNullOrEmpty(m_instanceName) ? Name.ToLower() : m_instanceName; + set => m_instanceName = value; + } + + /// + /// Gets or sets TCP server based connection string to use for the historian data channel. + /// + [ConnectionStringParameter] + [Description("Defines TCP server based connection string to use for the historian data channel.")] + [DefaultValue(DefaultDataChannel)] + public string DataChannel { get; set; } - /// - /// Gets or sets TCP server based connection string to use for the historian data channel. - /// - [ConnectionStringParameter] - [Description("Defines TCP server based connection string to use for the historian data channel.")] - [DefaultValue(DefaultDataChannel)] - public string DataChannel { get; set; } - - /// - /// Gets or sets the working directory which is used to write working data before it is moved into its permanent file. - /// - [ConnectionStringParameter] - [Description("Define the working directory used for working data and intermediate files and before moving data to its permanent location (see ArchiveDirectories). Leave blank to default to \".\\Archive\\\".")] - [DefaultValue("")] - public string WorkingDirectory + /// + /// Gets or sets the working directory which is used to write working data before it is moved into its permanent file. + /// + [ConnectionStringParameter] + [Description("Define the working directory used for working data and intermediate files and before moving data to its permanent location (see ArchiveDirectories). Leave blank to default to \".\\Archive\\\".")] + [DefaultValue("")] + public string WorkingDirectory + { + get => m_workingDirectory; + set { - get => m_workingDirectory; - set - { - string localPath = FilePath.GetAbsolutePath(value); + string localPath = FilePath.GetAbsolutePath(value); - if (!Directory.Exists(localPath)) + if (!Directory.Exists(localPath)) + { + try { - try - { - Directory.CreateDirectory(localPath); - } - catch (Exception ex) - { - OnProcessException(MessageLevel.Error, new InvalidOperationException($"Failed to create working directory \"{localPath}\": {ex.Message}", ex)); - } + Directory.CreateDirectory(localPath); + } + catch (Exception ex) + { + OnProcessException(MessageLevel.Error, new InvalidOperationException($"Failed to create working directory \"{localPath}\": {ex.Message}", ex)); } - - m_workingDirectory = localPath; } + + m_workingDirectory = localPath; } + } - /// - /// Gets or sets the write directories for the historian. - /// - [ConnectionStringParameter] - [Description("Define the write directories for this historian instance. Leave empty to default to WorkingDirectory. Separate multiple directories with a semi-colon.")] - [DefaultValue("")] - public string ArchiveDirectories + /// + /// Gets or sets the write directories for the historian. + /// + [ConnectionStringParameter] + [Description("Define the write directories for this historian instance. Leave empty to default to WorkingDirectory. Separate multiple directories with a semi-colon.")] + [DefaultValue("")] + public string ArchiveDirectories + { + get => m_archiveDirectories is null ? "" : string.Join(";", m_archiveDirectories); + set { - get => m_archiveDirectories is null ? "" : string.Join(";", m_archiveDirectories); - set + if (value is null) { - if (value is null) - { - m_archiveDirectories = null; - } - else + m_archiveDirectories = null; + } + else + { + List archivePaths = []; + + foreach (string archivePath in value.Split(';')) { - List archivePaths = []; + string localPath = FilePath.GetAbsolutePath(archivePath.Trim()); - foreach (string archivePath in value.Split(';')) + if (!Directory.Exists(localPath)) { - string localPath = FilePath.GetAbsolutePath(archivePath.Trim()); - - if (!Directory.Exists(localPath)) + try { - try - { - Directory.CreateDirectory(localPath); - } - catch (Exception ex) - { - OnProcessException(MessageLevel.Error, new InvalidOperationException($"Failed to create archive directory \"{localPath}\": {ex.Message}", ex)); - } + Directory.CreateDirectory(localPath); + } + catch (Exception ex) + { + OnProcessException(MessageLevel.Error, new InvalidOperationException($"Failed to create archive directory \"{localPath}\": {ex.Message}", ex)); } - - archivePaths.Add(localPath); } - m_archiveDirectories = archivePaths.ToArray(); + archivePaths.Add(localPath); } + + m_archiveDirectories = archivePaths.ToArray(); } } + } + + /// + /// Gets or sets directory naming mode for archive directory files. + /// + [ConnectionStringParameter] + [Description("Define the directory naming mode for archive directory files.")] + [DefaultValue(DefaultDirectoryNamingMode)] + public ArchiveDirectoryMethod DirectoryNamingMode { get; set; } + + /// + /// Gets or sets the method used to select a write directory when multiple archive directories are configured. + /// + /// + /// (default) fills each archive directory in order, only + /// advancing to the next when the current drive lacks space - this matches the historical behavior. + /// distributes new files across the configured directories. + /// + [ConnectionStringParameter] + [Description("Define how a write directory is selected when multiple archive directories are configured: 'Sequential' fills each directory in order until full (default), 'RoundRobin' distributes new files across directories.")] + [DefaultValue(DefaultDirectoryFillMethod)] + public ArchiveDirectoryFillMethod DirectoryFillMethod { get; set; } - /// - /// Gets or sets directory naming mode for archive directory files. - /// - [ConnectionStringParameter] - [Description("Define the directory naming mode for archive directory files.")] - [DefaultValue(DefaultDirectoryNamingMode)] - public ArchiveDirectoryMethod DirectoryNamingMode { get; set; } - - /// - /// Gets or sets the method used to select a write directory when multiple archive directories are configured. - /// - /// - /// (default) fills each archive directory in order, only - /// advancing to the next when the current drive lacks space - this matches the historical behavior. - /// distributes new files across the configured directories. - /// - [ConnectionStringParameter] - [Description("Define how a write directory is selected when multiple archive directories are configured: 'Sequential' fills each directory in order until full (default), 'RoundRobin' distributes new files across directories.")] - [DefaultValue(DefaultDirectoryFillMethod)] - public ArchiveDirectoryFillMethod DirectoryFillMethod { get; set; } - - /// - /// Gets or sets the default interval, in seconds, over which the archive curtailment will operate. Set to zero to disable. - /// - [ConnectionStringParameter] - [Description("Define the default interval, in seconds, over which the archive curtailment will operate. Set to zero to disable.")] - [DefaultValue(DefaultArchiveCurtailmentInterval)] - public int ArchiveCurtailmentInterval { get; set; } - - /// - /// Gets or sets the directories and/or individual files to attach to the historian. - /// - [ConnectionStringParameter] - [Description("Define the directories and/or individual files to attach to this historian instance. Separate multiple paths with a semi-colon.")] - [DefaultValue("")] - public string AttachedPaths + /// + /// Gets or sets the default interval, in seconds, over which the archive curtailment will operate. Set to zero to disable. + /// + [ConnectionStringParameter] + [Description("Define the default interval, in seconds, over which the archive curtailment will operate. Set to zero to disable.")] + [DefaultValue(DefaultArchiveCurtailmentInterval)] + public int ArchiveCurtailmentInterval { get; set; } + + /// + /// Gets or sets the directories and/or individual files to attach to the historian. + /// + [ConnectionStringParameter] + [Description("Define the directories and/or individual files to attach to this historian instance. Separate multiple paths with a semi-colon.")] + [DefaultValue("")] + public string AttachedPaths + { + get => m_attachedPaths is null ? "" : string.Join(";", m_attachedPaths); + set { - get => m_attachedPaths is null ? "" : string.Join(";", m_attachedPaths); - set + if (value is null) { - if (value is null) + m_attachedPaths = null; + } + else + { + List attachedPaths = []; + + foreach (string archivePath in value.Split(';')) { - m_attachedPaths = null; + string localPath = FilePath.GetAbsolutePath(archivePath); + + if (Directory.Exists(localPath) || File.Exists(localPath)) + attachedPaths.Add(localPath); + else + OnProcessException(MessageLevel.Error, new InvalidOperationException($"Failed to locate \"{localPath}\"")); } - else - { - List attachedPaths = []; - foreach (string archivePath in value.Split(';')) - { - string localPath = FilePath.GetAbsolutePath(archivePath); + m_attachedPaths = attachedPaths.ToArray(); + } + } + } - if (Directory.Exists(localPath) || File.Exists(localPath)) - attachedPaths.Add(localPath); - else - OnProcessException(MessageLevel.Error, new InvalidOperationException($"Failed to locate \"{localPath}\"")); - } + /// + /// Gets or sets the flag which determines whether to set up file watchers to monitor the attached paths. + /// + [ConnectionStringParameter] + [Description("Determines whether to set up file watchers to monitor the attached paths.")] + [DefaultValue(DefaultWatchAttachedPaths)] + public bool WatchAttachedPaths { get; set; } - m_attachedPaths = attachedPaths.ToArray(); - } - } + /// + /// Gets or sets target file size, in GigaBytes. + /// + [ConnectionStringParameter] + [Description("Define desired target file size in GigaBytes.")] + [DefaultValue(DefaultTargetFileSize)] + public double TargetFileSize + { + get => m_targetFileSize; + set + { + if (value < 0.1D || value > SI2.Tera) + throw new ArgumentOutOfRangeException(nameof(value)); + + m_targetFileSize = value; } + } - /// - /// Gets or sets the flag which determines whether to set up file watchers to monitor the attached paths. - /// - [ConnectionStringParameter] - [Description("Determines whether to set up file watchers to monitor the attached paths.")] - [DefaultValue(DefaultWatchAttachedPaths)] - public bool WatchAttachedPaths { get; set; } - - /// - /// Gets or sets target file size, in GigaBytes. - /// - [ConnectionStringParameter] - [Description("Define desired target file size in GigaBytes.")] - [DefaultValue(DefaultTargetFileSize)] - public double TargetFileSize + /// + /// Gets or sets desired remaining space, in GigaBytes. + /// + [ConnectionStringParameter] + [Description("Define desired remaining disk space in GigaBytes.")] + [DefaultValue(DefaultDesiredRemainingSpace)] + public double DesiredRemainingSpace + { + get => m_desiredRemainingSpace; + set { - get => m_targetFileSize; - set - { - if (value < 0.1D || value > SI2.Tera) - throw new ArgumentOutOfRangeException(nameof(value)); + if (value < 0.1D || value > SI2.Tera) + throw new ArgumentOutOfRangeException(nameof(value)); - m_targetFileSize = value; - } + m_desiredRemainingSpace = value; } + } - /// - /// Gets or sets desired remaining space, in GigaBytes. - /// - [ConnectionStringParameter] - [Description("Define desired remaining disk space in GigaBytes.")] - [DefaultValue(DefaultDesiredRemainingSpace)] - public double DesiredRemainingSpace + /// + /// Gets or sets the maximum number of days of data to maintain in the archive. + /// + [ConnectionStringParameter] + [Description("Define the maximum number of days of data to maintain, i.e., any archives files with data older than current date minus value will be deleted daily. Defaults to zero meaning no maximum.")] + [DefaultValue(DefaultMaximumArchiveDays)] + public int MaximumArchiveDays { get; set; } = DefaultMaximumArchiveDays; + + /// + /// Gets or sets the scheduled downsampling intervals for this archive. + /// + [ConnectionStringParameter] + [Description("Define the scheduled downsampling intervals for this archive. Format: any number of downsampling-start-days=samples-per-second separated by semi-colons, e.g., 180=10; 365=1; 650=0.1")] + [DefaultValue(DefaultDownsamplingIntervals)] + public string DownsamplingIntervals + { + get { - get => m_desiredRemainingSpace; - set + if (m_downsamplingIntervals.Count == 0) + return string.Empty; + + StringBuilder downsampling = new(); + + foreach (KeyValuePair interval in m_downsamplingIntervals) { - if (value < 0.1D || value > SI2.Tera) - throw new ArgumentOutOfRangeException(nameof(value)); + if (downsampling.Length > 0) + downsampling.Append("; "); - m_desiredRemainingSpace = value; + downsampling.Append($"{interval.Key}={interval.Value}"); } - } - /// - /// Gets or sets the maximum number of days of data to maintain in the archive. - /// - [ConnectionStringParameter] - [Description("Define the maximum number of days of data to maintain, i.e., any archives files with data older than current date minus value will be deleted daily. Defaults to zero meaning no maximum.")] - [DefaultValue(DefaultMaximumArchiveDays)] - public int MaximumArchiveDays { get; set; } = DefaultMaximumArchiveDays; - - /// - /// Gets or sets the scheduled downsampling intervals for this archive. - /// - [ConnectionStringParameter] - [Description("Define the scheduled downsampling intervals for this archive. Format: any number of downsampling-start-days=samples-per-second separated by semi-colons, e.g., 180=10; 365=1; 650=0.1")] - [DefaultValue(DefaultDownsamplingIntervals)] - public string DownsamplingIntervals + return downsampling.ToString(); + } + set { - get - { - if (m_downsamplingIntervals.Count == 0) - return string.Empty; + m_downsamplingIntervals.Clear(); - StringBuilder downsampling = new(); + if (string.IsNullOrWhiteSpace(value)) + return; - foreach (KeyValuePair interval in m_downsamplingIntervals) - { - if (downsampling.Length > 0) - downsampling.Append("; "); + foreach (string interval in value.Split(';')) + { + string[] parts = interval.Split('='); - downsampling.Append($"{interval.Key}={interval.Value}"); - } + if (parts.Length != 2) + continue; - return downsampling.ToString(); + if (int.TryParse(parts[0].Trim(), out int startDay) && startDay > 0 && double.TryParse(parts[1].Trim(), out double samplesPerSecond) && samplesPerSecond > 0.0D) + m_downsamplingIntervals[startDay] = samplesPerSecond; } - set - { - m_downsamplingIntervals.Clear(); + } + } - if (string.IsNullOrWhiteSpace(value)) - return; + /// + /// Gets or sets the flag that determines if oldest archive files should be removed before running out of archive space. + /// + [ConnectionStringParameter] + [Description("Define the flag that determines if oldest archive files should be removed before running out of archive space.")] + [DefaultValue(DefaultAutoRemoveOldestFilesBeforeFull)] + public bool AutoRemoveOldestFilesBeforeFull { get; set; } = DefaultAutoRemoveOldestFilesBeforeFull; - foreach (string interval in value.Split(';')) - { - string[] parts = interval.Split('='); + /// + /// Gets or sets flag that indicates if incoming timestamps to the historian should be validated for reasonability. + /// + [ConnectionStringParameter] + [Description("Define the flag that indicates if incoming timestamps to the historian should be validated for reasonability.")] + [DefaultValue(DefaultEnableTimeReasonabilityCheck)] + public bool EnableTimeReasonabilityCheck { get; set; } - if (parts.Length != 2) - continue; + /// + /// Gets or sets the maximum number of seconds that a past timestamp, as compared to local clock, will be considered valid. + /// + [ConnectionStringParameter] + [Description("Define the maximum number of seconds that a past timestamp, as compared to local clock, will be considered valid.")] + [DefaultValue(DefaultPastTimeReasonabilityLimit)] + public double PastTimeReasonabilityLimit + { + get => new Ticks(m_pastTimeReasonabilityLimit).ToSeconds(); + set => m_pastTimeReasonabilityLimit = Ticks.FromSeconds(Math.Abs(value)); + } - if (int.TryParse(parts[0].Trim(), out int startDay) && startDay > 0 && double.TryParse(parts[1].Trim(), out double samplesPerSecond) && samplesPerSecond > 0.0D) - m_downsamplingIntervals[startDay] = samplesPerSecond; - } - } - } + /// + /// Gets or sets the maximum number of seconds that a future timestamp, as compared to local clock, will be considered valid. + /// + [ConnectionStringParameter] + [Description("Define the maximum number of seconds that a future timestamp, as compared to local clock, will be considered valid.")] + [DefaultValue(DefaultFutureTimeReasonabilityLimit)] + public double FutureTimeReasonabilityLimit + { + get => new Ticks(m_futureTimeReasonabilityLimit).ToSeconds(); + set => m_futureTimeReasonabilityLimit = Ticks.FromSeconds(Math.Abs(value)); + } - /// - /// Gets or sets the flag that determines if oldest archive files should be removed before running out of archive space. - /// - [ConnectionStringParameter] - [Description("Define the flag that determines if oldest archive files should be removed before running out of archive space.")] - [DefaultValue(DefaultAutoRemoveOldestFilesBeforeFull)] - public bool AutoRemoveOldestFilesBeforeFull { get; set; } = DefaultAutoRemoveOldestFilesBeforeFull; - - /// - /// Gets or sets flag that indicates if incoming timestamps to the historian should be validated for reasonability. - /// - [ConnectionStringParameter] - [Description("Define the flag that indicates if incoming timestamps to the historian should be validated for reasonability.")] - [DefaultValue(DefaultEnableTimeReasonabilityCheck)] - public bool EnableTimeReasonabilityCheck { get; set; } - - /// - /// Gets or sets the maximum number of seconds that a past timestamp, as compared to local clock, will be considered valid. - /// - [ConnectionStringParameter] - [Description("Define the maximum number of seconds that a past timestamp, as compared to local clock, will be considered valid.")] - [DefaultValue(DefaultPastTimeReasonabilityLimit)] - public double PastTimeReasonabilityLimit - { - get => new Ticks(m_pastTimeReasonabilityLimit).ToSeconds(); - set => m_pastTimeReasonabilityLimit = Ticks.FromSeconds(Math.Abs(value)); - } + /// + /// Gets or sets the flag that determines if swinging door compression is enabled for this historian instance. + /// + [ConnectionStringParameter] + [Description("Define the flag that determines if swinging door compression is enabled for this historian instance.")] + [DefaultValue(DefaultSwingingDoorCompressionEnabled)] + public bool SwingingDoorCompressionEnabled { get; set; } - /// - /// Gets or sets the maximum number of seconds that a future timestamp, as compared to local clock, will be considered valid. - /// - [ConnectionStringParameter] - [Description("Define the maximum number of seconds that a future timestamp, as compared to local clock, will be considered valid.")] - [DefaultValue(DefaultFutureTimeReasonabilityLimit)] - public double FutureTimeReasonabilityLimit - { - get => new Ticks(m_futureTimeReasonabilityLimit).ToSeconds(); - set => m_futureTimeReasonabilityLimit = Ticks.FromSeconds(Math.Abs(value)); - } + /// + /// Returns a flag that determines if measurements sent to this are destined for archival. + /// + public override bool OutputIsForArchive => true; - /// - /// Gets or sets the flag that determines if swinging door compression is enabled for this historian instance. - /// - [ConnectionStringParameter] - [Description("Define the flag that determines if swinging door compression is enabled for this historian instance.")] - [DefaultValue(DefaultSwingingDoorCompressionEnabled)] - public bool SwingingDoorCompressionEnabled { get; set; } - - /// - /// Returns a flag that determines if measurements sent to this are destined for archival. - /// - public override bool OutputIsForArchive => true; - - /// - /// Gets flag that determines if this uses an asynchronous connection. - /// - protected override bool UseAsyncConnect => true; - - /// - /// Gets or sets based data source available to this . - /// - public override DataSet DataSource + /// + /// Gets flag that determines if this uses an asynchronous connection. + /// + protected override bool UseAsyncConnect => true; + + /// + /// Gets or sets based data source available to this . + /// + public override DataSet DataSource + { + get => base.DataSource; + set { - get => base.DataSource; - set - { - base.DataSource = value; + base.DataSource = value; - if (value is null) - return; + if (value is null) + return; - Dictionary measurements = new(); - string instanceName = InstanceName; + Dictionary measurements = new(); + string instanceName = InstanceName; - // Create dictionary of metadata for this server instance - foreach (DataRow row in value.Tables["ActiveMeasurements"].Rows) - { - if (MeasurementKey.TryParse(row["ID"].ToString(), out MeasurementKey key) && (key.Source?.Equals(instanceName, StringComparison.OrdinalIgnoreCase) ?? false)) - measurements[key.ID] = row; - } + // Create dictionary of metadata for this server instance + foreach (DataRow row in value.Tables["ActiveMeasurements"].Rows) + { + if (MeasurementKey.TryParse(row["ID"].ToString(), out MeasurementKey key) && (key.Source?.Equals(instanceName, StringComparison.OrdinalIgnoreCase) ?? false)) + measurements[key.ID] = row; + } - Dictionary> compressionSettings = new(); + Dictionary> compressionSettings = new(); - if (value.Tables.Contains("CompressionSettings")) + if (value.Tables.Contains("CompressionSettings")) + { + // Extract compression settings for defined measurements + foreach (DataRow row in value.Tables["CompressionSettings"].Rows) { - // Extract compression settings for defined measurements - foreach (DataRow row in value.Tables["CompressionSettings"].Rows) - { - uint pointID = row.ConvertField("PointID"); + uint pointID = row.ConvertField("PointID"); - if (InputMeasurementKeys.All(key => key.ID != pointID)) - continue; + if (InputMeasurementKeys.All(key => key.ID != pointID)) + continue; - // Get compression settings - int compressionMinTime = row.ConvertField("CompressionMinTime"); - int compressionMaxTime = row.ConvertField("CompressionMaxTime"); - double compressionLimit = row.ConvertField("CompressionLimit"); + // Get compression settings + int compressionMinTime = row.ConvertField("CompressionMinTime"); + int compressionMaxTime = row.ConvertField("CompressionMaxTime"); + double compressionLimit = row.ConvertField("CompressionLimit"); - compressionSettings[pointID] = new Tuple(compressionMinTime, compressionMaxTime, compressionLimit); - } + compressionSettings[pointID] = new Tuple(compressionMinTime, compressionMaxTime, compressionLimit); } + } - Interlocked.Exchange(ref m_measurements, measurements); - Interlocked.Exchange(ref m_compressionSettings, compressionSettings); + Interlocked.Exchange(ref m_measurements, measurements); + Interlocked.Exchange(ref m_compressionSettings, compressionSettings); - // When metadata is updated for an output adapter, reset sliding memory caches for Grafana data sources - TargetCaches.ResetAll(); - } + // When metadata is updated for an output adapter, reset sliding memory caches for Grafana data sources + TargetCaches.ResetAll(); } + } - /// - /// Returns the detailed status of the data output source. - /// - public override string Status + /// + /// Returns the detailed status of the data output source. + /// + public override string Status + { + get { - get + StringBuilder status = new(); + + status.Append(base.Status); + status.AppendLine(); + + status.AppendLine($" Historian instance name: {InstanceName}"); + status.AppendLine($" Working directory: {FilePath.TrimFileName(WorkingDirectory, 51)}"); + status.AppendLine($" Network data channel: {DataChannel.ToNonNullString(DefaultDataChannel)}"); + status.AppendLine($" Target file size: {TargetFileSize:N4}GB"); + status.AppendLine($" Desired remaining space: {DesiredRemainingSpace:N4}GB"); + status.AppendLine($" Directory naming mode: {DirectoryNamingMode}"); + status.AppendLine($" Disk flush interval: {m_archiveInfo.DiskFlushInterval:N0}ms"); + status.AppendLine($" Cache flush interval: {m_archiveInfo.CacheFlushInterval:N0}ms"); + status.AppendLine($" Staging count: {m_archiveInfo.StagingCount:N0}"); + status.AppendLine($" Point ID index enabled: {EnablePointIDIndex}"); + status.AppendLine($" Bloom filter enabled: {EnableTimeBucketBloomFilter}"); + + if (EnableTimeBucketBloomFilter) { - StringBuilder status = new(); - - status.Append(base.Status); - status.AppendLine(); - - status.AppendLine($" Historian instance name: {InstanceName}"); - status.AppendLine($" Working directory: {FilePath.TrimFileName(WorkingDirectory, 51)}"); - status.AppendLine($" Network data channel: {DataChannel.ToNonNullString(DefaultDataChannel)}"); - status.AppendLine($" Target file size: {TargetFileSize:N4}GB"); - status.AppendLine($" Desired remaining space: {DesiredRemainingSpace:N4}GB"); - status.AppendLine($" Directory naming mode: {DirectoryNamingMode}"); - status.AppendLine($" Disk flush interval: {m_archiveInfo.DiskFlushInterval:N0}ms"); - status.AppendLine($" Cache flush interval: {m_archiveInfo.CacheFlushInterval:N0}ms"); - status.AppendLine($" Staging count: {m_archiveInfo.StagingCount:N0}"); - status.AppendLine($" Memory pool size: {Globals.MemoryPool.MaximumPoolSize / SI2.Giga:N4}GB"); - status.AppendLine($" Maximum archive days: {(MaximumArchiveDays < 1 ? "No limit" : MaximumArchiveDays.ToString("N0"))}"); - status.AppendLine($" Downsampling intervals: {(string.IsNullOrWhiteSpace(DownsamplingIntervals) ? "None defined" : DownsamplingIntervals)}"); - status.AppendLine($" Auto-remove old archives: {AutoRemoveOldestFilesBeforeFull}"); - status.AppendLine($" Time reasonability check: {(EnableTimeReasonabilityCheck ? "Enabled" : "Not Enabled")}"); - status.AppendLine($" Archive curtailment timer: {Time.ToElapsedTimeString(ArchiveCurtailmentInterval, 0)}"); - - if (EnableTimeReasonabilityCheck) - { - status.AppendLine($" Maximum past time limit: {PastTimeReasonabilityLimit:N4}s, i.e., {new Ticks(m_pastTimeReasonabilityLimit).ToElapsedTimeString(4)}"); - status.AppendLine($" Maximum future time limit: {FutureTimeReasonabilityLimit:N4}s, i.e., {new Ticks(m_futureTimeReasonabilityLimit).ToElapsedTimeString(4)}"); - } + status.AppendLine($" Bloom index max size: {BloomFilterMaximumIndexSizeMB} MB/archive"); + status.AppendLine($" Bloom bucket duration: {BloomFilterBucketDurationSeconds:N0} seconds"); + status.AppendLine($" Bloom false-positive prob: {BloomFilterFalsePositiveProbability:P2}"); + } + + status.AppendLine($" Memory pool size: {Globals.MemoryPool.MaximumPoolSize / SI2.Giga:N4}GB"); + status.AppendLine($" Maximum archive days: {(MaximumArchiveDays < 1 ? "No limit" : MaximumArchiveDays.ToString("N0"))}"); + status.AppendLine($" Downsampling intervals: {(string.IsNullOrWhiteSpace(DownsamplingIntervals) ? "None defined" : DownsamplingIntervals)}"); + status.AppendLine($" Auto-remove old archives: {AutoRemoveOldestFilesBeforeFull}"); + status.AppendLine($" Time reasonability check: {(EnableTimeReasonabilityCheck ? "Enabled" : "Not Enabled")}"); + status.AppendLine($" Archive curtailment timer: {Time.ToElapsedTimeString(ArchiveCurtailmentInterval, 0)}"); + + if (EnableTimeReasonabilityCheck) + { + status.AppendLine($" Maximum past time limit: {PastTimeReasonabilityLimit:N4}s, i.e., {new Ticks(m_pastTimeReasonabilityLimit).ToElapsedTimeString(4)}"); + status.AppendLine($" Maximum future time limit: {FutureTimeReasonabilityLimit:N4}s, i.e., {new Ticks(m_futureTimeReasonabilityLimit).ToElapsedTimeString(4)}"); + } - if (m_dataServices is not null) - status.Append(m_dataServices.Status); + if (m_dataServices is not null) + status.Append(m_dataServices.Status); - if (m_replicationProviders is not null) - status.Append(m_replicationProviders.Status); + if (m_replicationProviders is not null) + status.Append(m_replicationProviders.Status); - Server?.Host?.GetFullStatus(status, 25); + Server?.Host?.GetFullStatus(status, 25); - return status.ToString(); - } + return status.ToString(); } + } - /// - /// Historian server instance. - /// - public HistorianServer Server { get; private set; } + /// + /// Historian server instance. + /// + public HistorianServer Server { get; private set; } - /// - /// Active measurement metadata dictionary. - /// - public Dictionary Measurements => m_measurements; + /// + /// Active measurement metadata dictionary. + /// + public Dictionary Measurements => m_measurements; - private SafeFileWatcher[] AttachedPathWatchers + private SafeFileWatcher[] AttachedPathWatchers + { + set { - set + if (m_attachedPathWatchers is not null) { - if (m_attachedPathWatchers is not null) + foreach (SafeFileWatcher watcher in m_attachedPathWatchers) { - foreach (SafeFileWatcher watcher in m_attachedPathWatchers) - { - watcher.EnableRaisingEvents = false; - watcher.Dispose(); - } + watcher.EnableRaisingEvents = false; + watcher.Dispose(); } - - m_attachedPathWatchers = value; } + + m_attachedPathWatchers = value; } + } + + #endregion - #endregion + #region [ Methods ] - #region [ Methods ] + /// + /// Releases the unmanaged resources used by this and optionally releases the managed resources. + /// + /// true to release both managed and unmanaged resources; false to release only unmanaged resources. + protected override void Dispose(bool disposing) + { + if (m_disposed) + return; - /// - /// Releases the unmanaged resources used by this and optionally releases the managed resources. - /// - /// true to release both managed and unmanaged resources; false to release only unmanaged resources. - protected override void Dispose(bool disposing) + try { - if (m_disposed) + if (!disposing) return; - try - { - if (!disposing) - return; - - AttachedPathWatchers = null; - - if (m_dataServices is not null) - { - m_dataServices.AdapterCreated -= DataServices_AdapterCreated; - m_dataServices.AdapterLoaded -= DataServices_AdapterLoaded; - m_dataServices.AdapterUnloaded -= DataServices_AdapterUnloaded; - m_dataServices.AdapterLoadException -= AdapterLoader_AdapterLoadException; - m_dataServices.Dispose(); - } + AttachedPathWatchers = null; - if (m_replicationProviders is not null) - { - m_replicationProviders.AdapterCreated -= ReplicationProviders_AdapterCreated; - m_replicationProviders.AdapterLoaded -= ReplicationProviders_AdapterLoaded; - m_replicationProviders.AdapterUnloaded -= ReplicationProviders_AdapterUnloaded; - m_replicationProviders.AdapterLoadException -= AdapterLoader_AdapterLoadException; - m_replicationProviders.Dispose(); - } + if (m_dataServices is not null) + { + m_dataServices.AdapterCreated -= DataServices_AdapterCreated; + m_dataServices.AdapterLoaded -= DataServices_AdapterLoaded; + m_dataServices.AdapterUnloaded -= DataServices_AdapterUnloaded; + m_dataServices.AdapterLoadException -= AdapterLoader_AdapterLoadException; + m_dataServices.Dispose(); + } - if (m_archiveCurtailmentTimer is not null) - { - m_archiveCurtailmentTimer.Stop(); - m_archiveCurtailmentTimer.Elapsed -= ArchiveCurtailmentTimerElapsed; - m_archiveCurtailmentTimer.Dispose(); - } + if (m_replicationProviders is not null) + { + m_replicationProviders.AdapterCreated -= ReplicationProviders_AdapterCreated; + m_replicationProviders.AdapterLoaded -= ReplicationProviders_AdapterLoaded; + m_replicationProviders.AdapterUnloaded -= ReplicationProviders_AdapterUnloaded; + m_replicationProviders.AdapterLoadException -= AdapterLoader_AdapterLoadException; + m_replicationProviders.Dispose(); } - finally + + if (m_archiveCurtailmentTimer is not null) { - m_disposed = true; // Prevent duplicate dispose. - base.Dispose(disposing); // Call base class Dispose(). + m_archiveCurtailmentTimer.Stop(); + m_archiveCurtailmentTimer.Elapsed -= ArchiveCurtailmentTimerElapsed; + m_archiveCurtailmentTimer.Dispose(); } } - - /// - /// Initializes this . - /// - public override void Initialize() + finally { - base.Initialize(); + m_disposed = true; // Prevent duplicate dispose. + base.Dispose(disposing); // Call base class Dispose(). + } + } - //const string errorMessage = "{0} is missing from Settings - Example: instanceName=default; ArchiveDirectories={{c:\\Archive1\\;d:\\Backups2\\}}; dataChannel={{port=9591; interface=0.0.0.0}}"; - Dictionary settings = Settings; + /// + /// Initializes this . + /// + public override void Initialize() + { + base.Initialize(); - // Validate settings. - if (!settings.TryGetValue(nameof(InstanceName), out m_instanceName) || string.IsNullOrWhiteSpace(m_instanceName)) - m_instanceName = Name; + //const string errorMessage = "{0} is missing from Settings - Example: instanceName=default; ArchiveDirectories={{c:\\Archive1\\;d:\\Backups2\\}}; dataChannel={{port=9591; interface=0.0.0.0}}"; + Dictionary settings = Settings; - // Track instance in static dictionary - Instances[InstanceName] = this; + // Validate settings. + if (!settings.TryGetValue(nameof(InstanceName), out m_instanceName) || string.IsNullOrWhiteSpace(m_instanceName)) + m_instanceName = Name; - if (!settings.TryGetValue(nameof(WorkingDirectory), out string setting) || string.IsNullOrEmpty(setting)) - setting = "Archive"; + // Track instance in static dictionary + Instances[InstanceName] = this; - WorkingDirectory = setting; + if (!settings.TryGetValue(nameof(WorkingDirectory), out string setting) || string.IsNullOrEmpty(setting)) + setting = "Archive"; - if (settings.TryGetValue(nameof(ArchiveDirectories), out setting)) - ArchiveDirectories = setting; + WorkingDirectory = setting; - ArchiveCurtailmentInterval = settings.TryGetValue(nameof(ArchiveCurtailmentInterval), out setting) ? int.Parse(setting) : DefaultArchiveCurtailmentInterval; + if (settings.TryGetValue(nameof(ArchiveDirectories), out setting)) + ArchiveDirectories = setting; - if (settings.TryGetValue(nameof(AttachedPaths), out setting)) - AttachedPaths = setting; + ArchiveCurtailmentInterval = settings.TryGetValue(nameof(ArchiveCurtailmentInterval), out setting) ? int.Parse(setting) : DefaultArchiveCurtailmentInterval; - WatchAttachedPaths = settings.TryGetValue(nameof(WatchAttachedPaths), out setting) && setting.ParseBoolean(); + if (settings.TryGetValue(nameof(AttachedPaths), out setting)) + AttachedPaths = setting; - DataChannel = settings.TryGetValue(nameof(DataChannel), out setting) && !string.IsNullOrWhiteSpace(setting) ? setting : DefaultDataChannel; + WatchAttachedPaths = settings.TryGetValue(nameof(WatchAttachedPaths), out setting) && setting.ParseBoolean(); - if (!settings.TryGetValue(nameof(TargetFileSize), out setting) || !double.TryParse(setting, out double targetFileSize)) - targetFileSize = DefaultTargetFileSize; + DataChannel = settings.TryGetValue(nameof(DataChannel), out setting) && !string.IsNullOrWhiteSpace(setting) ? setting : DefaultDataChannel; - if (targetFileSize < 0.1D || targetFileSize > SI2.Tera) - targetFileSize = DefaultTargetFileSize; + if (!settings.TryGetValue(nameof(TargetFileSize), out setting) || !double.TryParse(setting, out double targetFileSize)) + targetFileSize = DefaultTargetFileSize; - if (!settings.TryGetValue(nameof(DesiredRemainingSpace), out setting) || !double.TryParse(setting, out double desiredRemainingSpace)) - desiredRemainingSpace = DefaultDesiredRemainingSpace; + if (targetFileSize < 0.1D || targetFileSize > SI2.Tera) + targetFileSize = DefaultTargetFileSize; - if (desiredRemainingSpace < 0.1D || desiredRemainingSpace > SI2.Tera) - desiredRemainingSpace = DefaultDesiredRemainingSpace; + if (!settings.TryGetValue(nameof(DesiredRemainingSpace), out setting) || !double.TryParse(setting, out double desiredRemainingSpace)) + desiredRemainingSpace = DefaultDesiredRemainingSpace; - if (settings.TryGetValue(nameof(MaximumArchiveDays), out setting) && int.TryParse(setting, out int maximumArchiveDays)) - MaximumArchiveDays = maximumArchiveDays; + if (desiredRemainingSpace < 0.1D || desiredRemainingSpace > SI2.Tera) + desiredRemainingSpace = DefaultDesiredRemainingSpace; - if (settings.TryGetValue(nameof(AutoRemoveOldestFilesBeforeFull), out setting)) - AutoRemoveOldestFilesBeforeFull = setting.ParseBoolean(); + if (settings.TryGetValue(nameof(MaximumArchiveDays), out setting) && int.TryParse(setting, out int maximumArchiveDays)) + MaximumArchiveDays = maximumArchiveDays; - if (MaximumArchiveDays < 1 && !AutoRemoveOldestFilesBeforeFull) - OnStatusMessage(MessageLevel.Warning, "Maximum archive days not set and automated removal of oldest files before full disk is not enabled: system will not initiate archive file curtailment operations in this configuration. Disk space for target archive paths should be monitored externally."); + if (settings.TryGetValue(nameof(AutoRemoveOldestFilesBeforeFull), out setting)) + AutoRemoveOldestFilesBeforeFull = setting.ParseBoolean(); - if (settings.TryGetValue(nameof(DownsamplingIntervals), out setting)) - DownsamplingIntervals = setting; + if (MaximumArchiveDays < 1 && !AutoRemoveOldestFilesBeforeFull) + OnStatusMessage(MessageLevel.Warning, "Maximum archive days not set and automated removal of oldest files before full disk is not enabled: system will not initiate archive file curtailment operations in this configuration. Disk space for target archive paths should be monitored externally."); - if (MaximumArchiveDays > 0 && m_downsamplingIntervals.Any(kvp => kvp.Key >= MaximumArchiveDays)) - { - OnStatusMessage(MessageLevel.Warning, $"Downsampling intervals defined for days greater than or equal to maximum archive days ({MaximumArchiveDays:N0}) will be ignored."); - m_downsamplingIntervals = new SortedList(m_downsamplingIntervals.Where(kvp => kvp.Key < MaximumArchiveDays).ToDictionary(kvp => kvp.Key, kvp => kvp.Value)); - } + if (settings.TryGetValue(nameof(DownsamplingIntervals), out setting)) + DownsamplingIntervals = setting; - EnableTimeReasonabilityCheck = settings.TryGetValue(nameof(EnableTimeReasonabilityCheck), out setting) && setting.ParseBoolean(); + if (MaximumArchiveDays > 0 && m_downsamplingIntervals.Any(kvp => kvp.Key >= MaximumArchiveDays)) + { + OnStatusMessage(MessageLevel.Warning, $"Downsampling intervals defined for days greater than or equal to maximum archive days ({MaximumArchiveDays:N0}) will be ignored."); + m_downsamplingIntervals = new SortedList(m_downsamplingIntervals.Where(kvp => kvp.Key < MaximumArchiveDays).ToDictionary(kvp => kvp.Key, kvp => kvp.Value)); + } - if (settings.TryGetValue(nameof(PastTimeReasonabilityLimit), out setting) && double.TryParse(setting, out double value)) - PastTimeReasonabilityLimit = value; - else - PastTimeReasonabilityLimit = DefaultPastTimeReasonabilityLimit; + EnableTimeReasonabilityCheck = settings.TryGetValue(nameof(EnableTimeReasonabilityCheck), out setting) && setting.ParseBoolean(); - if (settings.TryGetValue(nameof(FutureTimeReasonabilityLimit), out setting) && double.TryParse(setting, out value)) - FutureTimeReasonabilityLimit = value; - else - FutureTimeReasonabilityLimit = DefaultFutureTimeReasonabilityLimit; + if (settings.TryGetValue(nameof(PastTimeReasonabilityLimit), out setting) && double.TryParse(setting, out double value)) + PastTimeReasonabilityLimit = value; + else + PastTimeReasonabilityLimit = DefaultPastTimeReasonabilityLimit; - SwingingDoorCompressionEnabled = settings.TryGetValue(nameof(SwingingDoorCompressionEnabled), out setting) && setting.ParseBoolean(); + if (settings.TryGetValue(nameof(FutureTimeReasonabilityLimit), out setting) && double.TryParse(setting, out value)) + FutureTimeReasonabilityLimit = value; + else + FutureTimeReasonabilityLimit = DefaultFutureTimeReasonabilityLimit; - if (settings.TryGetValue(nameof(DirectoryNamingMode), out setting) && Enum.TryParse(setting, true, out ArchiveDirectoryMethod directoryNamingMode)) - DirectoryNamingMode = directoryNamingMode; - else - DirectoryNamingMode = DefaultDirectoryNamingMode; + SwingingDoorCompressionEnabled = settings.TryGetValue(nameof(SwingingDoorCompressionEnabled), out setting) && setting.ParseBoolean(); - if (settings.TryGetValue(nameof(DirectoryFillMethod), out setting) && Enum.TryParse(setting, true, out ArchiveDirectoryFillMethod directoryFillMethod)) - DirectoryFillMethod = directoryFillMethod; - else - DirectoryFillMethod = DefaultDirectoryFillMethod; - - // Handle advanced settings - there are hidden but available from manual entry into connection string - if (!settings.TryGetValue("StagingCount", out setting) || !int.TryParse(setting, out int stagingCount)) - stagingCount = 3; - - if (!settings.TryGetValue("DiskFlushInterval", out setting) || !int.TryParse(setting, out int diskFlushInterval)) - diskFlushInterval = 10000; - - if (!settings.TryGetValue("CacheFlushInterval", out setting) || !int.TryParse(setting, out int cacheFlushInterval)) - cacheFlushInterval = 100; - - // Validate maximum downsampling intervals. Currently only 10 stages are defined, 0 to 9 - since configured - // staging count is typically 3, this allows 6 total downsampling intervals (as extra stages): - if (m_downsamplingIntervals.Count > 9 - stagingCount) - throw new InvalidOperationException($"Maximum of {9 - stagingCount} downsampling intervals are allowed."); - - // Establish archive information for this historian instance - m_archiveInfo = new HistorianServerDatabaseConfig(InstanceName, WorkingDirectory, true); - - if (m_archiveDirectories is not null) - m_archiveInfo.FinalWritePaths.AddRange(m_archiveDirectories); - - if (m_attachedPaths is not null) - m_archiveInfo.ImportPaths.AddRange(m_attachedPaths); - - m_archiveInfo.ImportAttachedPathsAtStartup = false; - m_archiveInfo.TargetFileSize = (long)(targetFileSize * SI.Giga); - m_archiveInfo.DesiredRemainingSpace = (long)(desiredRemainingSpace * SI.Giga); - m_archiveInfo.DirectoryMethod = DirectoryNamingMode; - m_archiveInfo.FillMethod = DirectoryFillMethod; - m_archiveInfo.StagingCount = stagingCount; - m_archiveInfo.DiskFlushInterval = diskFlushInterval; - m_archiveInfo.CacheFlushInterval = cacheFlushInterval; - - // Provide web service support - m_dataServices = new DataServices(); - m_dataServices.AdapterCreated += DataServices_AdapterCreated; - m_dataServices.AdapterLoaded += DataServices_AdapterLoaded; - m_dataServices.AdapterUnloaded += DataServices_AdapterUnloaded; - m_dataServices.AdapterLoadException += AdapterLoader_AdapterLoadException; - - // Provide archive replication support - m_replicationProviders = new ReplicationProviders(); - m_replicationProviders.AdapterCreated += ReplicationProviders_AdapterCreated; - m_replicationProviders.AdapterLoaded += ReplicationProviders_AdapterLoaded; - m_replicationProviders.AdapterUnloaded += ReplicationProviders_AdapterUnloaded; - m_replicationProviders.AdapterLoadException += AdapterLoader_AdapterLoadException; - - if (MaximumArchiveDays > 0) - { - m_archiveCurtailmentTimer = new Timer(ArchiveCurtailmentInterval * 1000.0D); - m_archiveCurtailmentTimer.AutoReset = true; - m_archiveCurtailmentTimer.Elapsed += ArchiveCurtailmentTimerElapsed; - m_archiveCurtailmentTimer.Enabled = true; - } + if (settings.TryGetValue(nameof(DirectoryNamingMode), out setting) && Enum.TryParse(setting, true, out ArchiveDirectoryMethod directoryNamingMode)) + DirectoryNamingMode = directoryNamingMode; + else + DirectoryNamingMode = DefaultDirectoryNamingMode; - // Initialize the file watchers for attached paths - AttachedPathWatchers = WatchAttachedPaths ? m_attachedPaths?.Select(WatchPath).ToArray() : null; - } + if (settings.TryGetValue(nameof(DirectoryFillMethod), out setting) && Enum.TryParse(setting, true, out ArchiveDirectoryFillMethod directoryFillMethod)) + DirectoryFillMethod = directoryFillMethod; + else + DirectoryFillMethod = DefaultDirectoryFillMethod; - /// - /// Gets a short one-line status of this . - /// - /// Maximum length of the status message. - /// Text of the status message. - public override string GetShortStatus(int maxLength) => - $"Archived {m_archivedMeasurements} measurements.".CenterText(maxLength); - - /// - /// Detaches an archive file from the historian. - /// - /// Archive file name to detach. - [AdapterCommand("Detaches an archive file from the historian. Wild cards are allowed in file name and folders to handle multiple files.", "Administrator", "Editor")] - public void DetachFile(string fileName) - { - ClientDatabaseBase database = GetClientDatabase(); + // Handle advanced settings - there are hidden but available from manual entry into connection string + if (!settings.TryGetValue("StagingCount", out setting) || !int.TryParse(setting, out int stagingCount)) + stagingCount = 3; - if (fileName.Contains('*')) - ExecuteWildCardFileOperation(database, Path.GetFullPath(fileName), database.DetatchFiles); - else - ExecuteFileOperation(database, Path.GetFullPath(fileName), database.DetatchFiles); - } + if (!settings.TryGetValue("DiskFlushInterval", out setting) || !int.TryParse(setting, out int diskFlushInterval)) + diskFlushInterval = 10000; - /// - /// Deletes an archive file from the historian. - /// - /// Archive file name to delete. - [AdapterCommand("Deletes an archive file from the historian. Wild cards are allowed in file name and folders to handle multiple files.", "Administrator", "Editor")] - public void DeleteFile(string fileName) - { - ClientDatabaseBase database = GetClientDatabase(); + if (!settings.TryGetValue("CacheFlushInterval", out setting) || !int.TryParse(setting, out int cacheFlushInterval)) + cacheFlushInterval = 100; - if (fileName.Contains('*')) - ExecuteWildCardFileOperation(database, Path.GetFullPath(fileName), database.DeleteFiles); - else - ExecuteFileOperation(database, Path.GetFullPath(fileName), database.DeleteFiles); - } + // Validate maximum downsampling intervals. Currently only 10 stages are defined, 0 to 9 - since configured + // staging count is typically 3, this allows 6 total downsampling intervals (as extra stages): + if (m_downsamplingIntervals.Count > 9 - stagingCount) + throw new InvalidOperationException($"Maximum of {9 - stagingCount} downsampling intervals are allowed."); - /// - /// Detaches files in an archive folder from the historian. - /// - /// Archive folder name to detach. - [AdapterCommand("Detaches all archive files in a specified folder from the historian.", "Administrator", "Editor")] - public void DetachFolder(string folderName) - { - ClientDatabaseBase database = GetClientDatabase(); - ExecuteFolderOperation(database, FilePath.GetDirectoryName(Path.GetFullPath(folderName)), database.DetatchFiles); - } + // Establish archive information for this historian instance + if (settings.TryGetValue(nameof(EnablePointIDIndex), out setting)) + EnablePointIDIndex = setting.ParseBoolean(); + + if (settings.TryGetValue(nameof(EnableTimeBucketBloomFilter), out setting)) + EnableTimeBucketBloomFilter = setting.ParseBoolean(); + + if (settings.TryGetValue(nameof(BloomFilterMaximumIndexSizeMB), out setting) && int.TryParse(setting, out int bloomFilterMaximumIndexSizeMB)) + BloomFilterMaximumIndexSizeMB = bloomFilterMaximumIndexSizeMB; + + if (settings.TryGetValue(nameof(BloomFilterBucketDurationSeconds), out setting) && int.TryParse(setting, out int bloomFilterBucketDurationSeconds)) + BloomFilterBucketDurationSeconds = bloomFilterBucketDurationSeconds; + + if (settings.TryGetValue(nameof(BloomFilterFalsePositiveProbability), out setting) && double.TryParse(setting, out double bloomFilterFalsePositiveProbability)) + BloomFilterFalsePositiveProbability = bloomFilterFalsePositiveProbability; + + TimeBucketBloomFilterSettings bloomFilter = CreateBloomFilterSettings(); + + m_archiveInfo = new HistorianServerDatabaseConfig(InstanceName, WorkingDirectory, true) + { + BloomFilter = bloomFilter, + EnablePointIDIndex = EnablePointIDIndex + }; + + if (m_archiveDirectories is not null) + m_archiveInfo.FinalWritePaths.AddRange(m_archiveDirectories); + + if (m_attachedPaths is not null) + m_archiveInfo.ImportPaths.AddRange(m_attachedPaths); - /// - /// Deletes files in an archive folder from the historian. - /// - /// Archive folder name to delete. - [AdapterCommand("Deletes all archive files in a specified folder from the historian.", "Administrator", "Editor")] - public void DeleteFolder(string folderName) + m_archiveInfo.ImportAttachedPathsAtStartup = false; + m_archiveInfo.TargetFileSize = (long)(targetFileSize * SI.Giga); + m_archiveInfo.DesiredRemainingSpace = (long)(desiredRemainingSpace * SI.Giga); + m_archiveInfo.DirectoryMethod = DirectoryNamingMode; + m_archiveInfo.FillMethod = DirectoryFillMethod; + m_archiveInfo.StagingCount = stagingCount; + m_archiveInfo.DiskFlushInterval = diskFlushInterval; + m_archiveInfo.CacheFlushInterval = cacheFlushInterval; + + // Provide web service support + m_dataServices = new DataServices(); + m_dataServices.AdapterCreated += DataServices_AdapterCreated; + m_dataServices.AdapterLoaded += DataServices_AdapterLoaded; + m_dataServices.AdapterUnloaded += DataServices_AdapterUnloaded; + m_dataServices.AdapterLoadException += AdapterLoader_AdapterLoadException; + + // Provide archive replication support + m_replicationProviders = new ReplicationProviders(); + m_replicationProviders.AdapterCreated += ReplicationProviders_AdapterCreated; + m_replicationProviders.AdapterLoaded += ReplicationProviders_AdapterLoaded; + m_replicationProviders.AdapterUnloaded += ReplicationProviders_AdapterUnloaded; + m_replicationProviders.AdapterLoadException += AdapterLoader_AdapterLoadException; + + if (MaximumArchiveDays > 0) { - ClientDatabaseBase database = GetClientDatabase(); - ExecuteFolderOperation(database, FilePath.GetDirectoryName(Path.GetFullPath(folderName)), database.DeleteFiles); + m_archiveCurtailmentTimer = new Timer(ArchiveCurtailmentInterval * 1000.0D); + m_archiveCurtailmentTimer.AutoReset = true; + m_archiveCurtailmentTimer.Elapsed += ArchiveCurtailmentTimerElapsed; + m_archiveCurtailmentTimer.Enabled = true; } - /// - /// Initiates archive file curtailment based on defined maximum archive days, auto removal of oldest files before full disk and defined downsampling intervals. - /// - [AdapterCommand("Initiates archive file curtailment based on defined maximum archive days, auto removal of oldest files before full disk and defined downsampling intervals.", "Administrator", "Editor")] - public void CurtailArchiveFiles() - { - if (AutoRemoveOldestFilesBeforeFull) - RemoveOldestFilesBeforeFull(); + // Initialize the file watchers for attached paths + AttachedPathWatchers = WatchAttachedPaths ? m_attachedPaths?.Select(WatchPath).ToArray() : null; + } - if (m_downsamplingIntervals.Count > 0) - DownsampleArchiveFiles(); + /// + /// Gets a short one-line status of this . + /// + /// Maximum length of the status message. + /// Text of the status message. + public override string GetShortStatus(int maxLength) => + $"Archived {m_archivedMeasurements} measurements.".CenterText(maxLength); - if (MaximumArchiveDays > 0) - RemoveFilesOlderThanMaxArchiveDays(); - } + /// + /// Detaches an archive file from the historian. + /// + /// Archive file name to detach. + [AdapterCommand("Detaches an archive file from the historian. Wild cards are allowed in file name and folders to handle multiple files.", "Administrator", "Editor")] + public void DetachFile(string fileName) + { + ClientDatabaseBase database = GetClientDatabase(); - // Builds the set of normalized write directories (absolute, trimmed, with a trailing separator) used to scope - // archive-reduction operations so that only files written by this instance are affected. Attached/import folders - // live outside these directories and are therefore left untouched by all reduction paths: disk-pressure removal, - // downsampling and maximum-archive-days deletion. - private string[] GetNormalizedArchiveDirectories() - { - string[] archiveDirectories = (m_archiveDirectories ?? [WorkingDirectory]).ToArray(); + if (fileName.Contains('*')) + ExecuteWildCardFileOperation(database, Path.GetFullPath(fileName), database.DetatchFiles); + else + ExecuteFileOperation(database, Path.GetFullPath(fileName), database.DetatchFiles); + } - for (int i = 0; i < archiveDirectories.Length; i++) - { - string normalizedDirectory = Path.GetFullPath(archiveDirectories[i]).TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar); + /// + /// Deletes an archive file from the historian. + /// + /// Archive file name to delete. + [AdapterCommand("Deletes an archive file from the historian. Wild cards are allowed in file name and folders to handle multiple files.", "Administrator", "Editor")] + public void DeleteFile(string fileName) + { + ClientDatabaseBase database = GetClientDatabase(); - // Add trailing separator to ensure only subdirectory, not prefix match which might be a file - if (!normalizedDirectory.EndsWith(Path.DirectorySeparatorChar.ToString())) - normalizedDirectory += Path.DirectorySeparatorChar; + if (fileName.Contains('*')) + ExecuteWildCardFileOperation(database, Path.GetFullPath(fileName), database.DeleteFiles); + else + ExecuteFileOperation(database, Path.GetFullPath(fileName), database.DeleteFiles); + } - archiveDirectories[i] = normalizedDirectory; - } + /// + /// Detaches files in an archive folder from the historian. + /// + /// Archive folder name to detach. + [AdapterCommand("Detaches all archive files in a specified folder from the historian.", "Administrator", "Editor")] + public void DetachFolder(string folderName) + { + ClientDatabaseBase database = GetClientDatabase(); + ExecuteFolderOperation(database, FilePath.GetDirectoryName(Path.GetFullPath(folderName)), database.DetatchFiles); + } - return archiveDirectories; - } + /// + /// Deletes files in an archive folder from the historian. + /// + /// Archive folder name to delete. + [AdapterCommand("Deletes all archive files in a specified folder from the historian.", "Administrator", "Editor")] + public void DeleteFolder(string folderName) + { + ClientDatabaseBase database = GetClientDatabase(); + ExecuteFolderOperation(database, FilePath.GetDirectoryName(Path.GetFullPath(folderName)), database.DeleteFiles); + } + + /// + /// Initiates archive file curtailment based on defined maximum archive days, auto removal of oldest files before full disk and defined downsampling intervals. + /// + [AdapterCommand("Initiates archive file curtailment based on defined maximum archive days, auto removal of oldest files before full disk and defined downsampling intervals.", "Administrator", "Editor")] + public void CurtailArchiveFiles() + { + if (AutoRemoveOldestFilesBeforeFull) + RemoveOldestFilesBeforeFull(); - // Determines whether the specified archive file resides within one of the supplied (normalized) write - // directories. Used to scope reduction operations to written files and skip attached/non-write folders. - private static bool FilePathTargetsArchiveDirectories(string fileName, string[] normalizedArchiveDirectories) + if (m_downsamplingIntervals.Count > 0) + DownsampleArchiveFiles(); + + if (MaximumArchiveDays > 0) + RemoveFilesOlderThanMaxArchiveDays(); + } + + // Builds the set of normalized write directories (absolute, trimmed, with a trailing separator) used to scope + // archive-reduction operations so that only files written by this instance are affected. Attached/import folders + // live outside these directories and are therefore left untouched by all reduction paths: disk-pressure removal, + // downsampling and maximum-archive-days deletion. + private string[] GetNormalizedArchiveDirectories() + { + string[] archiveDirectories = (m_archiveDirectories ?? [WorkingDirectory]).ToArray(); + + for (int i = 0; i < archiveDirectories.Length; i++) { - // Normalize file path: absolute, trimmed - string filePath = Path.GetFullPath(fileName); + string normalizedDirectory = Path.GetFullPath(archiveDirectories[i]).TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar); - // Case-insensitive compare on Windows, case-sensitive on Posix environments - StringComparison comparison = Common.IsPosixEnvironment ? StringComparison.Ordinal : StringComparison.OrdinalIgnoreCase; + // Add trailing separator to ensure only subdirectory, not prefix match which might be a file + if (!normalizedDirectory.EndsWith(Path.DirectorySeparatorChar.ToString())) + normalizedDirectory += Path.DirectorySeparatorChar; - return normalizedArchiveDirectories.Any(directory => filePath.StartsWith(directory, comparison)); + archiveDirectories[i] = normalizedDirectory; } - private void RemoveOldestFilesBeforeFull() - { - // Set target space to be three times the target file size plus minimum disk space before considered full, - // this will allow three new final stage archive files to be written before next curtailment interval. - // Curtailment interval, desired remaining space and target file size can all be adjusted to better - // accommodate high volume data archiving in very low disk space environments. - long neededSpace = m_archiveInfo.DesiredRemainingSpace + 3 * m_archiveInfo.TargetFileSize; + return archiveDirectories; + } - string[] archiveDirectories = GetNormalizedArchiveDirectories(); + // Determines whether the specified archive file resides within one of the supplied (normalized) write + // directories. Used to scope reduction operations to written files and skip attached/non-write folders. + private static bool FilePathTargetsArchiveDirectories(string fileName, string[] normalizedArchiveDirectories) + { + // Normalize file path: absolute, trimmed + string filePath = Path.GetFullPath(fileName); - try - { - // Check if any target archive destination has enough disk space - foreach (string path in archiveDirectories) - { - FilePath.GetAvailableFreeSpace(path, out long freeSpace, out _); + // Case-insensitive compare on Windows, case-sensitive on Posix environments + StringComparison comparison = Common.IsPosixEnvironment ? StringComparison.Ordinal : StringComparison.OrdinalIgnoreCase; - // If any path has needed space, then we are done - if (freeSpace > neededSpace) - return; - } - } - catch (Exception ex) + return normalizedArchiveDirectories.Any(directory => filePath.StartsWith(directory, comparison)); + } + + private void RemoveOldestFilesBeforeFull() + { + // Set target space to be three times the target file size plus minimum disk space before considered full, + // this will allow three new final stage archive files to be written before next curtailment interval. + // Curtailment interval, desired remaining space and target file size can all be adjusted to better + // accommodate high volume data archiving in very low disk space environments. + long neededSpace = m_archiveInfo.DesiredRemainingSpace + 3 * m_archiveInfo.TargetFileSize; + + string[] archiveDirectories = GetNormalizedArchiveDirectories(); + + try + { + // Check if any target archive destination has enough disk space + foreach (string path in archiveDirectories) { - OnProcessException(MessageLevel.Warning, new InvalidOperationException($"Oldest file removal operation cancelled: failed during check for full disk: {ex.Message}", ex)); + FilePath.GetAvailableFreeSpace(path, out long freeSpace, out _); - // Do not continue with archive curtailment if disk space check failed - return; + // If any path has needed space, then we are done + if (freeSpace > neededSpace) + return; } + } + catch (Exception ex) + { + OnProcessException(MessageLevel.Warning, new InvalidOperationException($"Oldest file removal operation cancelled: failed during check for full disk: {ex.Message}", ex)); - try - { - OnStatusMessage(MessageLevel.Warning, "Disk space is near full, scanning for oldest archive files..."); + // Do not continue with archive curtailment if disk space check failed + return; + } - ClientDatabaseBase database = GetClientDatabase(); + try + { + OnStatusMessage(MessageLevel.Warning, "Disk space is near full, scanning for oldest archive files..."); - long fileSizeSum = 0; + ClientDatabaseBase database = GetClientDatabase(); - // Find the oldest archive files until we have reached target disk space. End time is preferred over - // start time for sorting since devices with an inaccurate GPS clock can provide bad start times when - // out of range timestamps are not configured to be filtered, and you don't want to accidentally delete - // a file with otherwise in-range data. - ArchiveDetails[] filesToDelete = database.GetAllAttachedFiles() - .Where(file => FilePathTargetsArchiveDirectories(file.FileName, archiveDirectories)) - .OrderBy(file => file.EndTime) - .TakeWhile(item => - { - fileSizeSum += item.FileSize; - return fileSizeSum < neededSpace; - }) - .ToArray(); + long fileSizeSum = 0; - database.DeleteFiles(filesToDelete.Select(file => file.Id).ToList()); + // Find the oldest archive files until we have reached target disk space. End time is preferred over + // start time for sorting since devices with an inaccurate GPS clock can provide bad start times when + // out of range timestamps are not configured to be filtered, and you don't want to accidentally delete + // a file with otherwise in-range data. + ArchiveDetails[] filesToDelete = database.GetAllAttachedFiles() + .Where(file => FilePathTargetsArchiveDirectories(file.FileName, archiveDirectories)) + .OrderBy(file => file.EndTime) + .TakeWhile(item => + { + fileSizeSum += item.FileSize; + return fileSizeSum < neededSpace; + }) + .ToArray(); - OnStatusMessage(MessageLevel.Warning, $"Deleted the following oldest archive files in order to free disk space:\r\n {filesToDelete.Select(file => FilePath.TrimFileName(file.FileName, 75)).ToDelimitedString($"{Environment.NewLine} ")}"); - } - catch (Exception ex) - { - OnProcessException(MessageLevel.Error, new InvalidOperationException($"Failed while attempting to delete oldest archive files in order to free disk space: {ex.Message}", ex)); - } - } + database.DeleteFiles(filesToDelete.Select(file => file.Id).ToList()); - private void DownsampleArchiveFiles() + OnStatusMessage(MessageLevel.Warning, $"Deleted the following oldest archive files in order to free disk space:\r\n {filesToDelete.Select(file => FilePath.TrimFileName(file.FileName, 75)).ToDelimitedString($"{Environment.NewLine} ")}"); + } + catch (Exception ex) { - ClientDatabaseBase database = GetClientDatabase(); - SortedList filesToDownsample = new(); + OnProcessException(MessageLevel.Error, new InvalidOperationException($"Failed while attempting to delete oldest archive files in order to free disk space: {ex.Message}", ex)); + } + } - try - { - OnStatusMessage(MessageLevel.Info, "Scanning archive files for downsampling..."); + private void DownsampleArchiveFiles() + { + ClientDatabaseBase database = GetClientDatabase(); + SortedList filesToDownsample = new(); - // Scope downsampling to files within the write directories so attached/non-write folders are never - // rewritten at a lower resolution. - string[] normalizedArchiveDirectories = GetNormalizedArchiveDirectories(); + try + { + OnStatusMessage(MessageLevel.Info, "Scanning archive files for downsampling..."); - List attachedFiles = database.GetAllAttachedFiles() - .Where(file => FilePathTargetsArchiveDirectories(file.FileName, normalizedArchiveDirectories)) - .ToList(); + // Scope downsampling to files within the write directories so attached/non-write folders are never + // rewritten at a lower resolution. + string[] normalizedArchiveDirectories = GetNormalizedArchiveDirectories(); - // Start with oldest files first, this way they can be excluded from further downsampling if they are already targeted - int[] startDays = m_downsamplingIntervals.Keys.OrderByDescending(value => value).ToArray(); - HashSet targetedArchives = []; + List attachedFiles = database.GetAllAttachedFiles() + .Where(file => FilePathTargetsArchiveDirectories(file.FileName, normalizedArchiveDirectories)) + .ToList(); - int downsamplingStage = m_archiveInfo.StagingCount + startDays.Length; + // Start with oldest files first, this way they can be excluded from further downsampling if they are already targeted + int[] startDays = m_downsamplingIntervals.Keys.OrderByDescending(value => value).ToArray(); + HashSet targetedArchives = []; - foreach (int startDay in startDays) - { - // Get list of files that have both a start time and an end time that are greater than the target - // start day for a given downsampling interval. We check both start and end times since devices - // with an inaccurate GPS clock can provide bad time when out of range timestamps are not configured - // to be filtered and you don't want to accidentally delete a file with otherwise in-range data. - // We also filter on stage number to avoid downsampling files that have already been downsampled. - HashSet matchingFiles = new(attachedFiles.Where(file => - (DateTime.UtcNow - file.StartTime).TotalDays > startDay && - (DateTime.UtcNow - file.EndTime).TotalDays > startDay && - FileFlags.GetStageNumber(file.Flags) < downsamplingStage && - !file.Flags.Contains(FileFlags.IntermediateFile))); - - // Start days are ordered from highest to lowest, so we decrement the downsampling stage - downsamplingStage--; - - if (matchingFiles.Count == 0) - continue; + int downsamplingStage = m_archiveInfo.StagingCount + startDays.Length; + + foreach (int startDay in startDays) + { + // Get list of files that have both a start time and an end time that are greater than the target + // start day for a given downsampling interval. We check both start and end times since devices + // with an inaccurate GPS clock can provide bad time when out of range timestamps are not configured + // to be filtered and you don't want to accidentally delete a file with otherwise in-range data. + // We also filter on stage number to avoid downsampling files that have already been downsampled. + HashSet matchingFiles = new(attachedFiles.Where(file => + (DateTime.UtcNow - file.StartTime).TotalDays > startDay && + (DateTime.UtcNow - file.EndTime).TotalDays > startDay && + FileFlags.GetStageNumber(file.Flags) < downsamplingStage && + !file.Flags.Contains(FileFlags.IntermediateFile))); + + // Start days are ordered from highest to lowest, so we decrement the downsampling stage + downsamplingStage--; + + if (matchingFiles.Count == 0) + continue; - matchingFiles.ExceptWith(targetedArchives); + matchingFiles.ExceptWith(targetedArchives); - if (matchingFiles.Count == 0) - continue; + if (matchingFiles.Count == 0) + continue; - filesToDownsample[startDay] = matchingFiles.ToArray(); - targetedArchives.UnionWith(matchingFiles); - } - } - catch (Exception ex) - { - OnProcessException(MessageLevel.Warning, new InvalidOperationException($"Failed while scanning archive files for downsampling: {ex.Message}", ex)); + filesToDownsample[startDay] = matchingFiles.ToArray(); + targetedArchives.UnionWith(matchingFiles); } + } + catch (Exception ex) + { + OnProcessException(MessageLevel.Warning, new InvalidOperationException($"Failed while scanning archive files for downsampling: {ex.Message}", ex)); + } - if (filesToDownsample.Count == 0) - return; + if (filesToDownsample.Count == 0) + return; - try - { - int downsamplingStage = 1; + try + { + int downsamplingStage = 1; - foreach (KeyValuePair kvp in filesToDownsample) - { - int startDay = kvp.Key; - ArchiveDetails[] files = kvp.Value; - double samplesPerSecond = m_downsamplingIntervals[startDay]; + foreach (KeyValuePair kvp in filesToDownsample) + { + int startDay = kvp.Key; + ArchiveDetails[] files = kvp.Value; + double samplesPerSecond = m_downsamplingIntervals[startDay]; - // Each successive downsampled file will be flagged with a higher stage count - Guid stageFlags = FileFlags.GetStage(m_archiveInfo.StagingCount + downsamplingStage++); + // Each successive downsampled file will be flagged with a higher stage count + Guid stageFlags = FileFlags.GetStage(m_archiveInfo.StagingCount + downsamplingStage++); - OnStatusMessage(MessageLevel.Info, $"Downsampling {files.Length:N0} archive files older than {startDay:N0} days to {samplesPerSecond} samples per second..."); + OnStatusMessage(MessageLevel.Info, $"Downsampling {files.Length:N0} archive files older than {startDay:N0} days to {samplesPerSecond} samples per second..."); - foreach (ArchiveDetails file in files) - { - string downsampledFile = DownsampleArchiveFile(file, samplesPerSecond, stageFlags); + foreach (ArchiveDetails file in files) + { + string downsampledFile = DownsampleArchiveFile(file, samplesPerSecond, stageFlags); - if (string.IsNullOrWhiteSpace(downsampledFile)) - continue; + if (string.IsNullOrWhiteSpace(downsampledFile)) + continue; - try - { - // Remove higher resolution file after successful downsampling - database.DeleteFiles([file.Id]); + try + { + // Remove higher resolution file after successful downsampling + database.DeleteFiles([file.Id]); - // Attach downsampled resolution file to active historian instance - database.AttachFilesOrPaths([downsampledFile]); + // Attach downsampled resolution file to active historian instance + database.AttachFilesOrPaths([downsampledFile]); - OnStatusMessage(MessageLevel.Info, $"Downsampled archive file \"{FilePath.GetFileName(file.FileName)}\" to {samplesPerSecond} samples per second."); - } - catch (Exception ex) - { - OnProcessException(MessageLevel.Warning, new InvalidOperationException($"Failed while attempting to transition to downsampled archive file \"{FilePath.GetFileName(downsampledFile)}\": {ex.Message}", ex)); - } + OnStatusMessage(MessageLevel.Info, $"Downsampled archive file \"{FilePath.GetFileName(file.FileName)}\" to {samplesPerSecond} samples per second."); + } + catch (Exception ex) + { + OnProcessException(MessageLevel.Warning, new InvalidOperationException($"Failed while attempting to transition to downsampled archive file \"{FilePath.GetFileName(downsampledFile)}\": {ex.Message}", ex)); } } } - catch (Exception ex) - { - OnProcessException(MessageLevel.Warning, new InvalidOperationException($"Failed downsampling archive files: {ex.Message}", ex)); - } } - - private string DownsampleArchiveFile(ArchiveDetails file, double samplesPerSecond, Guid stageFlags) + catch (Exception ex) { - try - { - DateTime startTime = file.StartTime; - DateTime endTime = file.EndTime; + OnProcessException(MessageLevel.Warning, new InvalidOperationException($"Failed downsampling archive files: {ex.Message}", ex)); + } + } - // Interval is the time spacing between retained samples, i.e., the reciprocal of the target sample - // rate (samplesPerSecond is validated > 0 by the DownsamplingIntervals parser, so this cannot divide - // by zero). - TimeSpan interval = new((long)(TimeSpan.TicksPerSecond / samplesPerSecond)); + private string DownsampleArchiveFile(ArchiveDetails file, double samplesPerSecond, Guid stageFlags) + { + try + { + DateTime startTime = file.StartTime; + DateTime endTime = file.EndTime; - using ArchiveList archiveList = new(); - archiveList.LoadFiles([file.FileName]); + // Interval is the time spacing between retained samples, i.e., the reciprocal of the target sample + // rate (samplesPerSecond is validated > 0 by the DownsamplingIntervals parser, so this cannot divide + // by zero). + TimeSpan interval = new((long)(TimeSpan.TicksPerSecond / samplesPerSecond)); - using SequentialReaderStream reader = new(archiveList, SortedTreeEngineReaderOptions.Default, TimestampSeekFilter.CreateFromIntervalData(startTime, endTime, interval, new TimeSpan(TimeSpan.TicksPerMillisecond))); + using ArchiveList archiveList = new(); + archiveList.LoadFiles([file.FileName]); - string sourcePath = FilePath.GetDirectoryName(file.FileName); - string completeFileName = Path.Combine(sourcePath, $"{FilePath.GetFileNameWithoutExtension(file.FileName)}-{samplesPerSecond}sps.d2"); - string pendingFileName = Path.Combine(sourcePath, $"{FilePath.GetFileNameWithoutExtension(completeFileName)}.~d2i"); + using SequentialReaderStream reader = new(archiveList, SortedTreeEngineReaderOptions.Default, TimestampSeekFilter.CreateFromIntervalData(startTime, endTime, interval, new TimeSpan(TimeSpan.TicksPerMillisecond))); - SortedTreeFileSimpleWriter.Create(pendingFileName, completeFileName, 4096, null, HistorianFileEncodingDefinition.TypeGuid, reader, stageFlags); + string sourcePath = FilePath.GetDirectoryName(file.FileName); + string completeFileName = Path.Combine(sourcePath, $"{FilePath.GetFileNameWithoutExtension(file.FileName)}-{samplesPerSecond}sps.d2"); + string pendingFileName = Path.Combine(sourcePath, $"{FilePath.GetFileNameWithoutExtension(completeFileName)}.~d2i"); - return completeFileName; - } - catch (Exception ex) - { - OnProcessException(MessageLevel.Warning, new InvalidOperationException($"Failed while attempting to downsample archive file \"{FilePath.GetFileName(file.FileName)}\" to {samplesPerSecond} samples per second: {ex.Message}", ex)); - } + SortedTreeFileSimpleWriter.CreateWithIndexes(pendingFileName, completeFileName, 4096, null, HistorianFileEncodingDefinition.TypeGuid, reader, CreateBloomFilterSettings(), EnablePointIDIndex, stageFlags); - return null; + return completeFileName; + } + catch (Exception ex) + { + OnProcessException(MessageLevel.Warning, new InvalidOperationException($"Failed while attempting to downsample archive file \"{FilePath.GetFileName(file.FileName)}\" to {samplesPerSecond} samples per second: {ex.Message}", ex)); } - private void RemoveFilesOlderThanMaxArchiveDays() + return null; + } + + private void RemoveFilesOlderThanMaxArchiveDays() + { + try { - try - { - OnStatusMessage(MessageLevel.Info, $"Scanning for archive files older than {MaximumArchiveDays:N0} days..."); + OnStatusMessage(MessageLevel.Info, $"Scanning for archive files older than {MaximumArchiveDays:N0} days..."); - ClientDatabaseBase database = GetClientDatabase(); + ClientDatabaseBase database = GetClientDatabase(); - // Scope deletion to files within the write directories so attached/non-write folders are never - // deleted by the maximum-archive-days retention window. - string[] normalizedArchiveDirectories = GetNormalizedArchiveDirectories(); + // Scope deletion to files within the write directories so attached/non-write folders are never + // deleted by the maximum-archive-days retention window. + string[] normalizedArchiveDirectories = GetNormalizedArchiveDirectories(); - // Get list of files that have both a start time and an end time that are greater than the maximum - // archive days. We check both start and end times since devices with an inaccurate GPS clock can - // provide bad time when out of range timestamps are not configured to be filtered and you don't - // want to accidentally delete a file with otherwise in-range data. - ArchiveDetails[] filesToDelete = database.GetAllAttachedFiles().Where(file => - FilePathTargetsArchiveDirectories(file.FileName, normalizedArchiveDirectories) && - (DateTime.UtcNow - file.StartTime).TotalDays > MaximumArchiveDays && - (DateTime.UtcNow - file.EndTime).TotalDays > MaximumArchiveDays).ToArray(); + // Get list of files that have both a start time and an end time that are greater than the maximum + // archive days. We check both start and end times since devices with an inaccurate GPS clock can + // provide bad time when out of range timestamps are not configured to be filtered and you don't + // want to accidentally delete a file with otherwise in-range data. + ArchiveDetails[] filesToDelete = database.GetAllAttachedFiles().Where(file => + FilePathTargetsArchiveDirectories(file.FileName, normalizedArchiveDirectories) && + (DateTime.UtcNow - file.StartTime).TotalDays > MaximumArchiveDays && + (DateTime.UtcNow - file.EndTime).TotalDays > MaximumArchiveDays).ToArray(); - if (filesToDelete.Length <= 0) - return; + if (filesToDelete.Length <= 0) + return; - database.DeleteFiles(filesToDelete.Select(file => file.Id).ToList()); + database.DeleteFiles(filesToDelete.Select(file => file.Id).ToList()); - OnStatusMessage(MessageLevel.Info, $"Deleted the following archive files that were older than {MaximumArchiveDays:N0} days:\r\n {filesToDelete.Select(file => FilePath.TrimFileName(file.FileName, 75)).ToDelimitedString($"{Environment.NewLine} ")}"); - } - catch (Exception ex) - { - OnProcessException(MessageLevel.Warning, new InvalidOperationException($"Failed while attempting to delete archive files older than {MaximumArchiveDays:N0} days: {ex.Message}", ex)); - } + OnStatusMessage(MessageLevel.Info, $"Deleted the following archive files that were older than {MaximumArchiveDays:N0} days:\r\n {filesToDelete.Select(file => FilePath.TrimFileName(file.FileName, 75)).ToDelimitedString($"{Environment.NewLine} ")}"); } - - private void ExecuteWildCardFileOperation(ClientDatabaseBase database, string fileName, Action> fileOperation) + catch (Exception ex) { - HashSet sourceFiles = new(FilePath.GetFileList(fileName).Select(Path.GetFullPath), StringComparer.OrdinalIgnoreCase); - List files = database.GetAllAttachedFiles().Where(file => sourceFiles.Contains(Path.GetFullPath(file.FileName))).Select(file => file.Id).ToList(); - fileOperation(files); + OnProcessException(MessageLevel.Warning, new InvalidOperationException($"Failed while attempting to delete archive files older than {MaximumArchiveDays:N0} days: {ex.Message}", ex)); } + } - private void ExecuteFileOperation(ClientDatabaseBase database, string fileName, Action> fileOperation) - { - List files = database.GetAllAttachedFiles().Where(file => Path.GetFullPath(file.FileName).Equals(fileName, StringComparison.OrdinalIgnoreCase)).Select(file => file.Id).ToList(); - fileOperation(files); - } + private void ExecuteWildCardFileOperation(ClientDatabaseBase database, string fileName, Action> fileOperation) + { + HashSet sourceFiles = new(FilePath.GetFileList(fileName).Select(Path.GetFullPath), StringComparer.OrdinalIgnoreCase); + List files = database.GetAllAttachedFiles().Where(file => sourceFiles.Contains(Path.GetFullPath(file.FileName))).Select(file => file.Id).ToList(); + fileOperation(files); + } - private void ExecuteFolderOperation(ClientDatabaseBase database, string folderName, Action> folderOperation) - { - List files = database.GetAllAttachedFiles().Where(file => Path.GetFullPath(file.FileName).StartsWith(folderName, StringComparison.OrdinalIgnoreCase)).Select(file => file.Id).ToList(); - folderOperation(files); - } + private void ExecuteFileOperation(ClientDatabaseBase database, string fileName, Action> fileOperation) + { + List files = database.GetAllAttachedFiles().Where(file => Path.GetFullPath(file.FileName).Equals(fileName, StringComparison.OrdinalIgnoreCase)).Select(file => file.Id).ToList(); + fileOperation(files); + } - private ClientDatabaseBase GetClientDatabase() - { - if (m_archive?.ClientDatabase is not null) - return m_archive.ClientDatabase; + private void ExecuteFolderOperation(ClientDatabaseBase database, string folderName, Action> folderOperation) + { + List files = database.GetAllAttachedFiles().Where(file => Path.GetFullPath(file.FileName).StartsWith(folderName, StringComparison.OrdinalIgnoreCase)).Select(file => file.Id).ToList(); + folderOperation(files); + } - throw new InvalidOperationException("Cannot execute historian operation, archive database is not open."); - } + private ClientDatabaseBase GetClientDatabase() + { + if (m_archive?.ClientDatabase is not null) + return m_archive.ClientDatabase; - private void ArchiveCurtailmentTimerElapsed(object sender, ElapsedEventArgs e) => CurtailArchiveFiles(); + throw new InvalidOperationException("Cannot execute historian operation, archive database is not open."); + } - /// - /// Attempts to connect to this . - /// - protected override void AttemptConnection() - { - // Open archive files - Dictionary settings = (DataChannel ?? DefaultDataChannel).ParseKeyValuePairs(); + private void ArchiveCurtailmentTimerElapsed(object sender, ElapsedEventArgs e) => CurtailArchiveFiles(); - if (!settings.TryGetValue("port", out string setting) || !int.TryParse(setting, out int port)) - port = DefaultPort; + /// + /// Attempts to connect to this . + /// + protected override void AttemptConnection() + { + // Open archive files + Dictionary settings = (DataChannel ?? DefaultDataChannel).ParseKeyValuePairs(); - if (!settings.TryGetValue("interface", out string networkInterfaceIP)) - networkInterfaceIP = "::0"; + if (!settings.TryGetValue("port", out string setting) || !int.TryParse(setting, out int port)) + port = DefaultPort; - Server = new HistorianServer(m_archiveInfo, port, networkInterfaceIP); - m_archive = Server[InstanceName]; + if (!settings.TryGetValue("interface", out string networkInterfaceIP)) + networkInterfaceIP = "::0"; - // Initialization of services needs to occur after files are open - m_dataServices.Initialize(); - m_replicationProviders.Initialize(); + Server = new HistorianServer(m_archiveInfo, port, networkInterfaceIP); + m_archive = Server[InstanceName]; - OnConnected(); + // Initialization of services needs to occur after files are open + m_dataServices.Initialize(); + m_replicationProviders.Initialize(); - if (m_attachedPathWatchers is not null) - { - foreach (SafeFileWatcher fileWatcher in m_attachedPathWatchers) - fileWatcher.EnableRaisingEvents = true; - } + OnConnected(); - new Thread(AttachArchiveDirectoriesAndAttachedPaths).Start(); + if (m_attachedPathWatchers is not null) + { + foreach (SafeFileWatcher fileWatcher in m_attachedPathWatchers) + fileWatcher.EnableRaisingEvents = true; } - /// - /// Attempts to disconnect from this . - /// - protected override void AttemptDisconnection() + new Thread(AttachArchiveDirectoriesAndAttachedPaths).Start(); + } + + /// + /// Attempts to disconnect from this . + /// + protected override void AttemptDisconnection() + { + if (m_attachedPathWatchers is not null) { - if (m_attachedPathWatchers is not null) - { - foreach (SafeFileWatcher fileWatcher in m_attachedPathWatchers) - fileWatcher.EnableRaisingEvents = false; - } + foreach (SafeFileWatcher fileWatcher in m_attachedPathWatchers) + fileWatcher.EnableRaisingEvents = false; + } - m_archive = null; - Server?.Dispose(); - Server = null; + m_archive = null; + Server?.Dispose(); + Server = null; - OnDisconnected(); - m_archivedMeasurements = 0; - } + OnDisconnected(); + m_archivedMeasurements = 0; + } - /// - /// Archives locally. - /// - /// Measurements to be archived. - /// Local archive is closed. - protected override void ProcessMeasurements(IMeasurement[] measurements) + /// + /// Archives locally. + /// + /// Measurements to be archived. + /// Local archive is closed. + protected override void ProcessMeasurements(IMeasurement[] measurements) + { + foreach (IMeasurement measurement in measurements) { - foreach (IMeasurement measurement in measurements) + // Validate timestamp reasonability as compared to local clock, when enabled + if (EnableTimeReasonabilityCheck) { - // Validate timestamp reasonability as compared to local clock, when enabled - if (EnableTimeReasonabilityCheck) - { - long deviation = DateTime.UtcNow.Ticks - measurement.Timestamp.Value; + long deviation = DateTime.UtcNow.Ticks - measurement.Timestamp.Value; - if (deviation < -m_futureTimeReasonabilityLimit || deviation > m_pastTimeReasonabilityLimit) - continue; - } + if (deviation < -m_futureTimeReasonabilityLimit || deviation > m_pastTimeReasonabilityLimit) + continue; + } - m_key.Timestamp = (ulong)measurement.Timestamp.Value; - m_key.PointID = measurement.Key.ID; + m_key.Timestamp = (ulong)measurement.Timestamp.Value; + m_key.PointID = measurement.Key.ID; - // Since current time-series measurements are basically all floats - values fit into first value, - // this will change as value types for time-series framework expands - m_value.Value1 = BitConvert.ToUInt64((float)measurement.AdjustedValue); - m_value.Value3 = (ulong)measurement.StateFlags; + // Since current time-series measurements are basically all floats - values fit into first value, + // this will change as value types for time-series framework expands + m_value.Value1 = BitConvert.ToUInt64((float)measurement.AdjustedValue); + m_value.Value3 = (ulong)measurement.StateFlags; - // Check to see if swinging door compression is enabled - if (SwingingDoorCompressionEnabled) + // Check to see if swinging door compression is enabled + if (SwingingDoorCompressionEnabled) + { + // Attempt to lookup compression settings for this measurement + if ((m_compressionSettings?.TryGetValue(m_key.PointID, out Tuple settings) ?? false) && settings is not null) { - // Attempt to lookup compression settings for this measurement - if ((m_compressionSettings?.TryGetValue(m_key.PointID, out Tuple settings) ?? false) && settings is not null) + // Get compression settings + int compressionMinTime = settings.Item1; + int compressionMaxTime = settings.Item2; + double compressionLimit = settings.Item3; + + // Get current swinging door compression state, creating state if needed + Tuple state = m_swingingDoorStates.GetOrAdd(m_key.PointID, _ => new Tuple(measurement, measurement, double.MinValue, double.MaxValue)); + IMeasurement currentData = measurement; + IMeasurement archivedData = state.Item1; + IMeasurement previousData = state.Item2; + double lastHighSlope = state.Item3; + double lastLowSlope = state.Item4; + double highSlope = 0.0D; + double lowSlope = 0.0D; + bool archiveData; + + // Data is to be compressed + if (compressionMinTime > 0 && currentData.Timestamp - archivedData.Timestamp < compressionMinTime) { - // Get compression settings - int compressionMinTime = settings.Item1; - int compressionMaxTime = settings.Item2; - double compressionLimit = settings.Item3; - - // Get current swinging door compression state, creating state if needed - Tuple state = m_swingingDoorStates.GetOrAdd(m_key.PointID, _ => new Tuple(measurement, measurement, double.MinValue, double.MaxValue)); - IMeasurement currentData = measurement; - IMeasurement archivedData = state.Item1; - IMeasurement previousData = state.Item2; - double lastHighSlope = state.Item3; - double lastLowSlope = state.Item4; - double highSlope = 0.0D; - double lowSlope = 0.0D; - bool archiveData; - - // Data is to be compressed - if (compressionMinTime > 0 && currentData.Timestamp - archivedData.Timestamp < compressionMinTime) - { - // CompressionMinTime is in effect - archiveData = false; - } - else if (currentData.StateFlags != archivedData.StateFlags || currentData.StateFlags != previousData.StateFlags || compressionMaxTime > 0 && previousData.Value - archivedData.Timestamp > compressionMaxTime) - { - // Quality changed or CompressionMaxTime is exceeded - archiveData = true; - } - else - { - // Perform a compression test - highSlope = (currentData.Value - (archivedData.Value + compressionLimit)) / (currentData.Timestamp - archivedData.Timestamp); - lowSlope = (currentData.Value - (archivedData.Value - compressionLimit)) / (currentData.Timestamp - archivedData.Timestamp); - double slope = (currentData.Value - archivedData.Value) / (currentData.Timestamp - archivedData.Timestamp); - - if (highSlope >= lastHighSlope) - lastHighSlope = highSlope; + // CompressionMinTime is in effect + archiveData = false; + } + else if (currentData.StateFlags != archivedData.StateFlags || currentData.StateFlags != previousData.StateFlags || compressionMaxTime > 0 && previousData.Value - archivedData.Timestamp > compressionMaxTime) + { + // Quality changed or CompressionMaxTime is exceeded + archiveData = true; + } + else + { + // Perform a compression test + highSlope = (currentData.Value - (archivedData.Value + compressionLimit)) / (currentData.Timestamp - archivedData.Timestamp); + lowSlope = (currentData.Value - (archivedData.Value - compressionLimit)) / (currentData.Timestamp - archivedData.Timestamp); + double slope = (currentData.Value - archivedData.Value) / (currentData.Timestamp - archivedData.Timestamp); - if (lowSlope <= lastLowSlope) - lastLowSlope = lowSlope; + if (highSlope >= lastHighSlope) + lastHighSlope = highSlope; - archiveData = slope <= lastHighSlope || slope >= lastLowSlope; - } + if (lowSlope <= lastLowSlope) + lastLowSlope = lowSlope; - // Update swinging door compression state - m_swingingDoorStates[m_key.PointID] = new Tuple - ( - archiveData ? currentData : archivedData, - currentData, - archiveData ? highSlope: lastHighSlope, - archiveData ? lowSlope : lastLowSlope - ); - - // Continue to next point if this point does not need to be archived - if (!archiveData) - continue; + archiveData = slope <= lastHighSlope || slope >= lastLowSlope; } - } - m_archive.Write(m_key, m_value); + // Update swinging door compression state + m_swingingDoorStates[m_key.PointID] = new Tuple + ( + archiveData ? currentData : archivedData, + currentData, + archiveData ? highSlope: lastHighSlope, + archiveData ? lowSlope : lastLowSlope + ); + + // Continue to next point if this point does not need to be archived + if (!archiveData) + continue; + } } - m_archivedMeasurements += measurements.Length; + m_archive.Write(m_key, m_value); } - private void AttachArchiveDirectoriesAndAttachedPaths() + m_archivedMeasurements += measurements.Length; + } + + private void AttachArchiveDirectoriesAndAttachedPaths() + { + try { - try - { - ClientDatabaseBase clientDatabase = GetClientDatabase(); + ClientDatabaseBase clientDatabase = GetClientDatabase(); - string[] archivedirectories = m_archiveDirectories ?? []; - string[] attachedPaths = m_attachedPaths ?? []; + string[] archivedirectories = m_archiveDirectories ?? []; + string[] attachedPaths = m_attachedPaths ?? []; - List files = archivedirectories - .Concat(attachedPaths) - .SelectMany(dir => FilePath.EnumerateFiles(dir, exceptionHandler: ex => - OnProcessException(MessageLevel.Error, ex, nameof(AttachArchiveDirectoriesAndAttachedPaths)))) - .ToList(); + List files = archivedirectories + .Concat(attachedPaths) + .SelectMany(dir => FilePath.EnumerateFiles(dir, exceptionHandler: ex => + OnProcessException(MessageLevel.Error, ex, nameof(AttachArchiveDirectoriesAndAttachedPaths)))) + .ToList(); - string[] filePtr = new string[1]; + string[] filePtr = new string[1]; - foreach (string file in files) - { - if (!Enabled) - return; + foreach (string file in files) + { + if (!Enabled) + return; - filePtr[0] = file; + filePtr[0] = file; - try - { - clientDatabase.AttachFilesOrPaths(filePtr); - } - catch (Exception ex) - { - OnProcessException(MessageLevel.Error, new InvalidOperationException( + try + { + clientDatabase.AttachFilesOrPaths(filePtr); + } + catch (Exception ex) + { + OnProcessException(MessageLevel.Error, new InvalidOperationException( $"{Name} failed while attempting to attach to file \"{file}\": {ex.Message}", ex), - nameof(AttachArchiveDirectoriesAndAttachedPaths)); - } + nameof(AttachArchiveDirectoriesAndAttachedPaths)); } } - catch (Exception ex) - { - OnProcessException(MessageLevel.Critical, new InvalidOperationException($"{Name} failed while attempting to attach to existing archives: {ex.Message}", ex)); - } } - - private SafeFileWatcher WatchPath(string path) + catch (Exception ex) { - SafeFileWatcher fileWatcher = new(path, "*.d2*"); + OnProcessException(MessageLevel.Critical, new InvalidOperationException($"{Name} failed while attempting to attach to existing archives: {ex.Message}", ex)); + } + } + + private SafeFileWatcher WatchPath(string path) + { + SafeFileWatcher fileWatcher = new(path, "*.d2*"); - fileWatcher.Created += (_, args) => ThreadPool.QueueUserWorkItem(state => + fileWatcher.Created += (_, args) => ThreadPool.QueueUserWorkItem(state => { ClientDatabaseBase clientDatabase = GetClientDatabase(); string[] filePtr = [(string)state]; @@ -1398,539 +1497,543 @@ private SafeFileWatcher WatchPath(string path) }, args.FullPath); - fileWatcher.InternalBufferSize = 65536; + fileWatcher.InternalBufferSize = 65536; - return fileWatcher; - } + return fileWatcher; + } + + private TimeBucketBloomFilterSettings CreateBloomFilterSettings() + { + return new TimeBucketBloomFilterSettings( + EnableTimeBucketBloomFilter, + BloomFilterBucketDurationSeconds, + BloomFilterFalsePositiveProbability, + BloomFilterMaximumIndexSizeMB); + } - private void DataServices_AdapterCreated(object sender, EventArgs e) => - e.Argument.SettingsCategory = $"{InstanceName.ToLowerInvariant()}{e.Argument.SettingsCategory}"; + private void DataServices_AdapterCreated(object sender, EventArgs e) => + e.Argument.SettingsCategory = $"{InstanceName.ToLowerInvariant()}{e.Argument.SettingsCategory}"; - private void DataServices_AdapterLoaded(object sender, EventArgs e) - { - e.Argument.Archive = m_archive; - e.Argument.ServiceProcessException += DataServices_ServiceProcessException; + private void DataServices_AdapterLoaded(object sender, EventArgs e) + { + e.Argument.Archive = m_archive; + e.Argument.ServiceProcessException += DataServices_ServiceProcessException; - OnStatusMessage(MessageLevel.Info, $"{e.Argument.GetType().Name} has been loaded."); - } + OnStatusMessage(MessageLevel.Info, $"{e.Argument.GetType().Name} has been loaded."); + } - private void DataServices_AdapterUnloaded(object sender, EventArgs e) - { - e.Argument.Archive = null; - e.Argument.ServiceProcessException -= DataServices_ServiceProcessException; + private void DataServices_AdapterUnloaded(object sender, EventArgs e) + { + e.Argument.Archive = null; + e.Argument.ServiceProcessException -= DataServices_ServiceProcessException; - OnStatusMessage(MessageLevel.Info, $"{e.Argument.GetType().Name} has been unloaded."); - } + OnStatusMessage(MessageLevel.Info, $"{e.Argument.GetType().Name} has been unloaded."); + } - private void ReplicationProviders_AdapterCreated(object sender, EventArgs e) => - e.Argument.SettingsCategory = InstanceName.ToLowerInvariant() + e.Argument.SettingsCategory; + private void ReplicationProviders_AdapterCreated(object sender, EventArgs e) => + e.Argument.SettingsCategory = InstanceName.ToLowerInvariant() + e.Argument.SettingsCategory; - private void ReplicationProviders_AdapterLoaded(object sender, EventArgs e) - { - e.Argument.ReplicationStart += ReplicationProvider_ReplicationStart; - e.Argument.ReplicationComplete += ReplicationProvider_ReplicationComplete; - e.Argument.ReplicationProgress += ReplicationProvider_ReplicationProgress; - e.Argument.ReplicationException += ReplicationProvider_ReplicationException; + private void ReplicationProviders_AdapterLoaded(object sender, EventArgs e) + { + e.Argument.ReplicationStart += ReplicationProvider_ReplicationStart; + e.Argument.ReplicationComplete += ReplicationProvider_ReplicationComplete; + e.Argument.ReplicationProgress += ReplicationProvider_ReplicationProgress; + e.Argument.ReplicationException += ReplicationProvider_ReplicationException; - OnStatusMessage(MessageLevel.Info, $"{e.Argument.GetType().Name} has been loaded."); - } + OnStatusMessage(MessageLevel.Info, $"{e.Argument.GetType().Name} has been loaded."); + } - private void ReplicationProviders_AdapterUnloaded(object sender, EventArgs e) - { - e.Argument.ReplicationStart -= ReplicationProvider_ReplicationStart; - e.Argument.ReplicationComplete -= ReplicationProvider_ReplicationComplete; - e.Argument.ReplicationProgress -= ReplicationProvider_ReplicationProgress; - e.Argument.ReplicationException -= ReplicationProvider_ReplicationException; + private void ReplicationProviders_AdapterUnloaded(object sender, EventArgs e) + { + e.Argument.ReplicationStart -= ReplicationProvider_ReplicationStart; + e.Argument.ReplicationComplete -= ReplicationProvider_ReplicationComplete; + e.Argument.ReplicationProgress -= ReplicationProvider_ReplicationProgress; + e.Argument.ReplicationException -= ReplicationProvider_ReplicationException; - OnStatusMessage(MessageLevel.Info, $"{e.Argument.GetType().Name} has been unloaded."); - } + OnStatusMessage(MessageLevel.Info, $"{e.Argument.GetType().Name} has been unloaded."); + } - private void AdapterLoader_AdapterLoadException(object sender, EventArgs e) => - OnProcessException(MessageLevel.Warning, e.Argument); + private void AdapterLoader_AdapterLoadException(object sender, EventArgs e) => + OnProcessException(MessageLevel.Warning, e.Argument); - private void DataServices_ServiceProcessException(object sender, EventArgs e) => - OnProcessException(MessageLevel.Warning, e.Argument); + private void DataServices_ServiceProcessException(object sender, EventArgs e) => + OnProcessException(MessageLevel.Warning, e.Argument); - private void ReplicationProvider_ReplicationStart(object sender, EventArgs e) => - OnStatusMessage(MessageLevel.Info, $"{sender.GetType().Name} has started archive replication..."); + private void ReplicationProvider_ReplicationStart(object sender, EventArgs e) => + OnStatusMessage(MessageLevel.Info, $"{sender.GetType().Name} has started archive replication..."); - private void ReplicationProvider_ReplicationComplete(object sender, EventArgs e) => - OnStatusMessage(MessageLevel.Info, $"{sender.GetType().Name} has finished archive replication."); + private void ReplicationProvider_ReplicationComplete(object sender, EventArgs e) => + OnStatusMessage(MessageLevel.Info, $"{sender.GetType().Name} has finished archive replication."); - private void ReplicationProvider_ReplicationProgress(object sender, EventArgs> e) => - OnStatusMessage(MessageLevel.Info, $"{sender.GetType().Name} has replicated archive file {e.Argument.ProgressMessage}."); + private void ReplicationProvider_ReplicationProgress(object sender, EventArgs> e) => + OnStatusMessage(MessageLevel.Info, $"{sender.GetType().Name} has replicated archive file {e.Argument.ProgressMessage}."); - private void ReplicationProvider_ReplicationException(object sender, EventArgs e) => - OnProcessException(MessageLevel.Warning, e.Argument); + private void ReplicationProvider_ReplicationException(object sender, EventArgs e) => + OnProcessException(MessageLevel.Warning, e.Argument); - #endregion + #endregion - #region [ Static ] + #region [ Static ] - // Static Fields + // Static Fields - /// - /// Accesses local output adapter instances (normally only one). - /// - public static readonly ConcurrentDictionary Instances = new(StringComparer.OrdinalIgnoreCase); + /// + /// Accesses local output adapter instances (normally only one). + /// + public static readonly ConcurrentDictionary Instances = new(StringComparer.OrdinalIgnoreCase); - private static int s_virtualProtocolID; + private static int s_virtualProtocolID; - // Static Constructor + // Static Constructor - static LocalOutputAdapter() - { - CategorizedSettingsElementCollection systemSettings = ConfigurationFile.Current.Settings["systemSettings"]; + static LocalOutputAdapter() + { + CategorizedSettingsElementCollection systemSettings = ConfigurationFile.Current.Settings["systemSettings"]; - systemSettings.Add("MemoryPoolSize", "0.0", "The fixed memory pool size in Gigabytes. Leave at zero for dynamically calculated setting."); - systemSettings.Add("MemoryPoolTargetUtilization", "Low", "The target utilization level for the memory pool. One of 'Low', 'Medium', or 'High'."); + systemSettings.Add("MemoryPoolSize", "0.0", "The fixed memory pool size in Gigabytes. Leave at zero for dynamically calculated setting."); + systemSettings.Add("MemoryPoolTargetUtilization", "Low", "The target utilization level for the memory pool. One of 'Low', 'Medium', or 'High'."); - // Set maximum buffer size - double memoryPoolSize = systemSettings["MemoryPoolSize"].ValueAs(0.0D); + // Set maximum buffer size + double memoryPoolSize = systemSettings["MemoryPoolSize"].ValueAs(0.0D); - if (memoryPoolSize > 0.0D) - Globals.MemoryPool.SetMaximumBufferSize((long)(memoryPoolSize * SI2.Giga)); + if (memoryPoolSize > 0.0D) + Globals.MemoryPool.SetMaximumBufferSize((long)(memoryPoolSize * SI2.Giga)); - if (!Enum.TryParse(systemSettings["MemoryPoolTargetUtilization"].Value, false, out TargetUtilizationLevels targetLevel)) - targetLevel = TargetUtilizationLevels.High; + if (!Enum.TryParse(systemSettings["MemoryPoolTargetUtilization"].Value, false, out TargetUtilizationLevels targetLevel)) + targetLevel = TargetUtilizationLevels.High; - Globals.MemoryPool.SetTargetUtilizationLevel(targetLevel); - } + Globals.MemoryPool.SetTargetUtilizationLevel(targetLevel); + } - // Static Methods + // Static Methods - // ReSharper disable UnusedMember.Local - // ReSharper disable UnusedParameter.Local - private static void OptimizeLocalHistorianSettings(AdoDataConnection connection, string nodeIDQueryString, ulong trackingVersion, string arguments, Action statusMessage, Action processException) + // ReSharper disable UnusedMember.Local + // ReSharper disable UnusedParameter.Local + private static void OptimizeLocalHistorianSettings(AdoDataConnection connection, string nodeIDQueryString, ulong trackingVersion, string arguments, Action statusMessage, Action processException) + { + TableOperations inputAdapterTable = new(connection); + TableOperations actionAdapterTable = new(connection); + TableOperations outputAdapterTable = new(connection); + TableOperations historianTable = new(connection); + TableOperations measurementTable = new(connection); + TableOperations deviceGroupClassTable = new(connection); + + // Make sure setting exists to allow user to by-pass local historian optimizations at startup + ConfigurationFile configFile = ConfigurationFile.Current; + CategorizedSettingsElementCollection settings = configFile.Settings["systemSettings"]; + settings.Add("OptimizeLocalHistorianSettings", true, "Determines if the defined local historians will have their settings optimized at startup"); + + // See if this node should optimize local historian settings + if (settings["OptimizeLocalHistorianSettings"].ValueAsBoolean()) { - TableOperations inputAdapterTable = new(connection); - TableOperations actionAdapterTable = new(connection); - TableOperations outputAdapterTable = new(connection); - TableOperations historianTable = new(connection); - TableOperations measurementTable = new(connection); - TableOperations deviceGroupClassTable = new(connection); - - // Make sure setting exists to allow user to by-pass local historian optimizations at startup - ConfigurationFile configFile = ConfigurationFile.Current; - CategorizedSettingsElementCollection settings = configFile.Settings["systemSettings"]; - settings.Add("OptimizeLocalHistorianSettings", true, "Determines if the defined local historians will have their settings optimized at startup"); - - // See if this node should optimize local historian settings - if (settings["OptimizeLocalHistorianSettings"].ValueAsBoolean()) - { - statusMessage("Optimizing settings for local historians..."); + statusMessage("Optimizing settings for local historians..."); - // Load the defined local system historians - IEnumerable historians = connection.RetrieveData($"SELECT AdapterName FROM RuntimeHistorian WHERE NodeID = {nodeIDQueryString} AND TypeName = 'openHistorian.Adapters.LocalOutputAdapter'").AsEnumerable(); + // Load the defined local system historians + IEnumerable historians = connection.RetrieveData($"SELECT AdapterName FROM RuntimeHistorian WHERE NodeID = {nodeIDQueryString} AND TypeName = 'openHistorian.Adapters.LocalOutputAdapter'").AsEnumerable(); - List validHistorians = []; + List validHistorians = []; - // Apply settings optimizations to local historians - foreach (DataRow row in historians) - { - string acronym = row.Field("AdapterName").ToLower(); - validHistorians.Add(acronym); - } + // Apply settings optimizations to local historians + foreach (DataRow row in historians) + { + string acronym = row.Field("AdapterName").ToLower(); + validHistorians.Add(acronym); + } - // Local statics historian is valid regardless of historian type - if (!validHistorians.Contains("stat")) - validHistorians.Add("stat"); + // Local statics historian is valid regardless of historian type + if (!validHistorians.Contains("stat")) + validHistorians.Add("stat"); - // Sort valid historians for binary search - validHistorians.Sort(); + // Sort valid historians for binary search + validHistorians.Sort(); - // Create a list to track categories to remove - HashSet categoriesToRemove = []; + // Create a list to track categories to remove + HashSet categoriesToRemove = []; - // Search for unused settings categories - foreach (PropertyInformation info in configFile.Settings.ElementInformation.Properties) - { - string name = info.Name; + // Search for unused settings categories + foreach (PropertyInformation info in configFile.Settings.ElementInformation.Properties) + { + string name = info.Name; - if (name.EndsWith("AdoMetadataProvider") && validHistorians.BinarySearch(name.Substring(0, name.IndexOf("AdoMetadataProvider", StringComparison.Ordinal))) < 0) - categoriesToRemove.Add(name); + if (name.EndsWith("AdoMetadataProvider") && validHistorians.BinarySearch(name.Substring(0, name.IndexOf("AdoMetadataProvider", StringComparison.Ordinal))) < 0) + categoriesToRemove.Add(name); - if (name.EndsWith("OleDbMetadataProvider") && validHistorians.BinarySearch(name.Substring(0, name.IndexOf("OleDbMetadataProvider", StringComparison.Ordinal))) < 0) - categoriesToRemove.Add(name); + if (name.EndsWith("OleDbMetadataProvider") && validHistorians.BinarySearch(name.Substring(0, name.IndexOf("OleDbMetadataProvider", StringComparison.Ordinal))) < 0) + categoriesToRemove.Add(name); - if (name.EndsWith("RestWebServiceMetadataProvider") && validHistorians.BinarySearch(name.Substring(0, name.IndexOf("RestWebServiceMetadataProvider", StringComparison.Ordinal))) < 0) - categoriesToRemove.Add(name); + if (name.EndsWith("RestWebServiceMetadataProvider") && validHistorians.BinarySearch(name.Substring(0, name.IndexOf("RestWebServiceMetadataProvider", StringComparison.Ordinal))) < 0) + categoriesToRemove.Add(name); - if (name.EndsWith("MetadataService") && validHistorians.BinarySearch(name.Substring(0, name.IndexOf("MetadataService", StringComparison.Ordinal))) < 0) - categoriesToRemove.Add(name); + if (name.EndsWith("MetadataService") && validHistorians.BinarySearch(name.Substring(0, name.IndexOf("MetadataService", StringComparison.Ordinal))) < 0) + categoriesToRemove.Add(name); - if (name.EndsWith("TimeSeriesDataService") && validHistorians.BinarySearch(name.Substring(0, name.IndexOf("TimeSeriesDataService", StringComparison.Ordinal))) < 0) - categoriesToRemove.Add(name); + if (name.EndsWith("TimeSeriesDataService") && validHistorians.BinarySearch(name.Substring(0, name.IndexOf("TimeSeriesDataService", StringComparison.Ordinal))) < 0) + categoriesToRemove.Add(name); - if (name.EndsWith("HadoopReplicationProvider") && validHistorians.BinarySearch(name.Substring(0, name.IndexOf("HadoopReplicationProvider", StringComparison.Ordinal))) < 0) - categoriesToRemove.Add(name); - } + if (name.EndsWith("HadoopReplicationProvider") && validHistorians.BinarySearch(name.Substring(0, name.IndexOf("HadoopReplicationProvider", StringComparison.Ordinal))) < 0) + categoriesToRemove.Add(name); + } - if (categoriesToRemove.Count > 0) - { - statusMessage("Removing unused local historian configuration settings..."); + if (categoriesToRemove.Count > 0) + { + statusMessage("Removing unused local historian configuration settings..."); - // Remove any unused settings categories - foreach (string category in categoriesToRemove) - configFile.Settings.Remove(category); - } + // Remove any unused settings categories + foreach (string category in categoriesToRemove) + configFile.Settings.Remove(category); + } + + // Save any applied changes + configFile.Save(); - // Save any applied changes - configFile.Save(); + // Convert valid historian list to proper acronym casing (and remove stat historian) + IEnumerable historianAcronyms = validHistorians.Where(value => !value.Equals("stat")).Select(value => value.ToUpperInvariant()); - // Convert valid historian list to proper acronym casing (and remove stat historian) - IEnumerable historianAcronyms = validHistorians.Where(value => !value.Equals("stat")).Select(value => value.ToUpperInvariant()); + foreach (string historianAcronym in historianAcronyms) + { + Historian historianAdapter = historianTable.QueryRecordWhere("Acronym = {0}", historianAcronym); + CustomOutputAdapter outputAdapter = null; - foreach (string historianAcronym in historianAcronyms) + if (historianAdapter is null) { - Historian historianAdapter = historianTable.QueryRecordWhere("Acronym = {0}", historianAcronym); - CustomOutputAdapter outputAdapter = null; + outputAdapter = outputAdapterTable.QueryRecordWhere("AdapterName = {0}", historianAcronym); - if (historianAdapter is null) + if (outputAdapter is null) { - outputAdapter = outputAdapterTable.QueryRecordWhere("AdapterName = {0}", historianAcronym); - - if (outputAdapter is null) - { - statusMessage($"WARNING: Could not check for associated historian input reader after failing to find historian adapter \"{historianAcronym}\" in either \"Historian\" or \"CustomOutputAdapter\" table. Record was just found in the \"RuntimeHistorian\" view, verify schema integrity."); - continue; - } + statusMessage($"WARNING: Could not check for associated historian input reader after failing to find historian adapter \"{historianAcronym}\" in either \"Historian\" or \"CustomOutputAdapter\" table. Record was just found in the \"RuntimeHistorian\" view, verify schema integrity."); + continue; } + } - // Parse connection string to get adapter settings for historian - Dictionary adapterSettings = (historianAdapter?.ConnectionString ?? outputAdapter?.ConnectionString ?? "").ParseKeyValuePairs(); - - // Get instance name for historian adapter - if (!adapterSettings.TryGetValue("instanceName", out string instanceName) || string.IsNullOrWhiteSpace(instanceName)) - instanceName = historianAcronym; + // Parse connection string to get adapter settings for historian + Dictionary adapterSettings = (historianAdapter?.ConnectionString ?? outputAdapter?.ConnectionString ?? "").ParseKeyValuePairs(); - // Reader name will always be associated with instance name of historian (this often matches historian adapter name) - string readerName = $"{instanceName}READER"; - CustomInputAdapter inputAdapter = inputAdapterTable.QueryRecordWhere("AdapterName = {0}", readerName); + // Get instance name for historian adapter + if (!adapterSettings.TryGetValue("instanceName", out string instanceName) || string.IsNullOrWhiteSpace(instanceName)) + instanceName = historianAcronym; - if (inputAdapter is not null) - continue; + // Reader name will always be associated with instance name of historian (this often matches historian adapter name) + string readerName = $"{instanceName}READER"; + CustomInputAdapter inputAdapter = inputAdapterTable.QueryRecordWhere("AdapterName = {0}", readerName); - ushort? port = null; + if (inputAdapter is not null) + continue; - // Check if historian has defined a custom data channel, if so attempt to parse port number - if (adapterSettings.ContainsKey(nameof(DataChannel))) - { - Dictionary configSettings = adapterSettings[nameof(DataChannel)].ParseKeyValuePairs(); + ushort? port = null; - if (configSettings.ContainsKey("port") && ushort.TryParse(configSettings["port"], out ushort value)) - port = value; - } + // Check if historian has defined a custom data channel, if so attempt to parse port number + if (adapterSettings.ContainsKey(nameof(DataChannel))) + { + Dictionary configSettings = adapterSettings[nameof(DataChannel)].ParseKeyValuePairs(); - string serverConnection = port is null ? "127.0.0.1" : $"127.0.0.1:{port.Value}"; - - // Add new associated historian input adapter reader to handle temporal queries - inputAdapter = inputAdapterTable.NewRecord(); - inputAdapter.NodeID = new Guid(nodeIDQueryString.RemoveCharacter('\'')); - inputAdapter.AdapterName = $"{historianAcronym}READER"; - inputAdapter.AssemblyName = "openHistorian.Adapters.dll"; - inputAdapter.TypeName = "openHistorian.Adapters.LocalInputAdapter"; - inputAdapter.ConnectionString = $"{nameof(LocalInputAdapter.InstanceName)}={instanceName}; {nameof(LocalInputAdapter.HistorianServer)}={serverConnection}; ConnectOnDemand=true"; - inputAdapter.LoadOrder = 0; - inputAdapter.Enabled = true; - inputAdapterTable.AddNewRecord(inputAdapter); + if (configSettings.ContainsKey("port") && ushort.TryParse(configSettings["port"], out ushort value)) + port = value; } - } - // TODO: Remove this code when device IDs have been updated to use GUIDs: - // Make sure gateway protocol data publishers filter out device groups from metadata since groups reference local device IDs - const string DefaultDeviceFilter = "FROM DeviceDetail WHERE IsConcentrator = 0;"; - const string GSFDataPublisherTypeName = $"{nameof(GSF)}.{nameof(GSF.TimeSeries)}.{nameof(GSF.TimeSeries.Transport)}.{nameof(GSF.TimeSeries.Transport.DataPublisher)}"; - const string GSFMetadataTables = nameof(GSFDataPublisher.MetadataTables); - const string STTPDataPublisherTypeName = $"{nameof(sttp)}.{nameof(sttp.DataPublisher)}"; - const string STTPMetadataTables = nameof(STTPDataPublisher.MetadataTables); - int virtualProtocolID = s_virtualProtocolID != 0 ? s_virtualProtocolID : s_virtualProtocolID = connection.ExecuteScalar("SELECT ID FROM Protocol WHERE Acronym='VirtualInput'"); - string deviceGroupFilter = $"FROM DeviceDetail WHERE IsConcentrator = 0 AND NOT (ProtocolID = {virtualProtocolID} AND AccessID = {DeviceGroup.DefaultAccessID});"; + string serverConnection = port is null ? "127.0.0.1" : $"127.0.0.1:{port.Value}"; + + // Add new associated historian input adapter reader to handle temporal queries + inputAdapter = inputAdapterTable.NewRecord(); + inputAdapter.NodeID = new Guid(nodeIDQueryString.RemoveCharacter('\'')); + inputAdapter.AdapterName = $"{historianAcronym}READER"; + inputAdapter.AssemblyName = "openHistorian.Adapters.dll"; + inputAdapter.TypeName = "openHistorian.Adapters.LocalInputAdapter"; + inputAdapter.ConnectionString = $"{nameof(LocalInputAdapter.InstanceName)}={instanceName}; {nameof(LocalInputAdapter.HistorianServer)}={serverConnection}; ConnectOnDemand=true"; + inputAdapter.LoadOrder = 0; + inputAdapter.Enabled = true; + inputAdapterTable.AddNewRecord(inputAdapter); + } + } - IEnumerable dataPublisherAdapters = actionAdapterTable.QueryRecordsWhere($"NodeID = {nodeIDQueryString} AND TypeName LIKE '%.DataPublisher'"); + // TODO: Remove this code when device IDs have been updated to use GUIDs: + // Make sure gateway protocol data publishers filter out device groups from metadata since groups reference local device IDs + const string DefaultDeviceFilter = "FROM DeviceDetail WHERE IsConcentrator = 0;"; + const string GSFDataPublisherTypeName = $"{nameof(GSF)}.{nameof(GSF.TimeSeries)}.{nameof(GSF.TimeSeries.Transport)}.{nameof(GSF.TimeSeries.Transport.DataPublisher)}"; + const string GSFMetadataTables = nameof(GSFDataPublisher.MetadataTables); + const string STTPDataPublisherTypeName = $"{nameof(sttp)}.{nameof(sttp.DataPublisher)}"; + const string STTPMetadataTables = nameof(STTPDataPublisher.MetadataTables); + int virtualProtocolID = s_virtualProtocolID != 0 ? s_virtualProtocolID : s_virtualProtocolID = connection.ExecuteScalar("SELECT ID FROM Protocol WHERE Acronym='VirtualInput'"); + string deviceGroupFilter = $"FROM DeviceDetail WHERE IsConcentrator = 0 AND NOT (ProtocolID = {virtualProtocolID} AND AccessID = {DeviceGroup.DefaultAccessID});"; - foreach (CustomActionAdapter actionAdapter in dataPublisherAdapters) - { - if (actionAdapter is null) - continue; + IEnumerable dataPublisherAdapters = actionAdapterTable.QueryRecordsWhere($"NodeID = {nodeIDQueryString} AND TypeName LIKE '%.DataPublisher'"); - Dictionary connectionString = actionAdapter.ConnectionString?.ParseKeyValuePairs() ?? new Dictionary(StringComparer.OrdinalIgnoreCase); - bool connectionStringUpdated = false; + foreach (CustomActionAdapter actionAdapter in dataPublisherAdapters) + { + if (actionAdapter is null) + continue; - switch (actionAdapter.TypeName) - { - case GSFDataPublisherTypeName: - if (connectionString.ContainsKey(GSFMetadataTables)) - continue; - - connectionString[GSFMetadataTables] = GSFDataPublisher.DefaultMetadataTables.Replace(DefaultDeviceFilter, deviceGroupFilter); - connectionStringUpdated = true; - break; - case STTPDataPublisherTypeName: - // For down-stream identity matched metadata replication, check option to allow device groups replicate - this will OK when device IDs match, e.g., - // when STTP data subscribers enable the "useIdentityInsertsForMetadata" option - if (connectionString.TryGetValue("allowDeviceGroupReplication", out string setting) && setting.ParseBoolean()) - { - connectionString.Remove(STTPMetadataTables); - } - else - { - if (connectionString.ContainsKey(STTPMetadataTables)) - continue; + Dictionary connectionString = actionAdapter.ConnectionString?.ParseKeyValuePairs() ?? new Dictionary(StringComparer.OrdinalIgnoreCase); + bool connectionStringUpdated = false; - connectionString[STTPMetadataTables] = STTPDataPublisher.DefaultMetadataTables.Replace(DefaultDeviceFilter, deviceGroupFilter); - connectionStringUpdated = true; - } - break; - default: + switch (actionAdapter.TypeName) + { + case GSFDataPublisherTypeName: + if (connectionString.ContainsKey(GSFMetadataTables)) continue; - } + + connectionString[GSFMetadataTables] = GSFDataPublisher.DefaultMetadataTables.Replace(DefaultDeviceFilter, deviceGroupFilter); + connectionStringUpdated = true; + break; + case STTPDataPublisherTypeName: + // For down-stream identity matched metadata replication, check option to allow device groups replicate - this will OK when device IDs match, e.g., + // when STTP data subscribers enable the "useIdentityInsertsForMetadata" option + if (connectionString.TryGetValue("allowDeviceGroupReplication", out string setting) && setting.ParseBoolean()) + { + connectionString.Remove(STTPMetadataTables); + } + else + { + if (connectionString.ContainsKey(STTPMetadataTables)) + continue; - if (!connectionStringUpdated) + connectionString[STTPMetadataTables] = STTPDataPublisher.DefaultMetadataTables.Replace(DefaultDeviceFilter, deviceGroupFilter); + connectionStringUpdated = true; + } + break; + default: continue; - - actionAdapter.ConnectionString = connectionString.JoinKeyValuePairs(); - actionAdapterTable.UpdateRecord(actionAdapter); } - // Make sure default device group classes are defined - int count = deviceGroupClassTable.QueryRecordCountWhere("Longitude = {0} and Latitude = {0}", DeviceGroupClass.DefaultGeoID); + if (!connectionStringUpdated) + continue; + + actionAdapter.ConnectionString = connectionString.JoinKeyValuePairs(); + actionAdapterTable.UpdateRecord(actionAdapter); + } + + // Make sure default device group classes are defined + int count = deviceGroupClassTable.QueryRecordCountWhere("Longitude = {0} and Latitude = {0}", DeviceGroupClass.DefaultGeoID); - if (count == 0) + if (count == 0) + { + // Add default device group classes + List classOptions = + [ + "Region", + "Substation", + "Generation", + "Other" + ]; + + foreach (string classOption in classOptions) { - // Add default device group classes - List classOptions = - [ - "Region", - "Substation", - "Generation", - "Other" - ]; - - foreach (string classOption in classOptions) - { - DeviceGroupClass deviceGroupClass = deviceGroupClassTable.NewRecord(); - deviceGroupClass.Acronym = classOption.ToUpperInvariant(); - deviceGroupClass.Name = classOption; - deviceGroupClassTable.AddNewRecord(deviceGroupClass); - } + DeviceGroupClass deviceGroupClass = deviceGroupClassTable.NewRecord(); + deviceGroupClass.Acronym = classOption.ToUpperInvariant(); + deviceGroupClass.Name = classOption; + deviceGroupClassTable.AddNewRecord(deviceGroupClass); } + } - IEnumerable dynamicCalculators = actionAdapterTable.QueryRecordsWhere("TypeName = 'DynamicCalculator.DynamicCalculator'"); + IEnumerable dynamicCalculators = actionAdapterTable.QueryRecordsWhere("TypeName = 'DynamicCalculator.DynamicCalculator'"); - foreach (CustomActionAdapter dynamicCalculator in dynamicCalculators) - { - if (dynamicCalculator is null) - continue; + foreach (CustomActionAdapter dynamicCalculator in dynamicCalculators) + { + if (dynamicCalculator is null) + continue; - Dictionary connectionString = dynamicCalculator.ConnectionString?.ParseKeyValuePairs() ?? new Dictionary(StringComparer.OrdinalIgnoreCase); + Dictionary connectionString = dynamicCalculator.ConnectionString?.ParseKeyValuePairs() ?? new Dictionary(StringComparer.OrdinalIgnoreCase); - // variableList={FREQ=bef9f097-d6f8-4b4f-9ce4-97ffa30a08e9}; expressionText=FREQ-60; framesPerSecond=30; lagTime=6; leadTime=3; outputMeasurements=5966846f-156d-4a2d-a181-08c4875447a1; useLatestValues=false - // variableList={VPOSA=62eb881c-4423-47bc-87bd-e3acfd5c7430; VAA=01564cc5-2451-4501-ba51-4fcea7173333}; expressionText=VPOSA-VAA; framesPerSecond=30; lagTime=6; leadTime=3; outputMeasurements=d59389a2-3816-4e84-a593-75f07369e582; useLatestValues=false - // variableList={IANG_DIFF_CA[]=9108df49-4f96-4e8c-a23f-1eb07b281fa1,a66b1545-16f2-4f4a-8469-f3942cf27a56}; expressionText=IF(Any(IANG_DIFF_CA, "< 110") OR Any(IANG_DIFF_CA, "> 130"), 0, 1); framesPerSecond=30; lagTime=6; leadTime=3; outputMeasurements=174b312a-cc47-4390-b073-ee68033956e9; useLatestValues=false - - if (!connectionString.ContainsKey("variableList")) - continue; + // variableList={FREQ=bef9f097-d6f8-4b4f-9ce4-97ffa30a08e9}; expressionText=FREQ-60; framesPerSecond=30; lagTime=6; leadTime=3; outputMeasurements=5966846f-156d-4a2d-a181-08c4875447a1; useLatestValues=false + // variableList={VPOSA=62eb881c-4423-47bc-87bd-e3acfd5c7430; VAA=01564cc5-2451-4501-ba51-4fcea7173333}; expressionText=VPOSA-VAA; framesPerSecond=30; lagTime=6; leadTime=3; outputMeasurements=d59389a2-3816-4e84-a593-75f07369e582; useLatestValues=false + // variableList={IANG_DIFF_CA[]=9108df49-4f96-4e8c-a23f-1eb07b281fa1,a66b1545-16f2-4f4a-8469-f3942cf27a56}; expressionText=IF(Any(IANG_DIFF_CA, "< 110") OR Any(IANG_DIFF_CA, "> 130"), 0, 1); framesPerSecond=30; lagTime=6; leadTime=3; outputMeasurements=174b312a-cc47-4390-b073-ee68033956e9; useLatestValues=false - string variableList = connectionString["variableList"]; + if (!connectionString.TryGetValue("variableList", out string variableList)) + continue; - if (string.IsNullOrWhiteSpace(variableList)) - continue; + if (string.IsNullOrWhiteSpace(variableList)) + continue; - Dictionary variables = variableList.ParseKeyValuePairs(); - HashSet signalIDs = new(variables.Values.SelectMany(ids => ids.Split(',')).Select(id => Guid.TryParse(id, out Guid signalID) ? signalID : Guid.Empty).Where(signalID => signalID != Guid.Empty)); + Dictionary variables = variableList.ParseKeyValuePairs(); + HashSet signalIDs = new(variables.Values.SelectMany(ids => ids.Split(',')).Select(id => Guid.TryParse(id, out Guid signalID) ? signalID : Guid.Empty).Where(signalID => signalID != Guid.Empty)); - if (signalIDs.Count == 0) - continue; + if (signalIDs.Count == 0) + continue; - int inputCount = connection.ExecuteScalar($"SELECT COUNT(*) FROM Measurement WHERE SignalID IN ({string.Join(",", signalIDs.Select(id => $"'{id}'"))})"); + int inputCount = connection.ExecuteScalar($"SELECT COUNT(*) FROM Measurement WHERE SignalID IN ({string.Join(",", signalIDs.Select(id => $"'{id}'"))})"); - if (inputCount == signalIDs.Count) - continue; + if (inputCount == signalIDs.Count) + continue; - // Remove any calculations that reference non-existent input measurements - int affectedRows = actionAdapterTable.DeleteRecord(dynamicCalculator); + // Remove any calculations that reference non-existent input measurements + int affectedRows = actionAdapterTable.DeleteRecord(dynamicCalculator); - if (affectedRows > 0) - statusMessage($"Removed dynamic calculation adapter \"{dynamicCalculator.AdapterName}\" that referenced non-existent input measurements."); - - // Remove result calculation output measurement - if (!connectionString.ContainsKey("outputMeasurements")) - continue; + if (affectedRows > 0) + statusMessage($"Removed dynamic calculation adapter \"{dynamicCalculator.AdapterName}\" that referenced non-existent input measurements."); - string outputMeasurement = connectionString["outputMeasurements"]; + // Remove result calculation output measurement + if (!connectionString.TryGetValue("outputMeasurements", out string outputMeasurement)) + continue; - if (string.IsNullOrWhiteSpace(outputMeasurement)) - continue; + if (string.IsNullOrWhiteSpace(outputMeasurement)) + continue; - if (Guid.TryParse(outputMeasurement, out Guid outputMeasurementID)) - measurementTable.DeleteRecordWhere($"SignalID = '{outputMeasurementID}'"); - } + if (Guid.TryParse(outputMeasurement, out Guid outputMeasurementID)) + measurementTable.DeleteRecordWhere($"SignalID = '{outputMeasurementID}'"); } - - #endregion - - #region [ Future Code ] - - // This code can be used to refresh the local metadata repository when/if this gets implemented - - // >> members: - - //private MetadataProviders m_metadataProviders; - //private readonly object m_queuedMetadataRefreshPending; - //private AutoResetEvent m_metadataRefreshComplete; - //private bool m_autoRefreshMetadata; - - // >> ctor: - - //m_autoRefreshMetadata = true; - //m_queuedMetadataRefreshPending = new object(); - //m_metadataRefreshComplete = new AutoResetEvent(true); - - // >> Status: - - //status.Append(m_metadataProviders.Status); - //status.AppendLine(); - - // >> Initialize(): - - //// Provide metadata sync support. - //m_metadataProviders = new MetadataProviders(); - //m_metadataProviders.AdapterCreated += MetadataProviders_AdapterCreated; - //m_metadataProviders.AdapterLoaded += MetadataProviders_AdapterLoaded; - //m_metadataProviders.AdapterUnloaded += MetadataProviders_AdapterUnloaded; - //m_metadataProviders.AdapterLoadException += AdapterLoader_AdapterLoadException; - - // >> Dispose(): - - //if (m_metadataProviders != null) - //{ - // m_metadataProviders.AdapterCreated -= MetadataProviders_AdapterCreated; - // m_metadataProviders.AdapterLoaded -= MetadataProviders_AdapterLoaded; - // m_metadataProviders.AdapterUnloaded -= MetadataProviders_AdapterUnloaded; - // m_metadataProviders.AdapterLoadException -= AdapterLoader_AdapterLoadException; - // m_metadataProviders.Dispose(); - //} - - // >> AttemptConnection(): - - //m_metadataProviders.Initialize(); - - //int waitCount = 0; - - //// Wait for adapter initialization to complete, up to 2 seconds - //while (waitCount < 20 && m_adapterLoadedCount != m_dataServices.Adapters.Count + m_metadataProviders.Adapters.Count + m_replicationProviders.Adapters.Count) - //{ - // Thread.Sleep(100); - // waitCount++; - //} - - //// Kick off a meta-data refresh... - //if (m_autoRefreshMetadata) - //{ - // RefreshMetadata(); - // m_autoRefreshMetadata = false; - //} - - ///// - ///// Gets or sets a boolean indicating whether or not metadata is - ///// refreshed when the adapter attempts to connect to the archive. - ///// - //[ConnectionStringParameter, - //Description("Define a boolean indicating whether to refresh metadata on connect."), - //DefaultValue(true)] - //public bool AutoRefreshMetadata - //{ - // get - // { - // return m_autoRefreshMetadata; - // } - // set - // { - // m_autoRefreshMetadata = value; - // } - //} - - //private void MetadataProviders_AdapterCreated(object sender, EventArgs e) - //{ - // e.Argument.SettingsCategory = InstanceName + e.Argument.SettingsCategory; - - // if (e.Argument.GetType() == typeof(AdoMetadataProvider)) - // { - // // Populate the default configuration for AdoMetadataProvider. - // AdoMetadataProvider provider = e.Argument as AdoMetadataProvider; - - // provider.Enabled = true; - // provider.SelectString = string.Format("SELECT * FROM HistorianMetadata WHERE PlantCode='{0}'", Name); - - // // The following connection information is now provided via configuration Eval mappings - // // provider.DataProviderString = config.Settings["SystemSettings"]["DataProviderString"].Value; - - // // ConfigurationFile config = ConfigurationFile.Current; - // // string connectionString = config.Settings["SystemSettings"]["ConnectionString"].Value; - // // Dictionary settings = connectionString.ParseKeyValuePairs(); - // // string setting; - - // // if (settings.TryGetValue("Provider", out setting)) - // // { - // // // Check if provider is for Access - // // if (setting.StartsWith("Microsoft.Jet.OLEDB", StringComparison.OrdinalIgnoreCase)) - // // { - // // // Make sure path to Access database is fully qualified - // // if (settings.TryGetValue("Data Source", out setting)) - // // { - // // settings["Data Source"] = FilePath.GetAbsolutePath(setting); - // // connectionString = settings.JoinKeyValuePairs(); - // // } - // // } - // // } - - // // provider.ConnectionString = connectionString; - // } - //} - - //private void MetadataProviders_AdapterLoaded(object sender, EventArgs e) - //{ - // e.Argument.Metadata = m_archive.MetadataFile; - - // e.Argument.MetadataRefreshStart += MetadataProviders_MetadataRefreshStart; - // e.Argument.MetadataRefreshComplete += MetadataProviders_MetadataRefreshComplete; - // e.Argument.MetadataRefreshTimeout += MetadataProviders_MetadataRefreshTimeout; - // e.Argument.MetadataRefreshException += MetadataProviders_MetadataRefreshException; - // OnStatusMessage("{0} has been loaded.", e.Argument.GetType().Name); - - // m_adapterLoadedCount++; - //} - - //private void MetadataProviders_AdapterUnloaded(object sender, EventArgs e) - //{ - // e.Argument.Metadata = null; - // e.Argument.MetadataRefreshStart -= MetadataProviders_MetadataRefreshStart; - // e.Argument.MetadataRefreshComplete -= MetadataProviders_MetadataRefreshComplete; - // e.Argument.MetadataRefreshTimeout -= MetadataProviders_MetadataRefreshTimeout; - // e.Argument.MetadataRefreshException -= MetadataProviders_MetadataRefreshException; - // OnStatusMessage("{0} has been unloaded.", e.Argument.GetType().Name); - //} - - //private void MetadataProviders_MetadataRefreshStart(object sender, EventArgs e) - //{ - // OnStatusMessage("{0} has started metadata refresh...", sender.GetType().Name); - //} - - //private void MetadataProviders_MetadataRefreshComplete(object sender, EventArgs e) - //{ - // OnStatusMessage("{0} has finished metadata refresh.", sender.GetType().Name); - //} - - //private void MetadataProviders_MetadataRefreshTimeout(object sender, EventArgs e) - //{ - // OnStatusMessage("{0} has timed-out on metadata refresh.", sender.GetType().Name); - //} - - //private void MetadataProviders_MetadataRefreshException(object sender, EventArgs e) - //{ - // OnProcessException(e.Argument); - //} - #endregion } -} + + #endregion + + #region [ Future Code ] + + // This code can be used to refresh the local metadata repository when/if this gets implemented + + // >> members: + + //private MetadataProviders m_metadataProviders; + //private readonly object m_queuedMetadataRefreshPending; + //private AutoResetEvent m_metadataRefreshComplete; + //private bool m_autoRefreshMetadata; + + // >> ctor: + + //m_autoRefreshMetadata = true; + //m_queuedMetadataRefreshPending = new object(); + //m_metadataRefreshComplete = new AutoResetEvent(true); + + // >> Status: + + //status.Append(m_metadataProviders.Status); + //status.AppendLine(); + + // >> Initialize(): + + //// Provide metadata sync support. + //m_metadataProviders = new MetadataProviders(); + //m_metadataProviders.AdapterCreated += MetadataProviders_AdapterCreated; + //m_metadataProviders.AdapterLoaded += MetadataProviders_AdapterLoaded; + //m_metadataProviders.AdapterUnloaded += MetadataProviders_AdapterUnloaded; + //m_metadataProviders.AdapterLoadException += AdapterLoader_AdapterLoadException; + + // >> Dispose(): + + //if (m_metadataProviders != null) + //{ + // m_metadataProviders.AdapterCreated -= MetadataProviders_AdapterCreated; + // m_metadataProviders.AdapterLoaded -= MetadataProviders_AdapterLoaded; + // m_metadataProviders.AdapterUnloaded -= MetadataProviders_AdapterUnloaded; + // m_metadataProviders.AdapterLoadException -= AdapterLoader_AdapterLoadException; + // m_metadataProviders.Dispose(); + //} + + // >> AttemptConnection(): + + //m_metadataProviders.Initialize(); + + //int waitCount = 0; + + //// Wait for adapter initialization to complete, up to 2 seconds + //while (waitCount < 20 && m_adapterLoadedCount != m_dataServices.Adapters.Count + m_metadataProviders.Adapters.Count + m_replicationProviders.Adapters.Count) + //{ + // Thread.Sleep(100); + // waitCount++; + //} + + //// Kick off a meta-data refresh... + //if (m_autoRefreshMetadata) + //{ + // RefreshMetadata(); + // m_autoRefreshMetadata = false; + //} + + ///// + ///// Gets or sets a boolean indicating whether or not metadata is + ///// refreshed when the adapter attempts to connect to the archive. + ///// + //[ConnectionStringParameter, + //Description("Define a boolean indicating whether to refresh metadata on connect."), + //DefaultValue(true)] + //public bool AutoRefreshMetadata + //{ + // get + // { + // return m_autoRefreshMetadata; + // } + // set + // { + // m_autoRefreshMetadata = value; + // } + //} + + //private void MetadataProviders_AdapterCreated(object sender, EventArgs e) + //{ + // e.Argument.SettingsCategory = InstanceName + e.Argument.SettingsCategory; + + // if (e.Argument.GetType() == typeof(AdoMetadataProvider)) + // { + // // Populate the default configuration for AdoMetadataProvider. + // AdoMetadataProvider provider = e.Argument as AdoMetadataProvider; + + // provider.Enabled = true; + // provider.SelectString = string.Format("SELECT * FROM HistorianMetadata WHERE PlantCode='{0}'", Name); + + // // The following connection information is now provided via configuration Eval mappings + // // provider.DataProviderString = config.Settings["SystemSettings"]["DataProviderString"].Value; + + // // ConfigurationFile config = ConfigurationFile.Current; + // // string connectionString = config.Settings["SystemSettings"]["ConnectionString"].Value; + // // Dictionary settings = connectionString.ParseKeyValuePairs(); + // // string setting; + + // // if (settings.TryGetValue("Provider", out setting)) + // // { + // // // Check if provider is for Access + // // if (setting.StartsWith("Microsoft.Jet.OLEDB", StringComparison.OrdinalIgnoreCase)) + // // { + // // // Make sure path to Access database is fully qualified + // // if (settings.TryGetValue("Data Source", out setting)) + // // { + // // settings["Data Source"] = FilePath.GetAbsolutePath(setting); + // // connectionString = settings.JoinKeyValuePairs(); + // // } + // // } + // // } + + // // provider.ConnectionString = connectionString; + // } + //} + + //private void MetadataProviders_AdapterLoaded(object sender, EventArgs e) + //{ + // e.Argument.Metadata = m_archive.MetadataFile; + + // e.Argument.MetadataRefreshStart += MetadataProviders_MetadataRefreshStart; + // e.Argument.MetadataRefreshComplete += MetadataProviders_MetadataRefreshComplete; + // e.Argument.MetadataRefreshTimeout += MetadataProviders_MetadataRefreshTimeout; + // e.Argument.MetadataRefreshException += MetadataProviders_MetadataRefreshException; + // OnStatusMessage("{0} has been loaded.", e.Argument.GetType().Name); + + // m_adapterLoadedCount++; + //} + + //private void MetadataProviders_AdapterUnloaded(object sender, EventArgs e) + //{ + // e.Argument.Metadata = null; + // e.Argument.MetadataRefreshStart -= MetadataProviders_MetadataRefreshStart; + // e.Argument.MetadataRefreshComplete -= MetadataProviders_MetadataRefreshComplete; + // e.Argument.MetadataRefreshTimeout -= MetadataProviders_MetadataRefreshTimeout; + // e.Argument.MetadataRefreshException -= MetadataProviders_MetadataRefreshException; + // OnStatusMessage("{0} has been unloaded.", e.Argument.GetType().Name); + //} + + //private void MetadataProviders_MetadataRefreshStart(object sender, EventArgs e) + //{ + // OnStatusMessage("{0} has started metadata refresh...", sender.GetType().Name); + //} + + //private void MetadataProviders_MetadataRefreshComplete(object sender, EventArgs e) + //{ + // OnStatusMessage("{0} has finished metadata refresh.", sender.GetType().Name); + //} + + //private void MetadataProviders_MetadataRefreshTimeout(object sender, EventArgs e) + //{ + // OnStatusMessage("{0} has timed-out on metadata refresh.", sender.GetType().Name); + //} + + //private void MetadataProviders_MetadataRefreshException(object sender, EventArgs e) + //{ + // OnProcessException(e.Argument); + //} + #endregion +} \ No newline at end of file diff --git a/Source/Libraries/GSF.SortedTreeStore/GSF.SortedTreeStore.SqlClr.csproj b/Source/Libraries/GSF.SortedTreeStore/GSF.SortedTreeStore.SqlClr.csproj index 9e6d84ea5b..7d2ab2090d 100755 --- a/Source/Libraries/GSF.SortedTreeStore/GSF.SortedTreeStore.SqlClr.csproj +++ b/Source/Libraries/GSF.SortedTreeStore/GSF.SortedTreeStore.SqlClr.csproj @@ -28,7 +28,7 @@ 1.0.0.%2a false true - 8 + latest AnyCPU @@ -142,11 +142,18 @@ + + + + + + + diff --git a/Source/Libraries/GSF.SortedTreeStore/GSF.SortedTreeStore.csproj b/Source/Libraries/GSF.SortedTreeStore/GSF.SortedTreeStore.csproj index db5bb6eee4..0d9f08b072 100755 --- a/Source/Libraries/GSF.SortedTreeStore/GSF.SortedTreeStore.csproj +++ b/Source/Libraries/GSF.SortedTreeStore/GSF.SortedTreeStore.csproj @@ -28,7 +28,7 @@ 1.0.0.%2a false true - 8 + latest AnyCPU @@ -272,6 +272,10 @@ + + + + @@ -279,6 +283,9 @@ + + + diff --git a/Source/Libraries/GSF.SortedTreeStore/IO/FileStructure/SimplifiedFileWriter.cs b/Source/Libraries/GSF.SortedTreeStore/IO/FileStructure/SimplifiedFileWriter.cs index ec6e2fe368..8bd582b5f7 100755 --- a/Source/Libraries/GSF.SortedTreeStore/IO/FileStructure/SimplifiedFileWriter.cs +++ b/Source/Libraries/GSF.SortedTreeStore/IO/FileStructure/SimplifiedFileWriter.cs @@ -33,22 +33,18 @@ namespace GSF.IO.FileStructure /// Assists in the writing of a simplified file. This file can only be appended to /// and it must be sequentially written. /// - public class SimplifiedFileWriter - : IDisposable + public class SimplifiedFileWriter : IDisposable { private static readonly LogPublisher Log = Logger.CreatePublisher(typeof(SimplifiedFileWriter), MessageClass.Component); private bool m_disposed; - private readonly FileHeaderBlock m_fileHeaderBlock; - private SimplifiedSubFileStream m_subFileStream; - - public FileStream m_stream; - private readonly string m_pendingFileName; private readonly string m_completeFileName; + public FileStream m_stream; + /// /// Creates a simplified file writer. /// @@ -65,21 +61,25 @@ public SimplifiedFileWriter(string pendingFileName, string completeFileName, int m_stream = new FileStream(pendingFileName, FileMode.CreateNew, FileAccess.ReadWrite, FileShare.None); } -#if DEBUG + #if DEBUG ~SimplifiedFileWriter() { Log.Publish(MessageLevel.Info, "Finalizer Called", GetType().FullName); } -#endif + #endif public Guid ArchiveId => m_fileHeaderBlock.ArchiveId; + internal uint SnapshotSequenceNumber => m_fileHeaderBlock.SnapshotSequenceNumber; + private void CloseCurrentFile() { if (m_subFileStream is null) return; + if (!m_subFileStream.IsDisposed) throw new Exception("The previous file must be disposed before completing this action"); + m_subFileStream = null; } @@ -91,10 +91,13 @@ public ISupportsBinaryStream CreateFile(SubFileName fileName) { if (m_disposed) throw new ObjectDisposedException(GetType().FullName); + CloseCurrentFile(); SubFileHeader subFile = m_fileHeaderBlock.CreateNewFile(fileName); + subFile.DirectBlock = m_fileHeaderBlock.LastAllocatedBlock + 1; + m_subFileStream = new SimplifiedSubFileStream(m_stream, subFile, m_fileHeaderBlock); return m_subFileStream; } @@ -106,15 +109,22 @@ public void Commit() { if (m_disposed) return; + CloseCurrentFile(); + m_stream.Position = 0; m_stream.Write(m_fileHeaderBlock.GetBytes()); m_stream.Flush(true); + WinApi.FlushFileBuffers(m_stream.SafeFileHandle); + m_stream.Dispose(); m_stream = null; + File.Move(m_pendingFileName, m_completeFileName); + m_disposed = true; + Dispose(); } @@ -133,31 +143,28 @@ public void Dispose() /// true to release both managed and unmanaged resources; false to release only unmanaged resources. protected virtual void Dispose(bool disposing) { - if (!m_disposed) + if (m_disposed) + return; + + try { - try - { - // This will be done regardless of whether the object is finalized or disposed. - - if (disposing) - { - if (m_subFileStream != null) - { - m_subFileStream.Dispose(); - m_subFileStream = null; - } - if (m_stream != null) - { - m_stream.Dispose(); - } - File.Delete(m_pendingFileName); - // This will be done only when the object is disposed by calling Dispose(). - } - } - finally + // This will be done regardless of whether the object is finalized or disposed. + if (!disposing) + return; + + if (m_subFileStream is not null) { - m_disposed = true; // Prevent duplicate dispose. + m_subFileStream.Dispose(); + m_subFileStream = null; } + + m_stream?.Dispose(); + + File.Delete(m_pendingFileName); + } + finally + { + m_disposed = true; // Prevent duplicate dispose. } } } diff --git a/Source/Libraries/GSF.SortedTreeStore/Snap/Filters/NearestTimestampSeekFilter.cs b/Source/Libraries/GSF.SortedTreeStore/Snap/Filters/NearestTimestampSeekFilter.cs new file mode 100644 index 0000000000..58eab9028a --- /dev/null +++ b/Source/Libraries/GSF.SortedTreeStore/Snap/Filters/NearestTimestampSeekFilter.cs @@ -0,0 +1,259 @@ +//****************************************************************************************************** +// NearestTimestampSeekFilter.cs - Gbtc +// +// Copyright © 2026, Grid Protection Alliance. All Rights Reserved. +// +// Licensed to the Grid Protection Alliance (GPA) under one or more contributor license agreements. See +// the NOTICE file distributed with this work for additional information regarding copyright ownership. +// The GPA licenses this file to you under the MIT License (MIT), the "License"; you may not use this +// file except in compliance with the License. You may obtain a copy of the License at: +// +// http://opensource.org/licenses/MIT +// +// Unless agreed to in writing, the subject software distributed under the License is distributed on an +// "AS-IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. Refer to the +// License for the specific language governing permissions and limitations. +// +// Code Modification History: +// ---------------------------------------------------------------------------------------------------- +// 09/21/2026 - J. Ritchie Carroll +// Generated original version of source code. +// +//****************************************************************************************************** + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using GSF.IO; +using GSF.Snap.Types; + +namespace GSF.Snap.Filters; + +/// +/// Selects the nearest eligible sample per point ID around regularly spaced target timestamps. +/// Requires timestamp-first key ordering, then point ID (as used by HistorianKey). +/// +/// +/// Targets start at firstTime and include lastTime when it falls on the grid. Samples must lie +/// within the query range and maximumDistance of the target. Equal distances prefer the earlier +/// timestamp, then the smaller key. Original keys and values are returned without interpolation. +/// Windows cannot overlap except at their midpoint, which belongs to the earlier target. +/// Consequently, results are sequential and a source sample is never returned twice. AtOrAfter +/// mode instead selects the first eligible sample at or after each target, using the same +/// distance limit. Missing IDs produce no result. Both endpoints must support the selected mode. +/// +public sealed class NearestTimestampSeekFilter : SeekFilterBase where TKey : SnapTypeBase, new() +{ + private readonly ulong m_first; + private readonly ulong m_firstTarget; + private readonly bool m_singleTarget; + private readonly ulong m_last; + private readonly ulong m_interval; + private readonly ulong m_distance; + private readonly ulong[] m_pointIDs; + private bool m_started; + private bool m_finished; + + /// + /// Creates a nearest-sample query. All time quantities are in timestamp ticks. + /// + /// Inclusive query start and first target. + /// Inclusive query end. + /// Positive target spacing. + /// Explicit set of IDs to sample. + /// Search radius, at most half the interval; defaults to half. + /// Search both sides, or only at/after each target using the same distance limit. + public NearestTimestampSeekFilter(ulong firstTime, ulong lastTime, ulong interval, + IEnumerable pointIDs, ulong? maximumDistance = null, TimestampSearchMode searchMode = TimestampSearchMode.Nearest) + { + if (new TKey() is not TimestampPointIDBase) + throw new NotSupportedException("Nearest sampling requires timestamp/point-ID keys."); + + if (firstTime > lastTime) + throw new ArgumentOutOfRangeException(nameof(lastTime)); + + if (interval == 0) + throw new ArgumentOutOfRangeException(nameof(interval)); + + if (pointIDs == null) + throw new ArgumentNullException(nameof(pointIDs)); + + ValidateSearchMode(searchMode); + + SearchMode = searchMode; + m_first = firstTime; + m_firstTarget = firstTime; + m_last = lastTime; + m_interval = interval; + m_distance = maximumDistance ?? interval / 2; + + if (m_distance > interval / 2) + throw new ArgumentOutOfRangeException(nameof(maximumDistance), "Search windows must not overlap."); + + m_pointIDs = pointIDs.Distinct().OrderBy(id => id).ToArray(); + StartOfRange = new TKey(); + EndOfRange = new TKey(); + StartOfFrame = new TKey(); + EndOfFrame = new TKey(); + + SetTimestamp(StartOfRange, firstTime, false); + SetTimestamp(EndOfRange, lastTime, true); + + Reset(); + } + + /// + /// Finds one nearest sample for an ID, searching on both sides of a timestamp. + /// + /// Target timestamp. + /// ID to find. + /// Maximum distance in timestamp ticks, inclusive. + /// Search both sides, or only at/after the target. + public NearestTimestampSeekFilter(ulong timestamp, ulong pointID, ulong maximumDistance, TimestampSearchMode searchMode = TimestampSearchMode.Nearest) + : this(searchMode == TimestampSearchMode.AtOrAfter ? timestamp : timestamp - Math.Min(timestamp, maximumDistance), + timestamp + Math.Min(ulong.MaxValue - timestamp, maximumDistance), + ulong.MaxValue, [ pointID ], 0, searchMode) + { + m_distance = maximumDistance; + m_firstTarget = timestamp; + m_singleTarget = true; + Reset(); + } + + /// + public override Guid FilterType => NearestTimestampSeekFilterDefinition.FilterGuid; + + /// The current target timestamp after NextWindow succeeds. + public ulong TargetTimestamp { get; private set; } + + /// Gets whether samples are selected on both sides or only at/after the target. + public TimestampSearchMode SearchMode { get; } + + internal IEnumerable PointIDs => m_pointIDs; + + internal static TimestampPointIDBase TimeKey(TKey key) => (TimestampPointIDBase)(object)key; + + internal static void SetTimestamp(TKey key, ulong timestamp, bool maximum) + { + if (maximum) + key.SetMax(); + else + key.SetMin(); + + TimeKey(key).Timestamp = timestamp; + } + + /// + public override void Reset() + { + m_started = false; + m_finished = false; + TargetTimestamp = m_firstTarget; + StartOfRange.CopyTo(StartOfFrame); + EndOfRange.CopyTo(EndOfFrame); + } + + /// + public override bool NextWindow() + { + if (m_finished || m_pointIDs.Length == 0) + return false; + + if (m_started) + { + if (m_singleTarget || m_last - TargetTimestamp < m_interval) + { + m_finished = true; + return false; + } + + TargetTimestamp += m_interval; + } + + ulong lower = SearchMode == TimestampSearchMode.AtOrAfter ? TargetTimestamp : TargetTimestamp - Math.Min(m_distance, TargetTimestamp - m_first); + ulong upper = TargetTimestamp + Math.Min(m_distance, m_last - TargetTimestamp); + + // When the interval is even, the shared midpoint belongs to the previous target. + if (SearchMode == TimestampSearchMode.Nearest && m_started && m_distance == m_interval - m_distance) + lower++; + + SetTimestamp(StartOfFrame, lower, false); + SetTimestamp(EndOfFrame, upper, true); + + m_started = true; + + return true; + } + + /// + public override void Save(BinaryStreamBase stream) + { + // Keep the original payload for nearest queries; older readers reject the new version + // for directional queries rather than silently changing their meaning. + stream.Write((byte)(SearchMode == TimestampSearchMode.Nearest ? 1 : 2)); + stream.Write((byte)(m_singleTarget ? 1 : 0)); + + if (SearchMode != TimestampSearchMode.Nearest) + stream.Write((byte)SearchMode); + + if (m_singleTarget) + { + stream.Write(m_firstTarget); + stream.Write(m_pointIDs[0]); + stream.Write(m_distance); + return; + } + + stream.Write(m_first); + stream.Write(m_last); + stream.Write(m_interval); + stream.Write(m_distance); + stream.Write(m_pointIDs.Length); + + foreach (ulong id in m_pointIDs) + stream.Write(id); + } + + private static void ValidateSearchMode(TimestampSearchMode searchMode) + { + if (searchMode != TimestampSearchMode.Nearest && searchMode != TimestampSearchMode.AtOrAfter) + throw new ArgumentOutOfRangeException(nameof(searchMode)); + } + internal static NearestTimestampSeekFilter Load(BinaryStreamBase stream) + { + byte version = stream.ReadUInt8(); + + if (version != 1 && version != 2) + throw new InvalidDataException("Unknown nearest-timestamp filter version."); + + byte mode = stream.ReadUInt8(); + TimestampSearchMode searchMode = version == 1 ? TimestampSearchMode.Nearest : (TimestampSearchMode)stream.ReadUInt8(); + + if (searchMode != TimestampSearchMode.Nearest && searchMode != TimestampSearchMode.AtOrAfter) + throw new InvalidDataException("Unknown timestamp search mode."); + + if (mode == 1) + return new NearestTimestampSeekFilter(stream.ReadUInt64(), stream.ReadUInt64(), stream.ReadUInt64(), searchMode); + + if (mode != 0) + throw new InvalidDataException("Unknown nearest-timestamp filter mode."); + + ulong first = stream.ReadUInt64(); + ulong last = stream.ReadUInt64(); + ulong interval = stream.ReadUInt64(); + ulong distance = stream.ReadUInt64(); + int count = stream.ReadInt32(); + + if (count < 0) + throw new InvalidDataException("Negative point-ID count."); + + // Read incrementally: do not allocate an attacker-controlled array before reading payload. + List ids = []; + + for (int i = 0; i < count; i++) + ids.Add(stream.ReadUInt64()); + + return new NearestTimestampSeekFilter(first, last, interval, ids, distance, searchMode); + } +} \ No newline at end of file diff --git a/Source/Libraries/GSF.SortedTreeStore/Snap/Filters/NearestTimestampSeekFilterDefinition.cs b/Source/Libraries/GSF.SortedTreeStore/Snap/Filters/NearestTimestampSeekFilterDefinition.cs new file mode 100644 index 0000000000..d0f5d43211 --- /dev/null +++ b/Source/Libraries/GSF.SortedTreeStore/Snap/Filters/NearestTimestampSeekFilterDefinition.cs @@ -0,0 +1,55 @@ +//****************************************************************************************************** +// NearestTimestampSeekFilterDefinition.cs - Gbtc +// +// Copyright © 2026, Grid Protection Alliance. All Rights Reserved. +// +// Licensed to the Grid Protection Alliance (GPA) under one or more contributor license agreements. See +// the NOTICE file distributed with this work for additional information regarding copyright ownership. +// The GPA licenses this file to you under the MIT License (MIT), the "License"; you may not use this +// file except in compliance with the License. You may obtain a copy of the License at: +// +// http://opensource.org/licenses/MIT +// +// Unless agreed to in writing, the subject software distributed under the License is distributed on an +// "AS-IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. Refer to the +// License for the specific language governing permissions and limitations. +// +// Code Modification History: +// ---------------------------------------------------------------------------------------------------- +// 09/21/2026 - J. Ritchie Carroll +// Generated original version of source code. +// +//****************************************************************************************************** + +using System; +using System.Reflection; +using GSF.IO; +using GSF.Snap.Definitions; + +namespace GSF.Snap.Filters; + +/// +/// Wire definition for nearest timestamp sampling. +/// +public sealed class NearestTimestampSeekFilterDefinition : SeekFilterDefinitionBase +{ + /// + /// Stable identifier for the nearest-sample wire format. + /// + public static readonly Guid FilterGuid = new("f906dfb8-692b-4a28-98ed-ff78ae74e4b5"); + + /// + public override Guid FilterType => FilterGuid; + + /// + public override SeekFilterBase Create(BinaryStreamBase stream) + { + Type filterType = typeof(NearestTimestampSeekFilter<>).MakeGenericType(typeof(TKey)); + + MethodInfo method = filterType.GetMethod("Load", BindingFlags.Static | BindingFlags.NonPublic) ?? + throw new InvalidOperationException("Nearest filter loader is missing."); + + return (SeekFilterBase)(method.Invoke(null, [stream]) ?? + throw new InvalidOperationException("Nearest filter loader returned no filter.")); + } +} \ No newline at end of file diff --git a/Source/Libraries/GSF.SortedTreeStore/Snap/Filters/TimestampPointIDSeekFilter.cs b/Source/Libraries/GSF.SortedTreeStore/Snap/Filters/TimestampPointIDSeekFilter.cs index 823c6c3622..a14f0a4973 100755 --- a/Source/Libraries/GSF.SortedTreeStore/Snap/Filters/TimestampPointIDSeekFilter.cs +++ b/Source/Libraries/GSF.SortedTreeStore/Snap/Filters/TimestampPointIDSeekFilter.cs @@ -26,112 +26,126 @@ using GSF.IO; using GSF.Snap.Types; -namespace GSF.Snap.Filters +namespace GSF.Snap.Filters; + +/// +/// Represents a seek filter for a specific timestamp and point ID. +/// +public static class TimestampPointIDSeekFilter { /// - /// Represents a seek filter for a specific timestamp and point ID. + /// Creates a filter to find the nearest key for the specified timestamp and point ID within the specified maximum distance. + /// + /// The specific timestamp to find. + /// The specific point ID to find. + /// The maximum distance to search for the nearest key. + /// The mode to use when searching for the nearest key. + /// Seek filter to find specific key. + public static SeekFilterBase FindNearestKey(ulong timestamp, ulong pointID, ulong maximumDistance, + TimestampSearchMode searchMode = TimestampSearchMode.Nearest) + where TKey : TimestampPointIDBase, new() + { + return new NearestTimestampSeekFilter(timestamp, pointID, maximumDistance, searchMode); + } + + /// + /// Creates a filter for the specified timestamp and point ID. /// - public static class TimestampPointIDSeekFilter + /// The specific timestamp to find. + /// The specific point ID to find. + /// Seek filter to find specific key. + public static SeekFilterBase FindKey(ulong timestamp, ulong pointID) + where TKey : TimestampPointIDBase, new() { + return new SeekToKey(timestamp, pointID); + } + + /// + /// Loads a from the provided . + /// + /// The stream to load the filter from + /// Seek filter to find specific key. + [MethodImpl(MethodImplOptions.NoOptimization)] + private static SeekFilterBase CreateFromStream(BinaryStreamBase stream) + where TKey : TimestampPointIDBase, new() + { + return new SeekToKey(stream); + } + + private class SeekToKey + : SeekFilterBase + where TKey : TimestampPointIDBase, new() + { + private readonly TKey m_keyToFind; + private bool m_isEndReached; + + private SeekToKey() + { + m_keyToFind = new TKey(); + StartOfFrame = new TKey(); + EndOfFrame = new TKey(); + StartOfRange = StartOfFrame; + EndOfRange = EndOfFrame; + } + /// - /// Creates a filter for the specified timestamp and point ID. + /// Creates a filter by reading from the stream. /// - /// The specific timestamp to find. - /// The specific point ID to find. - /// Seek filter to find specific key. - public static SeekFilterBase FindKey(ulong timestamp, ulong pointID) - where TKey : TimestampPointIDBase, new() + /// the stream to read from + public SeekToKey(BinaryStreamBase stream) + : this() { - return new SeekToKey(timestamp, pointID); + m_keyToFind.Timestamp = stream.ReadUInt64(); + m_keyToFind.PointID = stream.ReadUInt64(); + m_keyToFind.CopyTo(StartOfRange); + m_keyToFind.CopyTo(EndOfRange); } /// - /// Loads a from the provided . + /// Creates a filter for the key. /// - /// The stream to load the filter from - /// Seek filter to find specific key. - [MethodImpl(MethodImplOptions.NoOptimization)] - private static SeekFilterBase CreateFromStream(BinaryStreamBase stream) - where TKey : TimestampPointIDBase, new() + /// The specific timestamp to find. + /// The specific point ID to find. + public SeekToKey(ulong timestamp, ulong pointID) + : this() { - return new SeekToKey(stream); + m_keyToFind.Timestamp = timestamp; + m_keyToFind.PointID = pointID; + m_keyToFind.CopyTo(StartOfRange); + m_keyToFind.CopyTo(EndOfRange); } - private class SeekToKey - : SeekFilterBase - where TKey : TimestampPointIDBase, new() + /// + /// Gets the next search window. + /// + /// true if window exists, false if finished. + public override bool NextWindow() { - private readonly TKey m_keyToFind; - private bool m_isEndReached; - - private SeekToKey() - { - m_keyToFind = new TKey(); - StartOfFrame = new TKey(); - EndOfFrame = new TKey(); - StartOfRange = StartOfFrame; - EndOfRange = EndOfFrame; - } - - /// - /// Creates a filter by reading from the stream. - /// - /// the stream to read from - public SeekToKey(BinaryStreamBase stream) - : this() - { - m_keyToFind.Timestamp = stream.ReadUInt64(); - m_keyToFind.PointID = stream.ReadUInt64(); - m_keyToFind.CopyTo(StartOfRange); - m_keyToFind.CopyTo(EndOfRange); - } - - /// - /// Creates a filter for the key. - /// - /// The specific timestamp to find. - /// The specific point ID to find. - public SeekToKey(ulong timestamp, ulong pointID) - : this() - { - m_keyToFind.Timestamp = timestamp; - m_keyToFind.PointID = pointID; - m_keyToFind.CopyTo(StartOfRange); - m_keyToFind.CopyTo(EndOfRange); - } - - /// - /// Gets the next search window. - /// - /// true if window exists, false if finished. - public override bool NextWindow() - { - if (m_isEndReached) - return false; - - m_isEndReached = true; - return true; - } + if (m_isEndReached) + return false; - /// - /// Resets the iterative nature of the filter. - /// - public override void Reset() - { - m_isEndReached = false; - } + m_isEndReached = true; + return true; + } - /// - /// Serializes the filter to a stream - /// - /// the stream to write to - public override void Save(BinaryStreamBase stream) - { - stream.Write(m_keyToFind.Timestamp); - stream.Write(m_keyToFind.PointID); - } + /// + /// Resets the iterative nature of the filter. + /// + public override void Reset() + { + m_isEndReached = false; + } - public override Guid FilterType => TimestampPointIDSeekFilterDefinition.FilterGuid; + /// + /// Serializes the filter to a stream + /// + /// the stream to write to + public override void Save(BinaryStreamBase stream) + { + stream.Write(m_keyToFind.Timestamp); + stream.Write(m_keyToFind.PointID); } + + public override Guid FilterType => TimestampPointIDSeekFilterDefinition.FilterGuid; } -} +} \ No newline at end of file diff --git a/Source/Libraries/GSF.SortedTreeStore/Snap/Filters/TimestampSearchMode.cs b/Source/Libraries/GSF.SortedTreeStore/Snap/Filters/TimestampSearchMode.cs new file mode 100644 index 0000000000..ec6282ce86 --- /dev/null +++ b/Source/Libraries/GSF.SortedTreeStore/Snap/Filters/TimestampSearchMode.cs @@ -0,0 +1,34 @@ +//****************************************************************************************************** +// TimestampSearchMode.cs - Gbtc +// +// Copyright © 2026, Grid Protection Alliance. All Rights Reserved. +// +// Licensed to the Grid Protection Alliance (GPA) under one or more contributor license agreements. See +// the NOTICE file distributed with this work for additional information regarding copyright ownership. +// The GPA licenses this file to you under the MIT License (MIT), the "License"; you may not use this +// file except in compliance with the License. You may obtain a copy of the License at: +// +// http://opensource.org/licenses/MIT +// +// Unless agreed to in writing, the subject software distributed under the License is distributed on an +// "AS-IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. Refer to the +// License for the specific language governing permissions and limitations. +// +// Code Modification History: +// ---------------------------------------------------------------------------------------------------- +// 09/21/2026 - J. Ritchie Carroll +// Generated original version of source code. +// +//****************************************************************************************************** + +namespace GSF.Snap.Filters; + +/// Controls which side of a target timestamp is searched for a sample. +public enum TimestampSearchMode : byte +{ + /// Search both sides and choose the closest eligible sample, preferring the earlier timestamp on ties. + Nearest = 0, + + /// Choose the first eligible sample at or after the target; never return a preceding sample. + AtOrAfter = 1 +} \ No newline at end of file diff --git a/Source/Libraries/GSF.SortedTreeStore/Snap/Filters/TimestampSeekFilter.cs b/Source/Libraries/GSF.SortedTreeStore/Snap/Filters/TimestampSeekFilter.cs index ec4d5ee6b6..377c53ef7a 100755 --- a/Source/Libraries/GSF.SortedTreeStore/Snap/Filters/TimestampSeekFilter.cs +++ b/Source/Libraries/GSF.SortedTreeStore/Snap/Filters/TimestampSeekFilter.cs @@ -27,136 +27,162 @@ using GSF.IO; using GSF.Snap.Types; -namespace GSF.Snap.Filters +namespace GSF.Snap.Filters; + +public partial class TimestampSeekFilter { - public partial class TimestampSeekFilter + /// + /// Creates a nearest-sample filter over a set of date ranges + /// + /// the first time if the query (inclusive) + /// the last time of the query (inclusive) + /// the interval between samples + /// the point IDs to include in the filter + /// the optional maximum distance to search for the nearest key + /// the mode to use when searching for the nearest key + /// A seek filter to find the nearest key for the specified interval data. + /// + /// Thrown when the interval is less than or equal to zero or the maximumDistance is less than zero. + /// + public static SeekFilterBase CreateNearestFromIntervalData(DateTime firstTime, DateTime lastTime, + TimeSpan interval, System.Collections.Generic.IEnumerable pointIDs, TimeSpan? maximumDistance = null, + TimestampSearchMode searchMode = TimestampSearchMode.Nearest) + where TKey : TimestampPointIDBase, new() { + if (interval.Ticks <= 0) + throw new ArgumentOutOfRangeException(nameof(interval)); + + if (maximumDistance is { Ticks: < 0 }) + throw new ArgumentOutOfRangeException(nameof(maximumDistance)); - /// - /// Creates a filter over a single date range. (Inclusive list) - /// - /// the first time if the query (inclusive) - /// the last time of the query (inclusive) - /// - public static SeekFilterBase CreateFromRange(DateTime firstTime, DateTime lastTime) - where TKey : TimestampPointIDBase, new() - { - return new FixedRange((ulong)firstTime.Ticks, (ulong)lastTime.Ticks); - } - - /// - /// Creates a filter over a single date range. (Inclusive list) - /// - /// the first time if the query (inclusive) - /// the last time of the query (inclusive) - /// - public static SeekFilterBase CreateFromRange(ulong firstTime, ulong lastTime) - where TKey : TimestampPointIDBase, new() - { - return new FixedRange(firstTime, lastTime); - } + return new NearestTimestampSeekFilter( + (ulong)firstTime.Ticks, + (ulong)lastTime.Ticks, + (ulong)interval.Ticks, + pointIDs, + maximumDistance is null ? null : (ulong?)maximumDistance.Value.Ticks, + searchMode); + } - /// - /// Creates a filter over a set of date ranges (Similar to down sampled queries) - /// - /// the first time if the query (inclusive) - /// the last time of the query (inclusive if contained in the intervals) - /// the smallest interval that is exact - /// the interval that will be parsed. Possible to be rounded - /// the width of every window - /// A that will be able to do this parsing - /// - /// Example uses. FirstTime = 1/1/2013. LastTime = 1/2/2013. - /// MainInterval = 0.1 seconds. SubInterval = 0.0333333 seconds. - /// Tolerance = 0.001 seconds. - /// - public static SeekFilterBase CreateFromIntervalData(ulong firstTime, ulong lastTime, ulong mainInterval, ulong subInterval, ulong tolerance) - where TKey : TimestampPointIDBase, new() - { - return new IntervalRanges(firstTime, lastTime, mainInterval, subInterval, tolerance); - } + /// + /// Creates a filter over a single date range. (Inclusive list) + /// + /// the first time if the query (inclusive) + /// the last time of the query (inclusive) + /// + public static SeekFilterBase CreateFromRange(DateTime firstTime, DateTime lastTime) + where TKey : TimestampPointIDBase, new() + { + return new FixedRange((ulong)firstTime.Ticks, (ulong)lastTime.Ticks); + } - /// - /// Creates a filter over a set of date ranges (Similar to down sampled queries) - /// - /// the first time if the query (inclusive) - /// the last time of the query (inclusive if contained in the intervals) - /// the exact interval - /// the width of every window - /// A that will be able to do this parsing - /// - /// Example uses. FirstTime = 1/1/2013. LastTime = 1/2/2013. - /// MainInterval = 0.1 seconds. SubInterval = 0.0333333 seconds. - /// Tolerance = 0.001 seconds. - /// - public static SeekFilterBase CreateFromIntervalData(ulong firstTime, ulong lastTime, ulong interval, ulong tolerance) - where TKey : TimestampPointIDBase, new() - { - return new IntervalRanges(firstTime, lastTime, interval, interval, tolerance); - } + /// + /// Creates a filter over a single date range. (Inclusive list) + /// + /// the first time if the query (inclusive) + /// the last time of the query (inclusive) + /// + public static SeekFilterBase CreateFromRange(ulong firstTime, ulong lastTime) + where TKey : TimestampPointIDBase, new() + { + return new FixedRange(firstTime, lastTime); + } - /// - /// Creates a filter over a single date range. (Inclusive list) - /// - /// the first time if the query (inclusive) - /// the last time of the query (inclusive) - /// the smallest interval that is exact - /// the interval that will be parsed. Possible to be rounded - /// the width of every window - /// A that will be able to do this parsing - /// - /// Example uses. FirstTime = 1/1/2013. LastTime = 1/2/2013. - /// MainInterval = 0.1 seconds. SubInterval = 0.0333333 seconds. - /// Tolerance = 0.001 seconds. - /// - public static SeekFilterBase CreateFromIntervalData(DateTime firstTime, DateTime lastTime, TimeSpan mainInterval, TimeSpan subInterval, TimeSpan tolerance) - where TKey : TimestampPointIDBase, new() - { - return new IntervalRanges((ulong)firstTime.Ticks, (ulong)lastTime.Ticks, (ulong)mainInterval.Ticks, (ulong)subInterval.Ticks, (ulong)tolerance.Ticks); - } + /// + /// Creates a filter over a set of date ranges (Similar to down sampled queries) + /// + /// the first time if the query (inclusive) + /// the last time of the query (inclusive if contained in the intervals) + /// the smallest interval that is exact + /// the interval that will be parsed. Possible to be rounded + /// the width of every window + /// A that will be able to do this parsing + /// + /// Example uses. FirstTime = 1/1/2013. LastTime = 1/2/2013. + /// MainInterval = 0.1 seconds. SubInterval = 0.0333333 seconds. + /// Tolerance = 0.001 seconds. + /// + public static SeekFilterBase CreateFromIntervalData(ulong firstTime, ulong lastTime, ulong mainInterval, ulong subInterval, ulong tolerance) + where TKey : TimestampPointIDBase, new() + { + return new IntervalRanges(firstTime, lastTime, mainInterval, subInterval, tolerance); + } - /// - /// Creates a filter over a single date range. (Inclusive list) - /// - /// the first time if the query (inclusive) - /// the last time of the query (inclusive) - /// the exact interval to do the scan - /// the width of every window - /// A that will be able to do this parsing - /// - /// Example uses. FirstTime = 1/1/2013. LastTime = 1/2/2013. - /// Interval = 0.1 seconds. - /// Tolerance = 0.001 seconds. - /// - public static SeekFilterBase CreateFromIntervalData(DateTime firstTime, DateTime lastTime, TimeSpan interval, TimeSpan tolerance) - where TKey : TimestampPointIDBase, new() - { - return new IntervalRanges((ulong)firstTime.Ticks, (ulong)lastTime.Ticks, (ulong)interval.Ticks, (ulong)interval.Ticks, (ulong)tolerance.Ticks); - } + /// + /// Creates a filter over a set of date ranges (Similar to down sampled queries) + /// + /// the first time if the query (inclusive) + /// the last time of the query (inclusive if contained in the intervals) + /// the exact interval + /// the width of every window + /// A that will be able to do this parsing + /// + /// Example uses. FirstTime = 1/1/2013. LastTime = 1/2/2013. + /// MainInterval = 0.1 seconds. SubInterval = 0.0333333 seconds. + /// Tolerance = 0.001 seconds. + /// + public static SeekFilterBase CreateFromIntervalData(ulong firstTime, ulong lastTime, ulong interval, ulong tolerance) + where TKey : TimestampPointIDBase, new() + { + return new IntervalRanges(firstTime, lastTime, interval, interval, tolerance); + } - /// - /// Loads a from the provided . - /// - /// The stream to load the filter from - /// - [MethodImpl(MethodImplOptions.NoOptimization)] - private static SeekFilterBase CreateFromStream(BinaryStreamBase stream) - where TKey : TimestampPointIDBase, new() - { - byte version = stream.ReadUInt8(); - switch (version) - { - case 0: - return null; - case 1: - return new FixedRange(stream); - case 2: - return new IntervalRanges(stream); - default: - throw new VersionNotFoundException("Unknown Version"); - } - } + /// + /// Creates a filter over a single date range. (Inclusive list) + /// + /// the first time if the query (inclusive) + /// the last time of the query (inclusive) + /// the smallest interval that is exact + /// the interval that will be parsed. Possible to be rounded + /// the width of every window + /// A that will be able to do this parsing + /// + /// Example uses. FirstTime = 1/1/2013. LastTime = 1/2/2013. + /// MainInterval = 0.1 seconds. SubInterval = 0.0333333 seconds. + /// Tolerance = 0.001 seconds. + /// + public static SeekFilterBase CreateFromIntervalData(DateTime firstTime, DateTime lastTime, TimeSpan mainInterval, TimeSpan subInterval, TimeSpan tolerance) + where TKey : TimestampPointIDBase, new() + { + return new IntervalRanges((ulong)firstTime.Ticks, (ulong)lastTime.Ticks, (ulong)mainInterval.Ticks, (ulong)subInterval.Ticks, (ulong)tolerance.Ticks); + } + /// + /// Creates a filter over a single date range. (Inclusive list) + /// + /// the first time if the query (inclusive) + /// the last time of the query (inclusive) + /// the exact interval to do the scan + /// the width of every window + /// A that will be able to do this parsing + /// + /// Example uses. FirstTime = 1/1/2013. LastTime = 1/2/2013. + /// Interval = 0.1 seconds. + /// Tolerance = 0.001 seconds. + /// + public static SeekFilterBase CreateFromIntervalData(DateTime firstTime, DateTime lastTime, TimeSpan interval, TimeSpan tolerance) + where TKey : TimestampPointIDBase, new() + { + return new IntervalRanges((ulong)firstTime.Ticks, (ulong)lastTime.Ticks, (ulong)interval.Ticks, (ulong)interval.Ticks, (ulong)tolerance.Ticks); + } + /// + /// Loads a from the provided . + /// + /// The stream to load the filter from + /// + [MethodImpl(MethodImplOptions.NoOptimization)] + private static SeekFilterBase CreateFromStream(BinaryStreamBase stream) + where TKey : TimestampPointIDBase, new() + { + byte version = stream.ReadUInt8(); + + return version switch + { + 0 => null, + 1 => new FixedRange(stream), + 2 => new IntervalRanges(stream), + _ => throw new VersionNotFoundException("Unknown Version") + }; } -} +} \ No newline at end of file diff --git a/Source/Libraries/GSF.SortedTreeStore/Snap/Services/Configuration/AdvancedServerDatabaseConfig.cs b/Source/Libraries/GSF.SortedTreeStore/Snap/Services/Configuration/AdvancedServerDatabaseConfig.cs index d3d8626e16..1a02110484 100755 --- a/Source/Libraries/GSF.SortedTreeStore/Snap/Services/Configuration/AdvancedServerDatabaseConfig.cs +++ b/Source/Libraries/GSF.SortedTreeStore/Snap/Services/Configuration/AdvancedServerDatabaseConfig.cs @@ -33,269 +33,283 @@ using GSF.Snap.Storage; using GSF.Units; -namespace GSF.Snap.Services.Configuration +namespace GSF.Snap.Services.Configuration; + +/// +/// Creates a configuration for the database to utilize. +/// +public class AdvancedServerDatabaseConfig + : IToServerDatabaseSettings + where TKey : SnapTypeBase, new() + where TValue : SnapTypeBase, new() { + private readonly string m_mainPath; + private string m_finalFileExtension; + private string m_intermediateFileExtension; + + /// + /// Whether finalized timestamp archives receive exact ID-presence indexes. + /// + public bool EnablePointIDIndex { get; set; } = true; + + /// + /// Options applied to Bloom sidecars at every disk rollover stage. + /// + public TimeBucketBloomFilterSettings BloomFilter { get; set; } = new(); + /// - /// Creates a configuration for the database to utilize. + /// Gets a database config. /// - public class AdvancedServerDatabaseConfig - : IToServerDatabaseSettings - where TKey : SnapTypeBase, new() - where TValue : SnapTypeBase, new() + public AdvancedServerDatabaseConfig(string databaseName, string mainPath, bool supportsWriting) { - private readonly string m_mainPath; - private string m_finalFileExtension; - private string m_intermediateFileExtension; - - /// - /// Gets a database config. - /// - public AdvancedServerDatabaseConfig(string databaseName, string mainPath, bool supportsWriting) - { - SupportsWriting = supportsWriting; - DatabaseName = databaseName; - m_mainPath = mainPath; - m_intermediateFileExtension = ".d2i"; - m_finalFileExtension = ".d2"; - ImportAttachedPathsAtStartup = true; - ImportPaths = new List(); - FinalWritePaths = new List(); - ArchiveEncodingMethod = EncodingDefinition.FixedSizeCombinedEncoding; - StreamingEncodingMethods = new List(); - TargetFileSize = 2L * SI2.Giga; - DesiredRemainingSpace = 5L * SI2.Giga; - StagingCount = 3; - DirectoryMethod = ArchiveDirectoryMethod.TopDirectoryOnly; - FillMethod = ArchiveDirectoryFillMethod.Sequential; - DiskFlushInterval = 10000; - CacheFlushInterval = 100; - } + SupportsWriting = supportsWriting; + DatabaseName = databaseName; + m_mainPath = mainPath; + m_intermediateFileExtension = ".d2i"; + m_finalFileExtension = ".d2"; + ImportAttachedPathsAtStartup = true; + ImportPaths = []; + FinalWritePaths = []; + ArchiveEncodingMethod = EncodingDefinition.FixedSizeCombinedEncoding; + StreamingEncodingMethods = []; + TargetFileSize = 2L * SI2.Giga; + DesiredRemainingSpace = 5L * SI2.Giga; + StagingCount = 3; + DirectoryMethod = ArchiveDirectoryMethod.TopDirectoryOnly; + FillMethod = ArchiveDirectoryFillMethod.Sequential; + DiskFlushInterval = 10000; + CacheFlushInterval = 100; + } - /// - /// Gets the method of how the directory will be stored. Defaults to - /// top directory only. - /// - public ArchiveDirectoryMethod DirectoryMethod { get; set; } - - /// - /// Gets or sets how a write path is selected from the final write paths when more than one is configured. - /// Defaults to . - /// - public ArchiveDirectoryFillMethod FillMethod { get; set; } - - /// - /// The name associated with the database. - /// - public string DatabaseName { get; } - - /// - /// Gets or sets the desired size of the final stage archive files. - /// - /// - /// Value must be between 100MB and 1TB. - /// - public long TargetFileSize { get; set; } - - /// - /// Gets or sets the desired remaining drive space, in bytes, for final stage files. - /// - /// - /// Value must be between 100MB and 1TB. - /// - public long DesiredRemainingSpace { get; set; } - - /// - /// The number of stages. - /// - public int StagingCount { get; set; } - - /// - /// The extension to use for the intermediate files - /// - public string IntermediateFileExtension - { - get => m_intermediateFileExtension; - set => m_intermediateFileExtension = PathHelpers.FormatExtension(value); - } + /// + /// Gets the method of how the directory will be stored. Defaults to + /// top directory only. + /// + public ArchiveDirectoryMethod DirectoryMethod { get; set; } - /// - /// The extension to use for the final file - /// - public string FinalFileExtension - { - get => m_finalFileExtension; - set => m_finalFileExtension = PathHelpers.FormatExtension(value); - } + /// + /// Gets or sets how a write path is selected from the final write paths when more than one is configured. + /// Defaults to . + /// + public ArchiveDirectoryFillMethod FillMethod { get; set; } + + /// + /// The name associated with the database. + /// + public string DatabaseName { get; } + + /// + /// Gets or sets the desired size of the final stage archive files. + /// + /// + /// Value must be between 100MB and 1TB. + /// + public long TargetFileSize { get; set; } + + /// + /// Gets or sets the desired remaining drive space, in bytes, for final stage files. + /// + /// + /// Value must be between 100MB and 1TB. + /// + public long DesiredRemainingSpace { get; set; } + + /// + /// The number of stages. + /// + public int StagingCount { get; set; } + + /// + /// The extension to use for the intermediate files + /// + public string IntermediateFileExtension + { + get => m_intermediateFileExtension; + set => m_intermediateFileExtension = PathHelpers.FormatExtension(value); + } + + /// + /// The extension to use for the final file + /// + public string FinalFileExtension + { + get => m_finalFileExtension; + set => m_finalFileExtension = PathHelpers.FormatExtension(value); + } + + /// + /// Determines whether the server should import attached paths at startup. + /// + public bool ImportAttachedPathsAtStartup { get; set; } + + /// + /// Gets all paths that are known by this historian. + /// A path can be a file name or a folder. + /// + public List ImportPaths { get; } - /// - /// Determines whether the server should import attached paths at startup. - /// - public bool ImportAttachedPathsAtStartup { get; set; } - - /// - /// Gets all paths that are known by this historian. - /// A path can be a file name or a folder. - /// - public List ImportPaths { get; } - - /// - /// The list of directories where final files can be placed written. - /// If nothing is specified, the main directory is used. - /// - public List FinalWritePaths { get; } - - /// - /// Gets the default encoding methods for storing files. - /// - public EncodingDefinition ArchiveEncodingMethod { get; set; } - - /// - /// Gets the supported encoding methods for streaming data. This list is in a prioritized order. - /// - public List StreamingEncodingMethods { get; } - /// - /// Gets if writing will be supported - /// - public bool SupportsWriting { get; } - - /// - /// The number of milliseconds before data is automatically flushed to the disk. - /// - /// - /// Must be between 1,000 ms and 60,000 ms. - /// - public int DiskFlushInterval { get; set; } - - /// - /// The number of milliseconds before data is taken from its cache and put in the - /// memory file. - /// - /// - /// Must be between 1 and 1,000 - /// - public int CacheFlushInterval { get; set; } - - #region [ IToServerDatabaseSettings ] - - public ServerDatabaseSettings ToServerDatabaseSettings() + /// + /// The list of directories where final files can be placed written. + /// If nothing is specified, the main directory is used. + /// + public List FinalWritePaths { get; } + + /// + /// Gets the default encoding methods for storing files. + /// + public EncodingDefinition ArchiveEncodingMethod { get; set; } + + /// + /// Gets the supported encoding methods for streaming data. This list is in a prioritized order. + /// + public List StreamingEncodingMethods { get; } + /// + /// Gets if writing will be supported + /// + public bool SupportsWriting { get; } + + /// + /// The number of milliseconds before data is automatically flushed to the disk. + /// + /// + /// Must be between 1,000 ms and 60,000 ms. + /// + public int DiskFlushInterval { get; set; } + + /// + /// The number of milliseconds before data is taken from its cache and put in the + /// memory file. + /// + /// + /// Must be between 1 and 1,000 + /// + public int CacheFlushInterval { get; set; } + + #region [ IToServerDatabaseSettings ] + + public ServerDatabaseSettings ToServerDatabaseSettings() + { + ServerDatabaseSettings settings = new() { - ServerDatabaseSettings settings = new ServerDatabaseSettings - { - DatabaseName = DatabaseName - }; + DatabaseName = DatabaseName + }; - if (SupportsWriting) - ToWriteProcessorSettings(settings.WriteProcessor); + if (SupportsWriting) + ToWriteProcessorSettings(settings.WriteProcessor); - settings.SupportsWriting = SupportsWriting; - ToArchiveListSettings(settings.ArchiveList); + settings.SupportsWriting = SupportsWriting; + ToArchiveListSettings(settings.ArchiveList); - settings.RolloverLog.LogPath = m_mainPath; - settings.KeyType = new TKey().GenericTypeGuid; - settings.ValueType = new TValue().GenericTypeGuid; + settings.RolloverLog.LogPath = m_mainPath; + settings.KeyType = new TKey().GenericTypeGuid; + settings.ValueType = new TValue().GenericTypeGuid; - if (StreamingEncodingMethods.Count == 0) - settings.StreamingEncodingMethods.Add(EncodingDefinition.FixedSizeCombinedEncoding); - else - settings.StreamingEncodingMethods.AddRange(StreamingEncodingMethods); + if (StreamingEncodingMethods.Count == 0) + settings.StreamingEncodingMethods.Add(EncodingDefinition.FixedSizeCombinedEncoding); + else + settings.StreamingEncodingMethods.AddRange(StreamingEncodingMethods); - return settings; - } + return settings; + } - private void ToWriteProcessorSettings(WriteProcessorSettings settings) - { - if (!SupportsWriting) - return; + private void ToWriteProcessorSettings(WriteProcessorSettings settings) + { + if (!SupportsWriting) + return; - ValidateExtension(IntermediateFileExtension, out string intermediateFilePendingExtension, out string intermediateFileFinalExtension); - ValidateExtension(FinalFileExtension, out string finalFilePendingExtension, out string finalFileFinalExtension); + ValidateExtension(IntermediateFileExtension, out string intermediateFilePendingExtension, out string intermediateFileFinalExtension); + ValidateExtension(FinalFileExtension, out string finalFilePendingExtension, out string finalFileFinalExtension); - List finalPaths = new List(); + List finalPaths = []; - if (FinalWritePaths.Count > 0) - finalPaths.AddRange(FinalWritePaths); - else - finalPaths.Add(m_mainPath); + if (FinalWritePaths.Count > 0) + finalPaths.AddRange(FinalWritePaths); + else + finalPaths.Add(m_mainPath); - settings.IsEnabled = true; + settings.IsEnabled = true; - // 0.1 seconds - settings.PrebufferWriter.RolloverInterval = CacheFlushInterval; - settings.PrebufferWriter.MaximumPointCount = 25000; - settings.PrebufferWriter.RolloverPointCount = 25000; + // 0.1 seconds + settings.PrebufferWriter.RolloverInterval = CacheFlushInterval; + settings.PrebufferWriter.MaximumPointCount = 25000; + settings.PrebufferWriter.RolloverPointCount = 25000; - // 10 seconds - settings.FirstStageWriter.MaximumAllowedMb = 100; // about 10 million points - settings.FirstStageWriter.RolloverSizeMb = 100; // about 10 million points - settings.FirstStageWriter.RolloverInterval = DiskFlushInterval; // 10 seconds - settings.FirstStageWriter.EncodingMethod = ArchiveEncodingMethod; - settings.FirstStageWriter.FinalSettings.ConfigureOnDisk(new[] { m_mainPath }, SI2.Giga, ArchiveDirectoryMethod.TopDirectoryOnly, ArchiveEncodingMethod, "stage1", intermediateFilePendingExtension, intermediateFileFinalExtension, FileFlags.Stage1, FileFlags.IntermediateFile); + // 10 seconds + settings.FirstStageWriter.MaximumAllowedMb = 100; // about 10 million points + settings.FirstStageWriter.RolloverSizeMb = 100; // about 10 million points + settings.FirstStageWriter.RolloverInterval = DiskFlushInterval; // 10 seconds + settings.FirstStageWriter.EncodingMethod = ArchiveEncodingMethod; + settings.FirstStageWriter.FinalSettings.ConfigureOnDisk([m_mainPath], SI2.Giga, ArchiveDirectoryMethod.TopDirectoryOnly, ArchiveEncodingMethod, "stage1", intermediateFilePendingExtension, intermediateFileFinalExtension, FileFlags.Stage1, FileFlags.IntermediateFile); - for (int stage = 2; stage <= StagingCount; stage++) - { - int remainingStages = StagingCount - stage; - - CombineFilesSettings rollover = new CombineFilesSettings(); - - if (remainingStages > 0) - { - rollover.ArchiveSettings.ConfigureOnDisk(new[] { m_mainPath }, SI2.Giga, - ArchiveDirectoryMethod.TopDirectoryOnly, ArchiveEncodingMethod, "stage" + stage, - intermediateFilePendingExtension, intermediateFileFinalExtension, FileFlags.GetStage(stage), FileFlags.IntermediateFile); - } - else - { - // Final staging file - this is the only stage that writes across the configured final paths, so the - // fill method (sequential vs round-robin) applies here - rollover.ArchiveSettings.ConfigureOnDisk(finalPaths, DesiredRemainingSpace, - DirectoryMethod, ArchiveEncodingMethod, DatabaseName.ToNonNullNorEmptyString("stage" + stage).RemoveInvalidFileNameCharacters(), - finalFilePendingExtension, finalFileFinalExtension, FileFlags.GetStage(stage)); - rollover.ArchiveSettings.FillMethod = FillMethod; - } - - rollover.LogPath = m_mainPath; - rollover.ExecuteTimer = 1000; - rollover.CombineOnFileCount = 60; - rollover.CombineOnFileSize = TargetFileSize / (long)Math.Pow(30, remainingStages); - rollover.MatchFlag = FileFlags.GetStage(stage - 1); - settings.StagingRollovers.Add(rollover); - } - } + settings.FirstStageWriter.FinalSettings.BloomFilter = BloomFilter; + settings.FirstStageWriter.FinalSettings.EnablePointIDIndex = EnablePointIDIndex; - private void ToArchiveListSettings(ArchiveListSettings listSettings) + for (int stage = 2; stage <= StagingCount; stage++) { - ValidateExtension(IntermediateFileExtension, out string _, out string intermediateFileFinalExtension); - ValidateExtension(FinalFileExtension, out string _, out string finalFileFinalExtension); + int remainingStages = StagingCount - stage; - listSettings.AddExtension(intermediateFileFinalExtension); - listSettings.AddExtension(finalFileFinalExtension); + CombineFilesSettings rollover = new(); - if (!string.IsNullOrWhiteSpace(m_mainPath)) - listSettings.AddPath(m_mainPath); - - if (ImportAttachedPathsAtStartup) + if (remainingStages > 0) + { + rollover.ArchiveSettings.ConfigureOnDisk([m_mainPath], SI2.Giga, + ArchiveDirectoryMethod.TopDirectoryOnly, ArchiveEncodingMethod, "stage" + stage, + intermediateFilePendingExtension, intermediateFileFinalExtension, FileFlags.GetStage(stage), FileFlags.IntermediateFile); + } + else { - listSettings.AddPaths(ImportPaths); - listSettings.AddPaths(FinalWritePaths); + // Final staging file - this is the only stage that writes across the configured final paths, so the + // fill method (sequential vs round-robin) applies here + rollover.ArchiveSettings.ConfigureOnDisk(finalPaths, DesiredRemainingSpace, + DirectoryMethod, ArchiveEncodingMethod, DatabaseName.ToNonNullNorEmptyString("stage" + stage).RemoveInvalidFileNameCharacters(), + finalFilePendingExtension, finalFileFinalExtension, FileFlags.GetStage(stage)); + rollover.ArchiveSettings.FillMethod = FillMethod; } - listSettings.LogSettings.LogPath = m_mainPath; + rollover.ArchiveSettings.BloomFilter = BloomFilter; + rollover.ArchiveSettings.EnablePointIDIndex = EnablePointIDIndex; + rollover.LogPath = m_mainPath; + rollover.ExecuteTimer = 1000; + rollover.CombineOnFileCount = 60; + rollover.CombineOnFileSize = TargetFileSize / (long)Math.Pow(30, remainingStages); + rollover.MatchFlag = FileFlags.GetStage(stage - 1); + settings.StagingRollovers.Add(rollover); } + } - private static void ValidateExtension(string extension, out string pending, out string final) - { - if (string.IsNullOrWhiteSpace(extension)) - throw new ArgumentException("Cannot be null or whitespace", nameof(extension)); + private void ToArchiveListSettings(ArchiveListSettings listSettings) + { + ValidateExtension(IntermediateFileExtension, out string _, out string intermediateFileFinalExtension); + ValidateExtension(FinalFileExtension, out string _, out string finalFileFinalExtension); - extension = extension.Trim(); - - if (extension.Contains(".")) - extension = extension.Substring(extension.IndexOf('.') + 1); + listSettings.AddExtension(intermediateFileFinalExtension); + listSettings.AddExtension(finalFileFinalExtension); + + if (!string.IsNullOrWhiteSpace(m_mainPath)) + listSettings.AddPath(m_mainPath); - pending = ".~" + extension; - final = "." + extension; + if (ImportAttachedPathsAtStartup) + { + listSettings.AddPaths(ImportPaths); + listSettings.AddPaths(FinalWritePaths); } - #endregion + listSettings.LogSettings.LogPath = m_mainPath; } + + private static void ValidateExtension(string extension, out string pending, out string final) + { + if (string.IsNullOrWhiteSpace(extension)) + throw new ArgumentException("Cannot be null or whitespace", nameof(extension)); + + extension = extension.Trim(); + + if (extension.Contains(".")) + extension = extension.Substring(extension.IndexOf('.') + 1); + + pending = ".~" + extension; + final = "." + extension; + } + + #endregion } \ No newline at end of file diff --git a/Source/Libraries/GSF.SortedTreeStore/Snap/Services/Reader/BufferedArchiveStream'2.cs b/Source/Libraries/GSF.SortedTreeStore/Snap/Services/Reader/BufferedArchiveStream'2.cs index 722b8df24b..9ad4dee2e1 100755 --- a/Source/Libraries/GSF.SortedTreeStore/Snap/Services/Reader/BufferedArchiveStream'2.cs +++ b/Source/Libraries/GSF.SortedTreeStore/Snap/Services/Reader/BufferedArchiveStream'2.cs @@ -18,7 +18,6 @@ // ---------------------------------------------------------------------------------------------------- // 10/25/2013 - Steven E. Chisholm // Generated original version of source code. -// // //****************************************************************************************************** @@ -26,63 +25,73 @@ using GSF.Snap.Storage; using GSF.Snap.Tree; -namespace GSF.Snap.Services.Reader +namespace GSF.Snap.Services.Reader; + +public class BufferedArchiveStream + : IDisposable + where TKey : SnapTypeBase, new() + where TValue : SnapTypeBase, new() { - public class BufferedArchiveStream - : IDisposable - where TKey : SnapTypeBase, new() - where TValue : SnapTypeBase, new() + private readonly ArchiveTableSummary m_table; + private SortedTreeTableReadSnapshot m_snapshot; + + public SortedTreeScannerBase Scanner; + + /// + /// An index value that is used to disassociate the archive file. Passed to this class from the + /// + public int Index { get; private set; } + + internal bool HasTimeBucketBloomIndex => m_table.ActiveSnapshotInfo.HasTimeBucketBloomIndex; + + internal bool MayContainPointID(ulong id, ulong lower, ulong upper) => m_table.ActiveSnapshotInfo.MayContainPointID(id, lower, upper); + + internal bool MayContainPointID(ulong id) => m_table.ActiveSnapshotInfo.MayContainPointID(id); + + internal bool ContainsRange(TKey lower, TKey upper) => m_table.Contains(lower, upper); + + /// + /// Creates the table reader. + /// + /// + /// + public BufferedArchiveStream(int index, ArchiveTableSummary table) + { + Index = index; + m_table = table; + m_snapshot = m_table.ActiveSnapshotInfo.CreateReadSnapshot(); + Scanner = m_snapshot.GetTreeScanner(); + } + + public bool CacheIsValid; + + public TKey CacheKey = new TKey(); + + public TValue CacheValue = new TValue(); + + public void UpdateCachedValue() + { + CacheIsValid = Scanner.Peek(CacheKey, CacheValue); + } + + public void SkipToNextKeyAndUpdateCachedValue() + { + CacheIsValid = Scanner.Read(CacheKey, CacheValue); + CacheIsValid = Scanner.Peek(CacheKey, CacheValue); + } + + public void SeekToKeyAndUpdateCacheValue(TKey key) + { + Scanner.SeekToKey(key); + CacheIsValid = Scanner.Peek(CacheKey, CacheValue); + } + + public void Dispose() { - public SortedTreeScannerBase Scanner; - private readonly ArchiveTableSummary m_table; - private SortedTreeTableReadSnapshot m_snapshot; - - /// - /// An index value that is used to disassociate the archive file. Passed to this class from the - /// - public int Index { get; private set; } - - /// - /// Creates the table reader. - /// - /// - /// - public BufferedArchiveStream(int index, ArchiveTableSummary table) - { - Index = index; - m_table = table; - m_snapshot = m_table.ActiveSnapshotInfo.CreateReadSnapshot(); - Scanner = m_snapshot.GetTreeScanner(); - } - - public bool CacheIsValid; - public TKey CacheKey = new TKey(); - public TValue CacheValue = new TValue(); - - public void UpdateCachedValue() - { - CacheIsValid = Scanner.Peek(CacheKey, CacheValue); - } - - public void SkipToNextKeyAndUpdateCachedValue() - { - CacheIsValid = Scanner.Read(CacheKey, CacheValue); - CacheIsValid = Scanner.Peek(CacheKey, CacheValue); - } - - public void SeekToKeyAndUpdateCacheValue(TKey key) - { - Scanner.SeekToKey(key); - CacheIsValid = Scanner.Peek(CacheKey, CacheValue); - } - - public void Dispose() - { - if (m_snapshot != null) - { - m_snapshot.Dispose(); - m_snapshot = null; - } - } + if (m_snapshot is null) + return; + + m_snapshot.Dispose(); + m_snapshot = null; } -} +} \ No newline at end of file diff --git a/Source/Libraries/GSF.SortedTreeStore/Snap/Services/Reader/NearestTimestampReader.cs b/Source/Libraries/GSF.SortedTreeStore/Snap/Services/Reader/NearestTimestampReader.cs new file mode 100644 index 0000000000..3133afe5fb --- /dev/null +++ b/Source/Libraries/GSF.SortedTreeStore/Snap/Services/Reader/NearestTimestampReader.cs @@ -0,0 +1,262 @@ +//****************************************************************************************************** +// NearestTimestampReader.cs - Gbtc +// +// Copyright © 2026, Grid Protection Alliance. All Rights Reserved. +// +// Licensed to the Grid Protection Alliance (GPA) under one or more contributor license agreements. See +// the NOTICE file distributed with this work for additional information regarding copyright ownership. +// The GPA licenses this file to you under the MIT License (MIT), the "License"; you may not use this +// file except in compliance with the License. You may obtain a copy of the License at: +// +// http://opensource.org/licenses/MIT +// +// Unless agreed to in writing, the subject software distributed under the License is distributed on an +// "AS-IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. Refer to the +// License for the specific language governing permissions and limitations. +// +// Code Modification History: +// ---------------------------------------------------------------------------------------------------- +// 09/21/2026 - J. Ritchie Carroll +// Generated original version of source code. +// +//****************************************************************************************************** + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using GSF.Snap.Filters; + +namespace GSF.Snap.Services.Reader; + +// Uses the existing reader's snapshot, match filter, timeout and cancellation lifetime. +internal sealed class NearestTimestampReader + where TKey : SnapTypeBase, new() + where TValue : SnapTypeBase, new() +{ + private sealed class Candidate + { + public readonly TKey Key = new(); + public readonly TValue Value = new(); + public bool Found; + public ulong Distance; + } + + private readonly NearestTimestampSeekFilter m_filter; + private readonly SeekFilterBase m_bounds; + private readonly List> m_tables; + private readonly MatchFilterBase m_match; + private readonly Func m_cancelled; + private readonly SortedTreeEngineReaderOptions m_options; + private readonly Dictionary m_candidates; + private readonly Dictionary, Dictionary> m_tableCandidates; + private Dictionary m_activeCandidates; + private readonly Dictionary m_windowCandidates = new(); + private readonly Queue m_results = new(); + private readonly TKey m_target = new(); + private readonly TKey m_lower = new(); + private readonly TKey m_upper = new(); + private long m_scanned; + private long m_seeks; + private long m_returned; + private bool m_stopped; + private int m_missing; + private ulong m_farthest; + private readonly Func m_forwardVisitor; + private readonly Func m_backwardVisitor; + + internal NearestTimestampReader(NearestTimestampSeekFilter filter, SeekFilterBase bounds, + List> tables, MatchFilterBase match, + SortedTreeEngineReaderOptions options, Func cancelled) + { + m_filter = filter; + m_bounds = bounds; + m_tables = tables; + m_match = match; + m_options = options; + m_cancelled = cancelled; + m_candidates = filter.PointIDs.ToDictionary(id => id, id => new Candidate()); + m_tableCandidates = tables.ToDictionary(table => table, loadTableCandidates); + m_forwardVisitor = (key, value) => Visit(key, value, false); + m_backwardVisitor = (key, value) => Visit(key, value, true); + m_bounds.Reset(); + return; + + Dictionary loadTableCandidates(BufferedArchiveStream table) + { + return m_candidates.Where(pair => !options.UsePointIDIndex || table.MayContainPointID(pair.Key)).ToDictionary(pair => pair.Key, pair => pair.Value); + } + } + + internal bool Read(TKey key, TValue value) + { + if (m_tables.Count == 0 || m_stopped || m_cancelled() || (m_options.MaxReturnedCount > 0 && m_returned >= m_options.MaxReturnedCount)) + return false; + + while (m_results.Count == 0) + { + if (!m_bounds.NextWindow() || m_cancelled()) + return false; + + m_bounds.StartOfFrame.CopyTo(m_lower); + m_bounds.EndOfFrame.CopyTo(m_upper); + + if (m_lower.IsLessThan(m_bounds.StartOfRange)) + m_bounds.StartOfRange.CopyTo(m_lower); + + if (m_upper.IsGreaterThan(m_bounds.EndOfRange)) + m_bounds.EndOfRange.CopyTo(m_upper); + + if (m_lower.IsGreaterThan(m_upper)) + continue; + + foreach (Candidate candidate in m_candidates.Values) + candidate.Found = false; + + NearestTimestampSeekFilter.SetTimestamp(m_target, m_filter.TargetTimestamp, false); + + foreach (BufferedArchiveStream table in m_tables) + { + m_activeCandidates = m_tableCandidates[table]; + + if (m_activeCandidates.Count == 0 || !table.ContainsRange(m_lower, m_upper)) + continue; + + if (m_options.UseTimeBucketBloomFilter && table.HasTimeBucketBloomIndex) + { + m_windowCandidates.Clear(); + + ulong lowerTime = NearestTimestampSeekFilter.TimeKey(m_lower).Timestamp; + ulong upperTime = NearestTimestampSeekFilter.TimeKey(m_upper).Timestamp; + + foreach (KeyValuePair pair in m_activeCandidates) + { + if (m_cancelled()) + return false; + + if (table.MayContainPointID(pair.Key, lowerTime, upperTime)) + m_windowCandidates.Add(pair.Key, pair.Value); + } + + m_activeCandidates = m_windowCandidates; + + if (m_activeCandidates.Count == 0) + continue; + } + + m_missing = 0; + m_farthest = 0; + + foreach (Candidate candidate in m_activeCandidates.Values) + { + if (!candidate.Found) + m_missing++; + else if (candidate.Distance > m_farthest) + m_farthest = candidate.Distance; + } + + if (!CanSeek()) + return false; + + table.Scanner.VisitForward(m_target.IsLessThan(m_lower) ? m_lower : m_target, + m_forwardVisitor); + + if (m_stopped || m_cancelled()) + return false; + + // An exact match for every requested ID cannot be improved by a preceding + // timestamp. Continue checking other archives forwards for same-time key ties. + if (m_filter.SearchMode == TimestampSearchMode.AtOrAfter || (m_missing == 0 && m_farthest == 0)) + continue; + + if (!CanSeek()) + return false; + + table.Scanner.VisitBackward(m_target, m_backwardVisitor); + + if (m_stopped || m_cancelled()) + return false; + } + + foreach (Candidate candidate in m_candidates.Values.Where(c => c.Found).OrderBy(c => c.Key, Comparer.Create((a, b) => a.CompareTo(b)))) + m_results.Enqueue(candidate); + } + + + Candidate result = m_results.Dequeue(); + + result.Key.CopyTo(key); + result.Value.CopyTo(value); + + m_returned++; + + return true; + } + + private bool CanSeek() + { + if (m_cancelled() || (m_options.MaxSeekCount > 0 && m_seeks >= m_options.MaxSeekCount)) + { + m_stopped = true; + return false; + } + + m_seeks++; + + Interlocked.Increment(ref Stats.SeeksRequested); + + return true; + } + + private bool Visit(TKey key, TValue value, bool backwards) + { + if (m_cancelled() || (m_options.MaxScanCount > 0 && m_scanned >= m_options.MaxScanCount)) + { + m_stopped = true; + return false; + } + + m_scanned++; + + Interlocked.Increment(ref Stats.PointsScanned); + + if (backwards ? key.IsLessThan(m_lower) : key.IsGreaterThan(m_upper)) + return false; + + if (key.IsLessThan(m_lower) || key.IsGreaterThan(m_upper)) + return true; + + ulong timestamp = NearestTimestampSeekFilter.TimeKey(key).Timestamp; + ulong target = m_filter.TargetTimestamp; + ulong distance = timestamp >= target ? timestamp - target : target - timestamp; + + // Once every ID has a better candidate than any remaining timestamp, the rest + // of this direction (potentially hours of dense data) is skipped. + if (m_missing == 0 && distance > m_farthest) + return false; + + ulong idValue = NearestTimestampSeekFilter.TimeKey(key).PointID; + + if (!m_activeCandidates.TryGetValue(idValue, out Candidate candidate) || (candidate.Found && distance > candidate.Distance) || !m_match.Contains(key, value)) + return true; + + if (candidate.Found && distance >= candidate.Distance && (distance != candidate.Distance || !key.IsLessThan(candidate.Key))) + return true; + + bool refreshFarthest = !candidate.Found || candidate.Distance == m_farthest; + + if (!candidate.Found) + m_missing--; + + key.CopyTo(candidate.Key); + value.CopyTo(candidate.Value); + + candidate.Distance = distance; + candidate.Found = true; + + if (m_missing == 0 && refreshFarthest) + m_farthest = m_activeCandidates.Values.Max(c => c.Distance); + + return true; + } +} \ No newline at end of file diff --git a/Source/Libraries/GSF.SortedTreeStore/Snap/Services/Reader/SequentialReaderStream'2.cs b/Source/Libraries/GSF.SortedTreeStore/Snap/Services/Reader/SequentialReaderStream'2.cs index fdbe5ffb08..393b465589 100755 --- a/Source/Libraries/GSF.SortedTreeStore/Snap/Services/Reader/SequentialReaderStream'2.cs +++ b/Source/Libraries/GSF.SortedTreeStore/Snap/Services/Reader/SequentialReaderStream'2.cs @@ -31,572 +31,596 @@ using GSF.Snap.Tree; using GSF.Threading; -namespace GSF.Snap.Services.Reader +namespace GSF.Snap.Services.Reader; + +internal class SequentialReaderStream + : TreeStream + where TKey : SnapTypeBase, new() + where TValue : SnapTypeBase, new() { - internal class SequentialReaderStream - : TreeStream - where TKey : SnapTypeBase, new() - where TValue : SnapTypeBase, new() + private static readonly LogPublisher Log = Logger.CreatePublisher(typeof(SequentialReaderStream), MessageClass.Framework); + + public event Action> Disposed; + + private ArchiveListSnapshot m_snapshot; + private volatile bool m_timedOut; + private long m_pointCount; + private readonly NearestTimestampReader m_nearestReader; + + private readonly bool m_keyMatchIsUniverse; + + private readonly SeekFilterBase m_keySeekFilter; + private readonly MatchFilterBase m_keyMatchFilter; + + private TimeoutOperation m_timeout; + private List> m_tablesOrigList; + private readonly CustomSortHelper> m_sortedArchiveStreams; + private BufferedArchiveStream m_firstTable; + private SortedTreeScannerBase m_firstTableScanner; + private readonly TKey m_readWhileUpperBounds = new(); + private readonly TKey m_nextArchiveStreamLowerBounds = new(); + private WorkerThreadSynchronization m_workerThreadSynchronization; + private readonly bool m_ownsWorkerThreadSynchronization; + + public SequentialReaderStream(ArchiveList archiveList, + SortedTreeEngineReaderOptions readerOptions = null, + SeekFilterBase keySeekFilter = null, + MatchFilterBase keyMatchFilter = null, + WorkerThreadSynchronization workerThreadSynchronization = null) { - private static readonly LogPublisher Log = Logger.CreatePublisher(typeof(SequentialReaderStream), MessageClass.Framework); - - public event Action> Disposed; - - private ArchiveListSnapshot m_snapshot; - private volatile bool m_timedOut; - private long m_pointCount; - - private readonly bool m_keyMatchIsUniverse; - - private readonly SeekFilterBase m_keySeekFilter; - private readonly MatchFilterBase m_keyMatchFilter; - - private TimeoutOperation m_timeout; - private List> m_tablesOrigList; - private readonly CustomSortHelper> m_sortedArchiveStreams; - private BufferedArchiveStream m_firstTable; - private SortedTreeScannerBase m_firstTableScanner; - private readonly TKey m_readWhileUpperBounds = new TKey(); - private readonly TKey m_nextArchiveStreamLowerBounds = new TKey(); - private WorkerThreadSynchronization m_workerThreadSynchronization; - private readonly bool m_ownsWorkerThreadSynchronization; - - public SequentialReaderStream(ArchiveList archiveList, - SortedTreeEngineReaderOptions readerOptions = null, - SeekFilterBase keySeekFilter = null, - MatchFilterBase keyMatchFilter = null, - WorkerThreadSynchronization workerThreadSynchronization = null) + readerOptions ??= SortedTreeEngineReaderOptions.Default; + keySeekFilter ??= new SeekFilterUniverse(); + keyMatchFilter ??= new MatchFilterUniverse(); + + if (workerThreadSynchronization is null) { - if (readerOptions is null) - readerOptions = SortedTreeEngineReaderOptions.Default; - if (keySeekFilter is null) - keySeekFilter = new SeekFilterUniverse(); - if (keyMatchFilter is null) - keyMatchFilter = new MatchFilterUniverse(); - if (workerThreadSynchronization is null) - { - m_ownsWorkerThreadSynchronization = true; - workerThreadSynchronization = new WorkerThreadSynchronization(); - } + m_ownsWorkerThreadSynchronization = true; + workerThreadSynchronization = new WorkerThreadSynchronization(); + } - m_workerThreadSynchronization = workerThreadSynchronization; - m_pointCount = 0; - m_keySeekFilter = keySeekFilter; - m_keyMatchFilter = keyMatchFilter; - m_keyMatchIsUniverse = m_keyMatchFilter as MatchFilterUniverse != null; + m_workerThreadSynchronization = workerThreadSynchronization; + m_pointCount = 0; + m_keySeekFilter = keySeekFilter; + m_keyMatchFilter = keyMatchFilter; + m_keyMatchIsUniverse = m_keyMatchFilter as MatchFilterUniverse is not null; - if (readerOptions.Timeout.Ticks > 0) - { - m_timeout = new TimeoutOperation(); - m_timeout.RegisterTimeout(readerOptions.Timeout, () => m_timedOut = true); - } + if (readerOptions.Timeout.Ticks > 0) + { + m_timeout = new TimeoutOperation(); + m_timeout.RegisterTimeout(readerOptions.Timeout, () => m_timedOut = true); + } - m_snapshot = archiveList.CreateNewClientResources(); - m_snapshot.UpdateSnapshot(); - m_tablesOrigList = new List>(); + m_snapshot = archiveList.CreateNewClientResources(); + m_snapshot.UpdateSnapshot(); + m_tablesOrigList = []; - for (int x = 0; x < m_snapshot.Tables.Count(); x++) + for (int x = 0; x < m_snapshot.Tables.Count(); x++) + { + ArchiveTableSummary table = m_snapshot.Tables[x]; + + if (table is null) + continue; + + if (table.Contains(keySeekFilter.StartOfRange, keySeekFilter.EndOfRange)) { - ArchiveTableSummary table = m_snapshot.Tables[x]; - if (table != null) + try { - if (table.Contains(keySeekFilter.StartOfRange, keySeekFilter.EndOfRange)) - { - try - { - m_tablesOrigList.Add(new BufferedArchiveStream(x, table)); - } - catch (Exception e) - { - //ToDo: Make sure firstkey.tostring doesn't ever throw an exception. - StringBuilder sb = new StringBuilder(); - sb.AppendLine($"Archive ID {table.FileId}"); - sb.AppendLine($"First Key {table.FirstKey.ToString()}"); - sb.AppendLine($"Last Key {table.LastKey.ToString()}"); - sb.AppendLine($"File Size {table.SortedTreeTable.BaseFile.ArchiveSize}"); - sb.AppendLine($"File Name {table.SortedTreeTable.BaseFile.FilePath}"); - Log.Publish(MessageLevel.Error, "Error while reading file", sb.ToString(), null, e); - } - } - else - { - m_snapshot.Tables[x] = null; - } + m_tablesOrigList.Add(new BufferedArchiveStream(x, table)); + } + catch (Exception ex) + { + //TODO: Make sure firstkey.tostring doesn't ever throw an exception. + StringBuilder sb = new(); + sb.AppendLine($"Archive ID {table.FileId}"); + sb.AppendLine($"First Key {table.FirstKey}"); + sb.AppendLine($"Last Key {table.LastKey}"); + sb.AppendLine($"File Size {table.SortedTreeTable.BaseFile.ArchiveSize}"); + sb.AppendLine($"File Name {table.SortedTreeTable.BaseFile.FilePath}"); + Log.Publish(MessageLevel.Error, "Error while reading file", sb.ToString(), null, ex); } - } - - m_sortedArchiveStreams = new CustomSortHelper>(m_tablesOrigList, IsLessThan); - - m_keySeekFilter.Reset(); - if (m_keySeekFilter.NextWindow()) - { - SeekToKey(m_keySeekFilter.StartOfFrame); } else { - Dispose(); + m_snapshot.Tables[x] = null; } } - ~SequentialReaderStream() - { - Dispose(false); - } + m_sortedArchiveStreams = new CustomSortHelper>(m_tablesOrigList, IsLessThan); - /// - /// Provides a thread safe way to cancel a reader. - /// - public void CancelReader() + SeekFilterBase nearestSource = keySeekFilter; + + if (nearestSource is NearestTimestampSeekFilter nearest) { - m_timedOut = true; - m_workerThreadSynchronization.RequestCallback(Dispose); + int cancellationPoll = 0; + + m_nearestReader = new NearestTimestampReader(nearest, keySeekFilter, m_tablesOrigList, keyMatchFilter, readerOptions, () => + { + if ((cancellationPoll++ & 1023) == 0) + m_workerThreadSynchronization.PulseSafeToCallback(); + + return m_timedOut; + }); + + return; } + m_keySeekFilter.Reset(); + + if (m_keySeekFilter.NextWindow()) + SeekToKey(m_keySeekFilter.StartOfFrame); + else + Dispose(); + } + + ~SequentialReaderStream() + { + Dispose(false); + } + + /// + /// Provides a thread safe way to cancel a reader. + /// + public void CancelReader() + { + m_timedOut = true; + m_workerThreadSynchronization.RequestCallback(Dispose); + } - public override bool IsAlwaysSequential => true; + public override bool IsAlwaysSequential => true; - public override bool NeverContainsDuplicates => true; + public override bool NeverContainsDuplicates => true; - protected override bool ReadNext(TKey key, TValue value) + protected override bool ReadNext(TKey key, TValue value) + { + if (m_nearestReader is not null) { - if (!m_timedOut && m_firstTableScanner != null) - { - if (m_keyMatchIsUniverse) - { - if (m_firstTableScanner.ReadWhile(key, value, m_readWhileUpperBounds)) - { - m_pointCount++; - if (m_pointCount > 10000) - { - m_workerThreadSynchronization.PulseSafeToCallback(); - Interlocked.Add(ref Stats.PointsReturned, m_pointCount); - m_pointCount = 0; - } - return true; - } - } - else - { - if (m_firstTableScanner.ReadWhile(key, value, m_readWhileUpperBounds, m_keyMatchFilter)) - { - m_pointCount++; - if (m_pointCount > 10000) - { - m_workerThreadSynchronization.PulseSafeToCallback(); - Interlocked.Add(ref Stats.PointsReturned, m_pointCount); - m_pointCount = 0; - } - return true; - } - } - } - return ReadCatchAll(key, value); + if (m_timedOut || !m_nearestReader.Read(key, value)) + return false; + + m_pointCount++; + return true; } - private bool ReadCatchAll(TKey key, TValue value) + if (m_timedOut || m_firstTableScanner is null) + return ReadCatchAll(key, value); + + if (m_keyMatchIsUniverse) { + if (!m_firstTableScanner.ReadWhile(key, value, m_readWhileUpperBounds)) + return ReadCatchAll(key, value); + + m_pointCount++; + + if (m_pointCount <= 10000) + return true; + m_workerThreadSynchronization.PulseSafeToCallback(); - if (m_pointCount > 10000) - { - Interlocked.Add(ref Stats.PointsReturned, m_pointCount); - m_pointCount = 0; - } + + Interlocked.Add(ref Stats.PointsReturned, m_pointCount); + + m_pointCount = 0; + return true; + } - long retries = 0; + if (!m_firstTableScanner.ReadWhile(key, value, m_readWhileUpperBounds, m_keyMatchFilter)) + return ReadCatchAll(key, value); + + m_pointCount++; + + if (m_pointCount <= 10000) + return true; + + m_workerThreadSynchronization.PulseSafeToCallback(); + + Interlocked.Add(ref Stats.PointsReturned, m_pointCount); + + m_pointCount = 0; + return true; + } - TryAgain: - if (!m_timedOut) + private bool ReadCatchAll(TKey key, TValue value) + { + m_workerThreadSynchronization.PulseSafeToCallback(); + + if (m_pointCount > 10000) + { + Interlocked.Add(ref Stats.PointsReturned, m_pointCount); + m_pointCount = 0; + } + + long retries = 0; + + TryAgain: + if (!m_timedOut) + { + retries++; + + if (retries >= 10000) { - retries++; + m_workerThreadSynchronization.PulseSafeToCallback(); + Interlocked.Add(ref Stats.PointsScanned, retries); + retries = 0; + } - if (retries >= 10000) - { - m_workerThreadSynchronization.PulseSafeToCallback(); - Interlocked.Add(ref Stats.PointsScanned, retries); - retries = 0; - } + if (m_keyMatchIsUniverse) + { + if (m_firstTableScanner is null) + return false; - if (m_keyMatchIsUniverse) - { - if (m_firstTableScanner is null) - return false; - - if (m_firstTableScanner.ReadWhile(key, value, m_readWhileUpperBounds) || ReadWhileFollowupActions(key, value, null)) - { - m_pointCount++; - return true; - } - goto TryAgain; - } - else + if (m_firstTableScanner.ReadWhile(key, value, m_readWhileUpperBounds) || ReadWhileFollowupActions(key, value, null)) { - if (m_firstTableScanner is null) - return false; - - if (m_firstTableScanner.ReadWhile(key, value, m_readWhileUpperBounds, m_keyMatchFilter) || ReadWhileFollowupActions(key, value, m_keyMatchFilter)) - { - m_pointCount++; - return true; - } - goto TryAgain; + m_pointCount++; + return true; } + goto TryAgain; } - Dispose(); - return false; - } - private bool ReadWhileFollowupActions(TKey key, TValue value, MatchFilterBase filter) - { - //There are certain followup requirements when a ReadWhile method returns false. - //Condition 1: - // The end of the node has been reached. - //Response: - // It returned false to allow for additional checks such as timeouts to occur. - // Do Nothing. - // - //Condition 2: - // The archive stream may no longer be in order and needs to be checked - //Response: - // Resort the archive stream - // - //Condition 3: - // The end of the frame has been reached - //Response: - // Advance to the next frame - // Also test the edge case where the current point might be equal to the end of the frame - // since this is an inclusive filter and ReadWhile is exclusive. - // If it's part of the frame, return true after Advancing the frame and the point. - // - - //Update the cached values for the table so proper analysis can be done. - m_firstTable.UpdateCachedValue(); - - //Check Condition 1 - if (m_firstTable.CacheIsValid && m_firstTable.CacheKey.IsLessThan(m_readWhileUpperBounds)) + if (m_firstTableScanner is null) return false; - //Since condition 2 and 3 can occur at the same time, verifying the sort of the Archive Stream is a good thing to do. - VerifyArchiveStreamSortingOrder(); - - if (EOS) - return false; - - //Check if Condition 3's exception occured. - if (m_firstTable.CacheKey.IsEqualTo(m_keySeekFilter.EndOfFrame)) + if (m_firstTableScanner.ReadWhile(key, value, m_readWhileUpperBounds, m_keyMatchFilter) || ReadWhileFollowupActions(key, value, m_keyMatchFilter)) { - //This is the exception clause. I will advance the frame, but will still need to return the current point. - m_firstTable.Scanner.Read(key, value); - AdvanceSeekableFilter(true, key); - SetReadWhileUpperBoundsValue(); - - return filter is null || filter.Contains(key, value); + m_pointCount++; + return true; } + + goto TryAgain; + } - //Check if condition 3 occured - if (m_firstTable.CacheKey.IsGreaterThan(m_keySeekFilter.EndOfFrame)) - { - AdvanceSeekableFilter(true, m_firstTable.CacheKey); - SetReadWhileUpperBoundsValue(); - } + Dispose(); + return false; + } + + private bool ReadWhileFollowupActions(TKey key, TValue value, MatchFilterBase filter) + { + // There are certain followup requirements when a ReadWhile method returns false. + // Condition 1: + // The end of the node has been reached. + // Response: + // It returned false to allow for additional checks such as timeouts to occur. + // Do Nothing. + // + // Condition 2: + // The archive stream may no longer be in order and needs to be checked + // Response: + // Resort the archive stream + // + // Condition 3: + // The end of the frame has been reached + // Response: + // Advance to the next frame + // Also test the edge case where the current point might be equal to the end of the frame + // since this is an inclusive filter and ReadWhile is exclusive. + // If it's part of the frame, return true after Advancing the frame and the point. + + // Update the cached values for the table so proper analysis can be done. + m_firstTable.UpdateCachedValue(); + + // Check Condition 1 + if (m_firstTable.CacheIsValid && m_firstTable.CacheKey.IsLessThan(m_readWhileUpperBounds)) return false; - } + // Since condition 2 and 3 can occur at the same time, verifying the sort of the Archive Stream is a good thing to do. + VerifyArchiveStreamSortingOrder(); - //------------------------------------------------------------- + if (EOS) + return false; - /// - /// Will verify that the stream is in the proper order and remove any duplicates that were found. - /// May be called after every single read, but better to be called - /// when a ReadWhile function returns false. - /// - private void VerifyArchiveStreamSortingOrder() + // Check if Condition 3's exception occured. + if (m_firstTable.CacheKey.IsEqualTo(m_keySeekFilter.EndOfFrame)) { - if (EOS) - return; - - m_sortedArchiveStreams[0].UpdateCachedValue(); + // This is the exception clause. I will advance the frame, but will still need to return the current point. + m_firstTable.Scanner.Read(key, value); + AdvanceSeekableFilter(true, key); + SetReadWhileUpperBoundsValue(); - if (m_sortedArchiveStreams.Items.Length > 1) - { - //If list is no longer in order - int compare = CompareStreams(m_sortedArchiveStreams[0], m_sortedArchiveStreams[1]); - if (compare == 0 && m_sortedArchiveStreams[0].CacheIsValid) - { - //If a duplicate entry is found, advance the position of the duplicate entry - RemoveDuplicatesFromList(); - SetReadWhileUpperBoundsValue(); - } - if (compare > 0) - { - m_sortedArchiveStreams.SortAssumingIncreased(0); - m_firstTable = m_sortedArchiveStreams[0]; - m_firstTableScanner = m_firstTable.Scanner; - SetReadWhileUpperBoundsValue(); - } - if (compare == 0 && !m_sortedArchiveStreams[0].CacheIsValid) - { - Dispose(); - m_firstTable = null; - m_firstTableScanner = null; - } - } - else - { - if (!m_sortedArchiveStreams[0].CacheIsValid) - { - Dispose(); - m_firstTable = null; - m_firstTableScanner = null; - } - } + return filter is null || filter.Contains(key, value); } + // Check if condition 3 occured + if (!m_firstTable.CacheKey.IsGreaterThan(m_keySeekFilter.EndOfFrame)) + return false; + + AdvanceSeekableFilter(true, m_firstTable.CacheKey); + SetReadWhileUpperBoundsValue(); + return false; + } - /// - /// Does a seek operation on the current stream when there is a seek filter on the reader. - /// - /// - /// True if the provided key is still valid within the next best fitting frame. - /// - private bool AdvanceSeekableFilter(bool isValid, TKey key) - { - TryAgain: - if (m_keySeekFilter != null && m_keySeekFilter.NextWindow()) - { - //If the current point is a valid point. - //Check to see if the seek operation can be avoided. - //or if the next available point does not exist in this window. - if (isValid) - { - //If the current point is within this window - if (key.IsGreaterThanOrEqualTo(m_keySeekFilter.StartOfFrame) && - key.IsLessThanOrEqualTo(m_keySeekFilter.EndOfFrame)) - { - return true; - } - - //If the current point is after this window, seek to the next window. - if (key.IsGreaterThan(m_keySeekFilter.EndOfFrame)) - goto TryAgain; - } + //------------------------------------------------------------- - //If the current point is not valid, or is before m_startKey - //Advance the scanner to the next window. - SeekAllArchiveStreamsForward(m_keySeekFilter.StartOfFrame); - return false; - } - Dispose(); - m_firstTableScanner = null; - m_firstTable = null; - return false; + /// + /// Will verify that the stream is in the proper order and remove any duplicates that were found. + /// May be called after every single read, but better to be called + /// when a ReadWhile function returns false. + /// + private void VerifyArchiveStreamSortingOrder() + { + if (EOS) + return; - } + m_sortedArchiveStreams[0].UpdateCachedValue(); - private bool IsLessThan(BufferedArchiveStream item1, BufferedArchiveStream item2) + if (m_sortedArchiveStreams.Items.Length > 1) { - if (!item1.CacheIsValid && !item2.CacheIsValid) - return false; - if (!item1.CacheIsValid) - return false; - if (!item2.CacheIsValid) - return true; - return item1.CacheKey.IsLessThan(item2.CacheKey);// item1.CurrentKey.CompareTo(item2.CurrentKey); - } - /// - /// Compares two Archive Streams together for proper sorting. - /// - /// - /// - /// - private int CompareStreams(BufferedArchiveStream item1, BufferedArchiveStream item2) - { - if (!item1.CacheIsValid && !item2.CacheIsValid) - return 0; - if (!item1.CacheIsValid) - return 1; - if (!item2.CacheIsValid) - return -1; - return item1.CacheKey.CompareTo(item2.CacheKey);// item1.CurrentKey.CompareTo(item2.CurrentKey); - } - - /// - /// Does an unconditional seek operation to the provided key. - /// - /// - private void SeekToKey(TKey key) - { - foreach (BufferedArchiveStream table in m_sortedArchiveStreams.Items) + // If list is no longer in order + int compare = CompareStreams(m_sortedArchiveStreams[0], m_sortedArchiveStreams[1]); + if (compare == 0 && m_sortedArchiveStreams[0].CacheIsValid) { - table.SeekToKeyAndUpdateCacheValue(key); + // If a duplicate entry is found, advance the position of the duplicate entry + RemoveDuplicatesFromList(); + SetReadWhileUpperBoundsValue(); } - m_sortedArchiveStreams.Sort(); - //Remove any duplicates - RemoveDuplicatesIfExists(); - - if (m_sortedArchiveStreams.Items.Length > 0) + if (compare > 0) { + m_sortedArchiveStreams.SortAssumingIncreased(0); m_firstTable = m_sortedArchiveStreams[0]; m_firstTableScanner = m_firstTable.Scanner; + SetReadWhileUpperBoundsValue(); } - else + + if (compare == 0 && !m_sortedArchiveStreams[0].CacheIsValid) { + Dispose(); m_firstTable = null; m_firstTableScanner = null; } - - SetReadWhileUpperBoundsValue(); } - - /// - /// Seeks the streams only in the forward direction. - /// This means that if the current position in any stream is invalid or past this point, - /// the stream will not seek backwards. - /// - /// the key to seek to - private void SeekAllArchiveStreamsForward(TKey key) + else { - foreach (BufferedArchiveStream table in m_sortedArchiveStreams.Items) - { - if (table.CacheIsValid && table.CacheKey.IsLessThan(key)) - { - table.SeekToKeyAndUpdateCacheValue(key); - } - } - //Resorts the entire list. - m_sortedArchiveStreams.Sort(); + if (m_sortedArchiveStreams[0].CacheIsValid) + return; + + Dispose(); + m_firstTable = null; + m_firstTableScanner = null; + } + } - //Remove any duplicates - RemoveDuplicatesIfExists(); - if (m_sortedArchiveStreams.Items.Length > 0) - { - m_firstTable = m_sortedArchiveStreams[0]; - m_firstTableScanner = m_firstTable.Scanner; - } - else + /// + /// Does a seek operation on the current stream when there is a seek filter on the reader. + /// + /// + /// True if the provided key is still valid within the next best fitting frame. + /// + private bool AdvanceSeekableFilter(bool isValid, TKey key) + { + + TryAgain: + if (m_keySeekFilter is not null && m_keySeekFilter.NextWindow()) + { + // If the current point is a valid point. + // Check to see if the seek operation can be avoided. + // or if the next available point does not exist in this window. + if (isValid) { - m_firstTable = null; - m_firstTableScanner = null; + // If the current point is within this window + if (key.IsGreaterThanOrEqualTo(m_keySeekFilter.StartOfFrame) && key.IsLessThanOrEqualTo(m_keySeekFilter.EndOfFrame)) + return true; + + // If the current point is after this window, seek to the next window. + if (key.IsGreaterThan(m_keySeekFilter.EndOfFrame)) + goto TryAgain; } - SetReadWhileUpperBoundsValue(); + // If the current point is not valid, or is before m_startKey + // Advance the scanner to the next window. + SeekAllArchiveStreamsForward(m_keySeekFilter.StartOfFrame); + return false; } - /// - /// Checks the first 2 Archive Streams for a duplicate entry. If one exists, then removes the duplicate and resorts the list. - /// - private void RemoveDuplicatesIfExists() + Dispose(); + + m_firstTableScanner = null; + m_firstTable = null; + + return false; + + } + + private bool IsLessThan(BufferedArchiveStream item1, BufferedArchiveStream item2) + { + if (!item1.CacheIsValid && !item2.CacheIsValid) + return false; + + if (!item1.CacheIsValid) + return false; + + if (!item2.CacheIsValid) + return true; + + return item1.CacheKey.IsLessThan(item2.CacheKey);// item1.CurrentKey.CompareTo(item2.CurrentKey); + } + /// + /// Compares two Archive Streams together for proper sorting. + /// + /// + /// + /// + private int CompareStreams(BufferedArchiveStream item1, BufferedArchiveStream item2) + { + if (!item1.CacheIsValid && !item2.CacheIsValid) + return 0; + + if (!item1.CacheIsValid) + return 1; + + if (!item2.CacheIsValid) + return -1; + + return item1.CacheKey.CompareTo(item2.CacheKey); // item1.CurrentKey.CompareTo(item2.CurrentKey); + } + + /// + /// Does an unconditional seek operation to the provided key. + /// + /// + private void SeekToKey(TKey key) + { + foreach (BufferedArchiveStream table in m_sortedArchiveStreams.Items) + table.SeekToKeyAndUpdateCacheValue(key); + + m_sortedArchiveStreams.Sort(); + + // Remove any duplicates + RemoveDuplicatesIfExists(); + + if (m_sortedArchiveStreams.Items.Length > 0) { - if (m_sortedArchiveStreams.Items.Length > 1) - { - if (CompareStreams(m_sortedArchiveStreams[0], m_sortedArchiveStreams[1]) == 0 && m_sortedArchiveStreams[0].CacheIsValid) - { - //If a duplicate entry is found, advance the position of the duplicate entry - RemoveDuplicatesFromList(); - } - } + m_firstTable = m_sortedArchiveStreams[0]; + m_firstTableScanner = m_firstTable.Scanner; + } + else + { + m_firstTable = null; + m_firstTableScanner = null; } - /// - /// Call this function when the same point exists in multiple archive files. It will - /// read past the duplicate point in all other archive files and then resort the tables. - /// - /// Assums that the archiveStream's cached value is current. - /// - private void RemoveDuplicatesFromList() + SetReadWhileUpperBoundsValue(); + } + + /// + /// Seeks the streams only in the forward direction. + /// This means that if the current position in any stream is invalid or past this point, + /// the stream will not seek backwards. + /// + /// the key to seek to + private void SeekAllArchiveStreamsForward(TKey key) + { + foreach (BufferedArchiveStream table in m_sortedArchiveStreams.Items) { - int lastDuplicateIndex = -1; - for (int index = 1; index < m_sortedArchiveStreams.Items.Length; index++) - { - if (CompareStreams(m_sortedArchiveStreams[0], m_sortedArchiveStreams[index]) == 0) - { - m_sortedArchiveStreams[index].SkipToNextKeyAndUpdateCachedValue(); - lastDuplicateIndex = index; - } - else - { - break; - } - } + if (table.CacheIsValid && table.CacheKey.IsLessThan(key)) + table.SeekToKeyAndUpdateCacheValue(key); + } - //Resorts the list in reverse order. - for (int j = lastDuplicateIndex; j > 0; j--) - m_sortedArchiveStreams.SortAssumingIncreased(j); + // Resorts the entire list. + m_sortedArchiveStreams.Sort(); - SetReadWhileUpperBoundsValue(); + // Remove any duplicates + RemoveDuplicatesIfExists(); + + if (m_sortedArchiveStreams.Items.Length > 0) + { + m_firstTable = m_sortedArchiveStreams[0]; + m_firstTableScanner = m_firstTable.Scanner; + } + else + { + m_firstTable = null; + m_firstTableScanner = null; } - /// - /// Sets the read while upper bounds value. - /// Which is the lesser of - /// The first point in the adjacent table or - /// The end of the current seek window. - /// - private void SetReadWhileUpperBoundsValue() + SetReadWhileUpperBoundsValue(); + } + + /// + /// Checks the first 2 Archive Streams for a duplicate entry. If one exists, then removes the duplicate and resorts the list. + /// + private void RemoveDuplicatesIfExists() + { + if (m_sortedArchiveStreams.Items.Length <= 1) + return; + + // If a duplicate entry is found, advance the position of the duplicate entry + if (CompareStreams(m_sortedArchiveStreams[0], m_sortedArchiveStreams[1]) == 0 && m_sortedArchiveStreams[0].CacheIsValid) + RemoveDuplicatesFromList(); + } + + /// + /// Call this function when the same point exists in multiple archive files. It will + /// read past the duplicate point in all other archive files and then resort the tables. + /// + /// Assumes that the archiveStream's cached value is current. + /// + private void RemoveDuplicatesFromList() + { + int lastDuplicateIndex = -1; + + for (int index = 1; index < m_sortedArchiveStreams.Items.Length; index++) { - if (m_sortedArchiveStreams.Items.Length > 1 && m_sortedArchiveStreams[1].CacheIsValid) + if (CompareStreams(m_sortedArchiveStreams[0], m_sortedArchiveStreams[index]) == 0) { - m_sortedArchiveStreams[1].CacheKey.CopyTo(m_nextArchiveStreamLowerBounds); + // If a duplicate entry is found, advance the position of the duplicate entry + m_sortedArchiveStreams[index].SkipToNextKeyAndUpdateCachedValue(); + lastDuplicateIndex = index; } else { - m_nextArchiveStreamLowerBounds.SetMax(); - } - m_nextArchiveStreamLowerBounds.CopyTo(m_readWhileUpperBounds); - - //If there is a key seek filter. adjust this bounds if necessary - if (m_keySeekFilter != null) - { - if (m_keySeekFilter.EndOfFrame.IsLessThan(m_readWhileUpperBounds)) - { - m_keySeekFilter.EndOfFrame.CopyTo(m_readWhileUpperBounds); - } + break; } } - protected override void Dispose(bool disposing) - { - try - { - if (Disposed != null) - Disposed(this); - } - catch (Exception) - { + //Resorts the list in reverse order. + for (int j = lastDuplicateIndex; j > 0; j--) + m_sortedArchiveStreams.SortAssumingIncreased(j); - } + SetReadWhileUpperBoundsValue(); + } - Interlocked.Add(ref Stats.PointsReturned, m_pointCount); - m_pointCount = 0; + /// + /// Sets the read while upper bounds value. + /// Which is the lesser of + /// The first point in the adjacent table or + /// The end of the current seek window. + /// + private void SetReadWhileUpperBoundsValue() + { + if (m_sortedArchiveStreams.Items.Length > 1 && m_sortedArchiveStreams[1].CacheIsValid) + m_sortedArchiveStreams[1].CacheKey.CopyTo(m_nextArchiveStreamLowerBounds); + else + m_nextArchiveStreamLowerBounds.SetMax(); + + m_nextArchiveStreamLowerBounds.CopyTo(m_readWhileUpperBounds); + + // If there is a key seek filter. adjust this bounds if necessary + if (m_keySeekFilter is null) + return; + + if (m_keySeekFilter.EndOfFrame.IsLessThan(m_readWhileUpperBounds)) + m_keySeekFilter.EndOfFrame.CopyTo(m_readWhileUpperBounds); + } - if (m_timeout != null) - { - m_timeout.Cancel(); - m_timeout = null; - } + protected override void Dispose(bool disposing) + { + try + { + Disposed?.Invoke(this); + } + catch (Exception) + { + // Ignore any exceptions thrown by the Disposed event handlers + } - if (m_tablesOrigList != null) - { - m_tablesOrigList.ForEach(x => x.Dispose()); - m_tablesOrigList = null; - Array.Clear(m_snapshot.Tables, 0, m_snapshot.Tables.Length); - } - m_timedOut = true; + Interlocked.Add(ref Stats.PointsReturned, m_pointCount); + m_pointCount = 0; - if (m_snapshot != null) - { - m_snapshot.Dispose(); - m_snapshot = null; - } + if (m_timeout is not null) + { + m_timeout.Cancel(); + m_timeout = null; + } - if (m_workerThreadSynchronization != null && m_ownsWorkerThreadSynchronization) - { - m_workerThreadSynchronization.Dispose(); - m_workerThreadSynchronization = null; - } + if (m_tablesOrigList is not null) + { + m_tablesOrigList.ForEach(x => x.Dispose()); + m_tablesOrigList = null; + Array.Clear(m_snapshot.Tables, 0, m_snapshot.Tables.Length); + } + + m_timedOut = true; + + if (m_snapshot is not null) + { + m_snapshot.Dispose(); + m_snapshot = null; } + if (m_workerThreadSynchronization is null || !m_ownsWorkerThreadSynchronization) + return; + + m_workerThreadSynchronization.Dispose(); + m_workerThreadSynchronization = null; } } \ No newline at end of file diff --git a/Source/Libraries/GSF.SortedTreeStore/Snap/Services/Reader/SortedTreeEngineReaderOptions.cs b/Source/Libraries/GSF.SortedTreeStore/Snap/Services/Reader/SortedTreeEngineReaderOptions.cs index ab686b62fa..761947cf44 100755 --- a/Source/Libraries/GSF.SortedTreeStore/Snap/Services/Reader/SortedTreeEngineReaderOptions.cs +++ b/Source/Libraries/GSF.SortedTreeStore/Snap/Services/Reader/SortedTreeEngineReaderOptions.cs @@ -25,110 +25,155 @@ using System.Data; using GSF.IO; -namespace GSF.Snap.Services.Reader +namespace GSF.Snap.Services.Reader; + +/// +/// Contains the options to use for executing an individual read request. +/// +public class SortedTreeEngineReaderOptions { /// - /// Contains the options to use for executing an individual read request. + /// Whether nearest reads may use exact per-file ID-presence indexes. + /// + public bool UsePointIDIndex { get; } = true; + + /// + /// Whether nearest reads may use time-bucket Bloom indexes. + /// + public bool UseTimeBucketBloomFilter { get; } = true; + + /// + /// Creates reader options with independent index bypass controls. /// - public class SortedTreeEngineReaderOptions + /// Whether to use the point ID index. + /// Whether to use the time-bucket Bloom filter. + /// The time before a query will end prematurely. + /// The maximum number of Key/Values to send to the client before ending prematurely. + /// The maximum number of points for the database to read before ending prematurely. + /// The maximum seeks that will occur before ending prematurely. + public SortedTreeEngineReaderOptions(bool usePointIDIndex, bool useTimeBucketBloomFilter, + TimeSpan timeout = default, long maxReturnedCount = 0, long maxScanCount = 0, long maxSeekCount = 0) + : this(timeout, maxReturnedCount, maxScanCount, maxSeekCount) { - /// - /// Default options. Same as default constructor - /// - public static SortedTreeEngineReaderOptions Default - { - get; - private set; - } + UsePointIDIndex = usePointIDIndex; + UseTimeBucketBloomFilter = useTimeBucketBloomFilter; + } - static SortedTreeEngineReaderOptions() - { - Default = new SortedTreeEngineReaderOptions(); - } + /// + /// Default options. Same as default constructor + /// + public static SortedTreeEngineReaderOptions Default + { + get; + private set; + } - /// - /// Creates . - /// - /// the time before a query will end prematurely - /// the maximum number of Key/Values to send to the client before ending prematurely - /// the maximum number of points for the database to read before ending prematurely - /// the maximum seeks that will occur before ending prematurely - public SortedTreeEngineReaderOptions(TimeSpan timeout = default, long maxReturnedCount = 0, long maxScanCount = 0, long maxSeekCount = 0) - { - Timeout = timeout; - MaxReturnedCount = maxReturnedCount; - MaxScanCount = maxScanCount; - MaxSeekCount = maxSeekCount; - } + static SortedTreeEngineReaderOptions() + { + Default = new SortedTreeEngineReaderOptions(); + } - /// - /// Creates a new from a stream - /// - /// the stream to read from - public SortedTreeEngineReaderOptions(BinaryStreamBase stream) - { - byte version = stream.ReadUInt8(); - switch (version) - { - case 0: - Timeout = new TimeSpan(stream.ReadInt64()); - MaxReturnedCount = stream.ReadInt64(); - MaxScanCount = stream.ReadInt64(); - MaxSeekCount = stream.ReadInt64(); - break; - default: - throw new VersionNotFoundException("Unknown Version"); - } - } + /// + /// Creates . + /// + /// the time before a query will end prematurely + /// the maximum number of Key/Values to send to the client before ending prematurely + /// the maximum number of points for the database to read before ending prematurely + /// the maximum seeks that will occur before ending prematurely + public SortedTreeEngineReaderOptions(TimeSpan timeout = default, long maxReturnedCount = 0, long maxScanCount = 0, long maxSeekCount = 0) + { + Timeout = timeout; + MaxReturnedCount = maxReturnedCount; + MaxScanCount = maxScanCount; + MaxSeekCount = maxSeekCount; + } - /// - /// Writes this data to the . - /// - /// the stream to write data to - public void Save(BinaryStreamBase stream) - { - stream.Write((byte)0); - stream.Write(Timeout.Ticks); - stream.Write(MaxReturnedCount); - stream.Write(MaxScanCount); - stream.Write(MaxSeekCount); - } + /// + /// Creates a new from a stream + /// + /// the stream to read from + public SortedTreeEngineReaderOptions(BinaryStreamBase stream) + { + byte version = stream.ReadUInt8(); - /// - /// The time before the query times out. - /// - public TimeSpan Timeout + switch (version) { - get; - private set; - } + case 1: + case 0: + Timeout = new TimeSpan(stream.ReadInt64()); + MaxReturnedCount = stream.ReadInt64(); + MaxScanCount = stream.ReadInt64(); + MaxSeekCount = stream.ReadInt64(); - /// - /// The maximum number of points to return. 0 means no limit. - /// - public long MaxReturnedCount - { - get; - private set; + if (version >= 1) + { + UsePointIDIndex = stream.ReadBoolean(); + UseTimeBucketBloomFilter = stream.ReadBoolean(); + } + break; + default: + throw new VersionNotFoundException("Unknown Version"); } + } - /// - /// The maximum number of points to scan to get the results set. - /// This includes any point that was filtered - /// - public long MaxScanCount - { - get; - private set; - } + /// + /// Writes this data to the . + /// + /// the stream to write data to + public void Save(BinaryStreamBase stream) + { + // Options travel over the wire from client to server and version 0 servers reject any + // other version, so version 1 is only written when an index is disabled. Both indexes + // being enabled is the default, which is what a version 0 payload implies on load. + bool extended = !UsePointIDIndex || !UseTimeBucketBloomFilter; - /// - /// The maximum number of seeks permitted - /// - public long MaxSeekCount + stream.Write((byte)(extended ? 1 : 0)); + stream.Write(Timeout.Ticks); + stream.Write(MaxReturnedCount); + stream.Write(MaxScanCount); + stream.Write(MaxSeekCount); + + if (extended) { - get; - private set; + stream.Write(UsePointIDIndex); + stream.Write(UseTimeBucketBloomFilter); } } + + /// + /// The time before the query times out. + /// + public TimeSpan Timeout + { + get; + private set; + } + + /// + /// The maximum number of points to return. 0 means no limit. + /// + public long MaxReturnedCount + { + get; + private set; + } + + /// + /// The maximum number of points to scan to get the results set. + /// This includes any point that was filtered + /// + public long MaxScanCount + { + get; + private set; + } + + /// + /// The maximum number of seeks permitted + /// + public long MaxSeekCount + { + get; + private set; + } } \ No newline at end of file diff --git a/Source/Libraries/GSF.SortedTreeStore/Snap/Services/Writer/SimplifiedArchiveInitializer'2.cs b/Source/Libraries/GSF.SortedTreeStore/Snap/Services/Writer/SimplifiedArchiveInitializer'2.cs index 153a58e3e6..17ff59b40a 100755 --- a/Source/Libraries/GSF.SortedTreeStore/Snap/Services/Writer/SimplifiedArchiveInitializer'2.cs +++ b/Source/Libraries/GSF.SortedTreeStore/Snap/Services/Writer/SimplifiedArchiveInitializer'2.cs @@ -19,12 +19,10 @@ // 07/24/2012 - Steven E. Chisholm // Generated original version of source code. // -// //****************************************************************************************************** using System; using System.IO; -using System.Linq; using System.Threading; using GSF.Immutable; using GSF.IO; @@ -32,154 +30,145 @@ using GSF.Snap.Types; using GSF.Threading; -namespace GSF.Snap.Services.Writer +namespace GSF.Snap.Services.Writer; + +/// +/// Creates new archive files based on user settings. +/// +public class SimplifiedArchiveInitializer + where TKey : SnapTypeBase, new() + where TValue : SnapTypeBase, new() { + private readonly ReaderWriterLockEasy m_lock; + private int m_writePathSeed; + /// - /// Creates new archive files based on user settings. + /// Creates a /// - public class SimplifiedArchiveInitializer - where TKey : SnapTypeBase, new() - where TValue : SnapTypeBase, new() + /// + public SimplifiedArchiveInitializer(SimplifiedArchiveInitializerSettings settings) { - private SimplifiedArchiveInitializerSettings m_settings; - private readonly ReaderWriterLockEasy m_lock; - private int m_writePathSeed; - - /// - /// Creates a - /// - /// - public SimplifiedArchiveInitializer(SimplifiedArchiveInitializerSettings settings) - { - m_settings = settings.CloneReadonly(); - m_settings.Validate(); - m_lock = new ReaderWriterLockEasy(); - } + Settings = settings.CloneReadonly(); + Settings.Validate(); + m_lock = new ReaderWriterLockEasy(); + } - /// - /// Gets current settings. - /// - public SimplifiedArchiveInitializerSettings Settings => m_settings; + /// + /// Gets current settings. + /// + public SimplifiedArchiveInitializerSettings Settings { get; private set; } - /// - /// Replaces the existing settings with this new set. - /// - /// - public void UpdateSettings(SimplifiedArchiveInitializerSettings settings) - { - settings = settings.CloneReadonly(); - settings.Validate(); - using (m_lock.EnterWriteLock()) - { - m_settings = settings; - } - } + /// + /// Replaces the existing settings with this new set. + /// + /// + public void UpdateSettings(SimplifiedArchiveInitializerSettings settings) + { + settings = settings.CloneReadonly(); + settings.Validate(); - /// - /// Creates a new based on the settings passed to this class. - /// Once created, it is up to the caller to make sure that this class is properly disposed of. - /// - /// the first key in the archive file - /// the last key in the archive file - /// The estimated size of the file. -1 to ignore this feature and write to the first available directory. - /// the archiveId to assign to the new file. - /// - public SortedTreeTable CreateArchiveFile(TKey startKey, TKey endKey, long estimatedSize, TreeStream data, Action archiveIdCallback) + using (m_lock.EnterWriteLock()) { - SimplifiedArchiveInitializerSettings settings = m_settings; + Settings = settings; + } + } - string pendingFile = CreateArchiveName(GetPathWithEnoughSpace(estimatedSize), startKey, endKey); - string finalFile = Path.ChangeExtension(pendingFile, settings.FinalExtension); + /// + /// Creates a new based on the settings passed to this class. + /// Once created, it is up to the caller to make sure that this class is properly disposed of. + /// + /// the first key in the archive file + /// the last key in the archive file + /// The estimated size of the file. -1 to ignore this feature and write to the first available directory. + /// the archiveId to assign to the new file. + /// + public SortedTreeTable CreateArchiveFile(TKey startKey, TKey endKey, long estimatedSize, TreeStream data, Action archiveIdCallback) + { + SimplifiedArchiveInitializerSettings settings = Settings; - SortedTreeFileSimpleWriter.Create(pendingFile, finalFile, 4096, archiveIdCallback, settings.EncodingMethod, data, settings.Flags.ToArray()); + string pendingFile = CreateArchiveName(GetPathWithEnoughSpace(estimatedSize), startKey, endKey); + string finalFile = Path.ChangeExtension(pendingFile, settings.FinalExtension); - return SortedTreeFile.OpenFile(finalFile, true).OpenTable(); - } + SortedTreeFileSimpleWriter.CreateWithIndexes(pendingFile, finalFile, 4096, archiveIdCallback, settings.EncodingMethod, data, settings.BloomFilter, settings.EnablePointIDIndex, [.. settings.Flags]); - /// - /// Creates a new random file in one of the provided folders in a round robin fashion. - /// - /// - private string CreateArchiveName(string path) - { - path = GetPath(path, DateTime.Now); - return Path.Combine(path, $"{m_settings.Prefix.ToLower()}-{Guid.NewGuid()}-{DateTime.UtcNow.Ticks}{m_settings.PendingExtension}"); - } - - /// - /// Creates a new random file in one of the provided folders in a round robin fashion. - /// - /// - private string CreateArchiveName(string path, TKey startKey, TKey endKey) - { - IHasTimestampField startTime = startKey as IHasTimestampField; - IHasTimestampField endTime = endKey as IHasTimestampField; + return SortedTreeFile.OpenFile(finalFile, true).OpenTable(); + } - if (startTime is null || endTime is null) - return CreateArchiveName(path); + /// + /// Creates a new random file in one of the provided folders in a round robin fashion. + /// + /// + private string CreateArchiveName(string path) + { + path = GetPath(path, DateTime.Now); + return Path.Combine(path, $"{Settings.Prefix.ToLower()}-{Guid.NewGuid()}-{DateTime.UtcNow.Ticks}{Settings.PendingExtension}"); + } - if (!startTime.TryGetDateTime(out DateTime startDate) || !endTime.TryGetDateTime(out DateTime endDate)) - return CreateArchiveName(path); + /// + /// Creates a new random file in one of the provided folders in a round robin fashion. + /// + /// + private string CreateArchiveName(string path, TKey startKey, TKey endKey) + { + if (startKey is not IHasTimestampField startTime || endKey is not IHasTimestampField endTime || !startTime.TryGetDateTime(out DateTime startDate) || !endTime.TryGetDateTime(out DateTime endDate)) + return CreateArchiveName(path); - path = GetPath(path, startDate); - return Path.Combine(path, $"{m_settings.Prefix.ToLower()}-{startDate:yyyy-MM-dd HH.mm.ss.fff}_to_{endDate:yyyy-MM-dd HH.mm.ss.fff}-{DateTime.UtcNow.Ticks}{m_settings.PendingExtension}"); - } + path = GetPath(path, startDate); + return Path.Combine(path, $"{Settings.Prefix.ToLower()}-{startDate:yyyy-MM-dd HH.mm.ss.fff}_to_{endDate:yyyy-MM-dd HH.mm.ss.fff}-{DateTime.UtcNow.Ticks}{Settings.PendingExtension}"); + } - private string GetPath(string rootPath, DateTime time) + private string GetPath(string rootPath, DateTime time) + { + switch (Settings.DirectoryMethod) { - switch (m_settings.DirectoryMethod) - { - case ArchiveDirectoryMethod.TopDirectoryOnly: - break; - case ArchiveDirectoryMethod.Year: - rootPath = Path.Combine(rootPath, time.Year.ToString()); - break; - case ArchiveDirectoryMethod.YearMonth: - rootPath = Path.Combine(rootPath, time.Year.ToString() + time.Month.ToString("00")); - break; - case ArchiveDirectoryMethod.YearThenMonth: - rootPath = Path.Combine(rootPath, $"{time.Year}\\{time.Month:00}"); - break; - } - - if (!Directory.Exists(rootPath)) - Directory.CreateDirectory(rootPath); - - return rootPath; + case ArchiveDirectoryMethod.TopDirectoryOnly: + break; + case ArchiveDirectoryMethod.Year: + rootPath = Path.Combine(rootPath, time.Year.ToString()); + break; + case ArchiveDirectoryMethod.YearMonth: + rootPath = Path.Combine(rootPath, time.Year.ToString() + time.Month.ToString("00")); + break; + case ArchiveDirectoryMethod.YearThenMonth: + rootPath = Path.Combine(rootPath, $"{time.Year}\\{time.Month:00}"); + break; } - // Selects a write path with enough free space using the configured fill method. Sequential fill always starts - // at the first path (filling each in order); round-robin rotates the starting candidate via a per-instance - // interlocked seed so new files are distributed across the configured write paths. - private string GetPathWithEnoughSpace(long estimatedSize) - { - ImmutableList paths = m_settings.WritePath; - int count = paths.Count; + if (!Directory.Exists(rootPath)) + Directory.CreateDirectory(rootPath); + + return rootPath; + } - int start = m_settings.FillMethod == ArchiveDirectoryFillMethod.RoundRobin ? - ((Interlocked.Increment(ref m_writePathSeed) - 1) % count + count) % count : 0; + // Selects a write path with enough free space using the configured fill method. Sequential fill always starts + // at the first path (filling each in order); round-robin rotates the starting candidate via a per-instance + // interlocked seed so new files are distributed across the configured write paths. + private string GetPathWithEnoughSpace(long estimatedSize) + { + ImmutableList paths = Settings.WritePath; + int count = paths.Count; - // No size estimate: return the selected path without a free-space check - if (estimatedSize < 0) - return paths[start]; + int start = Settings.FillMethod == ArchiveDirectoryFillMethod.RoundRobin ? + ((Interlocked.Increment(ref m_writePathSeed) - 1) % count + count) % count : 0; - long remainingSpace = m_settings.DesiredRemainingSpace; + // No size estimate: return the selected path without a free-space check + if (estimatedSize < 0) + return paths[start]; - // Scan all candidate paths once, beginning at the selected position and wrapping around, returning the - // first whose drive has enough free space to store the estimated file while leaving the desired remaining space - for (int offset = 0; offset < count; offset++) - { - string path = paths[(start + offset) % count]; + long remainingSpace = Settings.DesiredRemainingSpace; - FilePath.GetAvailableFreeSpace(path, out long freeSpace, out _); + // Scan all candidate paths once, beginning at the selected position and wrapping around, returning the + // first whose drive has enough free space to store the estimated file while leaving the desired remaining space + for (int offset = 0; offset < count; offset++) + { + string path = paths[(start + offset) % count]; - if (freeSpace - estimatedSize > remainingSpace) - return path; - } + FilePath.GetAvailableFreeSpace(path, out long freeSpace, out _); - throw new Exception("Out of free space"); + if (freeSpace - estimatedSize > remainingSpace) + return path; } - + throw new Exception("Out of free space"); } } \ No newline at end of file diff --git a/Source/Libraries/GSF.SortedTreeStore/Snap/Services/Writer/SimplifiedArchiveInitializerSettings.cs b/Source/Libraries/GSF.SortedTreeStore/Snap/Services/Writer/SimplifiedArchiveInitializerSettings.cs index 588c9b028e..2493800a61 100755 --- a/Source/Libraries/GSF.SortedTreeStore/Snap/Services/Writer/SimplifiedArchiveInitializerSettings.cs +++ b/Source/Libraries/GSF.SortedTreeStore/Snap/Services/Writer/SimplifiedArchiveInitializerSettings.cs @@ -19,7 +19,6 @@ // 10/18/2014 - Steven E. Chisholm // Generated original version of source code. // -// //****************************************************************************************************** using System; @@ -28,269 +27,323 @@ using System.IO; using GSF.Immutable; using GSF.IO; +using GSF.Snap.Storage; + +namespace GSF.Snap.Services.Writer; -namespace GSF.Snap.Services.Writer +/// +/// Settings for . +/// +public class SimplifiedArchiveInitializerSettings + : SettingsBase { + private bool m_enablePointIDIndex = true; + private TimeBucketBloomFilterSettings m_bloomFilter = new(); + private ArchiveDirectoryMethod m_directoryMethod; + private ArchiveDirectoryFillMethod m_fillMethod; + private string m_prefix; + private string m_pendingExtension; + private string m_finalExtension; + private long m_desiredRemainingSpace; + private EncodingDefinition m_encodingMethod; + /// - /// Settings for . + /// Creates a new /// - public class SimplifiedArchiveInitializerSettings - : SettingsBase + public SimplifiedArchiveInitializerSettings() + { + Initialize(); + } + + private void Initialize() { - private ArchiveDirectoryMethod m_directoryMethod; - private ArchiveDirectoryFillMethod m_fillMethod; - private string m_prefix; - private string m_pendingExtension; - private string m_finalExtension; - private long m_desiredRemainingSpace; - private EncodingDefinition m_encodingMethod; - private ImmutableList m_writePath; - private ImmutableList m_flags; - - /// - /// Creates a new - /// - public SimplifiedArchiveInitializerSettings() + m_enablePointIDIndex = true; + m_bloomFilter = new TimeBucketBloomFilterSettings(); + m_directoryMethod = ArchiveDirectoryMethod.TopDirectoryOnly; + m_fillMethod = ArchiveDirectoryFillMethod.Sequential; + m_prefix = string.Empty; + m_pendingExtension = ".~d2i"; + m_finalExtension = ".d2i"; + m_desiredRemainingSpace = 5 * 1024 * 1024 * 1024L; //5GB + m_encodingMethod = EncodingDefinition.FixedSizeCombinedEncoding; + + WritePath = new ImmutableList(pathName => { - Initialize(); + PathHelpers.ValidatePathName(pathName); + return pathName; + }); + + Flags = []; + } + + /// + /// Whether newly finalized timestamp archives receive exact ID-presence indexes. + /// + public bool EnablePointIDIndex + { + get => m_enablePointIDIndex; + set + { + TestForEditable(); + m_enablePointIDIndex = value; } + } - private void Initialize() + /// + /// Options for optional time-bucket Bloom sidecars. + /// + public TimeBucketBloomFilterSettings BloomFilter + { + get => m_bloomFilter; + set { - m_directoryMethod = ArchiveDirectoryMethod.TopDirectoryOnly; - m_fillMethod = ArchiveDirectoryFillMethod.Sequential; - m_prefix = string.Empty; - m_pendingExtension = ".~d2i"; - m_finalExtension = ".d2i"; - m_desiredRemainingSpace = 5 * 1024 * 1024 * 1024L; //5GB - m_encodingMethod = EncodingDefinition.FixedSizeCombinedEncoding; - m_writePath = new ImmutableList((x) => - { - PathHelpers.ValidatePathName(x); - return x; - }); - m_flags = new ImmutableList(); + TestForEditable(); + m_bloomFilter = value ?? throw new ArgumentNullException(nameof(value)); } + } - /// - /// Gets the method that the directory structure will follow when writing a new file. - /// - public ArchiveDirectoryMethod DirectoryMethod + /// + /// Gets the method that the directory structure will follow when writing a new file. + /// + public ArchiveDirectoryMethod DirectoryMethod + { + get => m_directoryMethod; + set { - get => m_directoryMethod; - set - { - TestForEditable(); - m_directoryMethod = value; - } + TestForEditable(); + m_directoryMethod = value; } + } - /// - /// Gets or sets the method used to select a write path when more than one is configured. Defaults to - /// . - /// - public ArchiveDirectoryFillMethod FillMethod + /// + /// Gets or sets the method used to select a write path when more than one is configured. Defaults to + /// . + /// + public ArchiveDirectoryFillMethod FillMethod + { + get => m_fillMethod; + set { - get => m_fillMethod; - set - { - TestForEditable(); - m_fillMethod = value; - } + TestForEditable(); + m_fillMethod = value; } + } - /// - /// Gets/Sets the file prefix. Can be String.Empty for no prefix. - /// - public string Prefix + /// + /// Gets/Sets the file prefix. Can be String.Empty for no prefix. + /// + public string Prefix + { + get => m_prefix; + set { - get => m_prefix; - set - { - TestForEditable(); + TestForEditable(); - if (string.IsNullOrWhiteSpace(value)) - { - m_prefix = string.Empty; - return; - } + if (string.IsNullOrWhiteSpace(value)) + { + m_prefix = string.Empty; + return; + } - if (value.IndexOfAny(Path.GetInvalidFileNameChars()) >= 0) - throw new ArgumentException("filename has invalid characters.", nameof(value)); + if (value.IndexOfAny(Path.GetInvalidFileNameChars()) >= 0) + throw new ArgumentException("filename has invalid characters.", nameof(value)); - m_prefix = value; - } + m_prefix = value; } + } - /// - /// The list of all available paths to write files to - /// - public ImmutableList WritePath => m_writePath; + /// + /// The list of all available paths to write files to + /// + public ImmutableList WritePath { get; private set; } - /// - /// The extension to name the file. - /// - public string PendingExtension + /// + /// The extension to name the file. + /// + public string PendingExtension + { + get => m_pendingExtension; + set { - get => m_pendingExtension; - set - { - TestForEditable(); - m_pendingExtension = PathHelpers.FormatExtension(value); - } + TestForEditable(); + m_pendingExtension = PathHelpers.FormatExtension(value); } + } - /// - /// The extension to name the file. - /// - public string FinalExtension + /// + /// The extension to name the file. + /// + public string FinalExtension + { + get => m_finalExtension; + set { - get => m_finalExtension; - set - { - TestForEditable(); - m_finalExtension = PathHelpers.FormatExtension(value); - } + TestForEditable(); + m_finalExtension = PathHelpers.FormatExtension(value); } + } - /// - /// The flags that will be added to any created archive files. - /// - public ImmutableList Flags => m_flags; + /// + /// The flags that will be added to any created archive files. + /// + public ImmutableList Flags { get; private set; } - /// - /// The encoding method that will be used to write files. - /// - public EncodingDefinition EncodingMethod + /// + /// The encoding method that will be used to write files. + /// + public EncodingDefinition EncodingMethod + { + get => m_encodingMethod; + set { - get => m_encodingMethod; - set - { - TestForEditable(); - m_encodingMethod = value ?? throw new ArgumentNullException(nameof(value)); - } + TestForEditable(); + m_encodingMethod = value ?? throw new ArgumentNullException(nameof(value)); } + } - /// - /// The desired number of bytes to leave on the disk after a rollover has completed. - /// Otherwise, pick a different directory or throw an out of disk space exception. - /// - /// - /// Value must be between 100MB and 1TB - /// - public long DesiredRemainingSpace + /// + /// The desired number of bytes to leave on the disk after a rollover has completed. + /// Otherwise, pick a different directory or throw an out of disk space exception. + /// + /// + /// Value must be between 100MB and 1TB + /// + public long DesiredRemainingSpace + { + get => m_desiredRemainingSpace; + set { - get => m_desiredRemainingSpace; - set + TestForEditable(); + + m_desiredRemainingSpace = value switch { - TestForEditable(); - - if (value < 100 * 1024L * 1024L) - m_desiredRemainingSpace = 100 * 1024L * 1024L; - else if (value > 1024 * 1024L * 1024L * 1024L) - m_desiredRemainingSpace = 1024 * 1024L * 1024L * 1024L; - else - m_desiredRemainingSpace = value; - } + < 100 * 1024L * 1024L => 100 * 1024L * 1024L, + > 1024 * 1024L * 1024L * 1024L => 1024 * 1024L * 1024L * 1024L, + _ => value + }; } + } + /// + /// Creates a that will reside on the disk. + /// + /// the paths to place the files. + /// The desired free space to leave on the disk before moving to another disk. + /// the method for storing files in a directory. + /// the encoding method to use for the archive file. + /// the prefix to affix to the files created. + /// the extension file name + /// the final extension to specify + /// flags to include in the archive that is created. + /// + public void ConfigureOnDisk(IEnumerable paths, long desiredRemainingSpace, ArchiveDirectoryMethod directoryMethod, EncodingDefinition encodingMethod, string prefix, string pendingExtension, string finalExtension, params Guid[] flags) + { + TestForEditable(); + Initialize(); - /// - /// Creates a that will reside on the disk. - /// - /// the paths to place the files. - /// The desired free space to leave on the disk before moving to another disk. - /// the method for storing files in a directory. - /// the encoding method to use for the archive file. - /// the prefix to affix to the files created. - /// the extension file name - /// the final extension to specify - /// flags to include in the archive that is created. - /// - public void ConfigureOnDisk(IEnumerable paths, long desiredRemainingSpace, ArchiveDirectoryMethod directoryMethod, EncodingDefinition encodingMethod, string prefix, string pendingExtension, string finalExtension, params Guid[] flags) - { - TestForEditable(); - Initialize(); - DirectoryMethod = directoryMethod; - PendingExtension = pendingExtension; - FinalExtension = finalExtension; - Flags.AddRange(flags); - Prefix = prefix; - WritePath.AddRange(paths); - DesiredRemainingSpace = desiredRemainingSpace; - EncodingMethod = encodingMethod; - } + DirectoryMethod = directoryMethod; + PendingExtension = pendingExtension; + FinalExtension = finalExtension; + Flags.AddRange(flags); + Prefix = prefix; + WritePath.AddRange(paths); + DesiredRemainingSpace = desiredRemainingSpace; + EncodingMethod = encodingMethod; + } - public override void Save(Stream stream) - { - stream.Write((byte)2); - stream.Write((int)m_directoryMethod); - stream.Write((int)m_fillMethod); - stream.Write(m_prefix); - stream.Write(m_pendingExtension); - stream.Write(m_finalExtension); - stream.Write(m_desiredRemainingSpace); - m_encodingMethod.Save(stream); - stream.Write(m_writePath.Count); - foreach (string path in m_writePath) - { - stream.Write(path); - } - stream.Write(m_flags.Count); - foreach (Guid flag in m_flags) - { - stream.Write(flag); - } - } + public override void Save(Stream stream) + { + stream.Write((byte)3); + stream.Write((int)m_directoryMethod); + stream.Write((int)m_fillMethod); + stream.Write(m_prefix); + stream.Write(m_pendingExtension); + stream.Write(m_finalExtension); + stream.Write(m_desiredRemainingSpace); + + m_encodingMethod.Save(stream); + stream.Write(WritePath.Count); + + foreach (string path in WritePath) + stream.Write(path); + + stream.Write(Flags.Count); + + foreach (Guid flag in Flags) + stream.Write(flag); + + stream.Write(m_bloomFilter.Enabled); + stream.Write(m_bloomFilter.BucketDurationSeconds); + stream.Write(m_bloomFilter.FalsePositiveProbability); + stream.Write(m_enablePointIDIndex); + stream.Write(m_bloomFilter.MaximumIndexSizeMiB); + } - public override void Load(Stream stream) + public override void Load(Stream stream) + { + TestForEditable(); + + byte version = stream.ReadNextByte(); + + switch (version) { - TestForEditable(); - byte version = stream.ReadNextByte(); - switch (version) - { - case 1: - case 2: - m_directoryMethod = (ArchiveDirectoryMethod)stream.ReadInt32(); - - // Fill method was introduced in version 2; version 1 streams default to Sequential - m_fillMethod = version >= 2 ? (ArchiveDirectoryFillMethod)stream.ReadInt32() : ArchiveDirectoryFillMethod.Sequential; - - m_prefix = stream.ReadString(); - m_pendingExtension = stream.ReadString(); - m_finalExtension = stream.ReadString(); - m_desiredRemainingSpace = stream.ReadInt64(); - m_encodingMethod = new EncodingDefinition(stream); + case 1: + case 2: + case 3: + m_directoryMethod = (ArchiveDirectoryMethod)stream.ReadInt32(); + + // Fill method was introduced in version 2; version 1 streams default to Sequential + m_fillMethod = version >= 2 ? (ArchiveDirectoryFillMethod)stream.ReadInt32() : ArchiveDirectoryFillMethod.Sequential; + + m_prefix = stream.ReadString(); + m_pendingExtension = stream.ReadString(); + m_finalExtension = stream.ReadString(); + m_desiredRemainingSpace = stream.ReadInt64(); + m_encodingMethod = new EncodingDefinition(stream); - int cnt = stream.ReadInt32(); - m_writePath.Clear(); + int cnt = stream.ReadInt32(); + WritePath.Clear(); - while (cnt > 0) - { - cnt--; - m_writePath.Add(stream.ReadString()); - } + while (cnt > 0) + { + cnt--; + WritePath.Add(stream.ReadString()); + } - cnt = stream.ReadInt32(); - m_flags.Clear(); + cnt = stream.ReadInt32(); + Flags.Clear(); - while (cnt > 0) - { - cnt--; - m_flags.Add(stream.ReadGuid()); - } - break; - default: - throw new VersionNotFoundException("Unknown Version Code: " + version); + while (cnt > 0) + { + cnt--; + Flags.Add(stream.ReadGuid()); + } - } - } + m_enablePointIDIndex = true; + m_bloomFilter = new TimeBucketBloomFilterSettings(); + + if (version >= 3) + { + bool enabled = stream.ReadBoolean(); + int seconds = stream.ReadInt32(); + double probability = stream.ReadDouble(); + + m_enablePointIDIndex = stream.ReadBoolean(); + int maximumMiB = stream.ReadInt32(); + + m_bloomFilter = new TimeBucketBloomFilterSettings(enabled, seconds, probability, maximumMiB); + } + break; + default: + throw new VersionNotFoundException("Unknown Version Code: " + version); - public override void Validate() - { - if (WritePath.Count == 0) - throw new Exception("Missing write paths."); } } + + public override void Validate() + { + if (WritePath.Count == 0) + throw new Exception("Missing write paths."); + } } \ No newline at end of file diff --git a/Source/Libraries/GSF.SortedTreeStore/Snap/Storage/ArchivePointIDIndex.cs b/Source/Libraries/GSF.SortedTreeStore/Snap/Storage/ArchivePointIDIndex.cs new file mode 100644 index 0000000000..562b4b4bc1 --- /dev/null +++ b/Source/Libraries/GSF.SortedTreeStore/Snap/Storage/ArchivePointIDIndex.cs @@ -0,0 +1,287 @@ +//****************************************************************************************************** +// ArchivePointIDIndex.cs - Gbtc +// +// Copyright © 2026, Grid Protection Alliance. All Rights Reserved. +// +// Licensed to the Grid Protection Alliance (GPA) under one or more contributor license agreements. See +// the NOTICE file distributed with this work for additional information regarding copyright ownership. +// The GPA licenses this file to you under the MIT License (MIT), the "License"; you may not use this +// file except in compliance with the License. You may obtain a copy of the License at: +// +// http://opensource.org/licenses/MIT +// +// Unless agreed to in writing, the subject software distributed under the License is distributed on an +// "AS-IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. Refer to the +// License for the specific language governing permissions and limitations. +// +// Code Modification History: +// ---------------------------------------------------------------------------------------------------- +// 09/21/2026 - J. Ritchie Carroll +// Generated original version of source code. +// +//****************************************************************************************************** + +using System; +using System.Collections.Generic; +using System.IO; +using System.IO.Compression; +using System.Linq; +using System.Security.Cryptography; +using GSF.Diagnostics; +using GSF.Snap.Types; + +namespace GSF.Snap.Storage; + +// Optional, exact presence synopsis for a finalized default timestamp/point-ID table. +// Missing, incomplete, stale or corrupt indexes must never become absence evidence. +internal static class ArchivePointIDIndex +{ + internal const string Extension = ".pidx"; + internal const int MaximumIDs = 4000000; + private const int FileFormatSignature = 0x31444950; // ASCII "PID1" (point ID, version 1), little endian + + internal static ulong[] Load(string path, Guid archive, uint sequence, Guid keyType, Guid valueType) + { + if (string.IsNullOrEmpty(path)) + return null; + + try + { + using FileStream file = File.OpenRead(path + Extension); + + // Bound both compressed input and decompressed output before allocating. + if (file.Length is < 32 or > MaximumIDs * 8L + 4096) + return null; + + byte[] expected = new byte[32]; + + if (file.Read(expected, 0, expected.Length) != expected.Length) + return null; + + byte[] payload; + + using (MemoryStream buffer = new()) + using (DeflateStream compressed = new(file, CompressionMode.Decompress)) + { + byte[] chunk = new byte[8192]; + int read; + + while ((read = compressed.Read(chunk, 0, chunk.Length)) != 0) + { + if (buffer.Length + read > MaximumIDs * 8L + 68) + return null; + + buffer.Write(chunk, 0, read); + } + + payload = buffer.ToArray(); + } + + using (SHA256 hash = SHA256.Create()) + { + if (!hash.ComputeHash(payload).SequenceEqual(expected)) + return null; + } + + using (BinaryReader reader = new(new MemoryStream(payload))) + { + if (reader.ReadInt32() != FileFormatSignature || new Guid(reader.ReadBytes(16)) != archive || reader.ReadUInt32() != sequence || new Guid(reader.ReadBytes(16)) != keyType || new Guid(reader.ReadBytes(16)) != valueType || reader.ReadInt64() != new FileInfo(path).Length) + return null; + + int count = reader.ReadInt32(); + + if (count < 0 || count > MaximumIDs || payload.Length != 68L + count * 8L) + return null; + + ulong[] ids = new ulong[count]; + + for (int i = 0; i < count; i++) + { + ids[i] = reader.ReadUInt64(); + + if (i > 0 && ids[i] <= ids[i - 1]) + return null; + } + + return ids; + } + } + catch (InvalidDataException) + { + return null; + } + catch (IOException) + { + return null; + } + catch (UnauthorizedAccessException) + { + return null; + } + catch (ArgumentException) + { + return null; + } + } + + internal static void Save(string path, Guid archive, uint sequence, Guid keyType, Guid valueType, IEnumerable ids) + { + string temporary = $"{path}{Extension}.{Guid.NewGuid():N}.tmp"; + + try + { + ulong[] ordered = [.. ids.OrderBy(id => id)]; + byte[] payload; + + using (MemoryStream buffer = new()) + { + using (BinaryWriter writer = new(buffer, System.Text.Encoding.UTF8, true)) + { + writer.Write(FileFormatSignature); + writer.Write(archive.ToByteArray()); + writer.Write(sequence); + writer.Write(keyType.ToByteArray()); + writer.Write(valueType.ToByteArray()); + writer.Write(new FileInfo(path).Length); + writer.Write(ordered.Length); + + foreach (ulong id in ordered) + writer.Write(id); + } + + payload = buffer.ToArray(); + } + + using (FileStream file = new(temporary, FileMode.CreateNew, FileAccess.Write, FileShare.None)) + { + using (SHA256 hash = SHA256.Create()) + { + byte[] checksum = hash.ComputeHash(payload); + file.Write(checksum, 0, checksum.Length); + } + + using (DeflateStream compressed = new(file, CompressionLevel.Optimal, true)) + compressed.Write(payload, 0, payload.Length); + + file.Flush(true); + } + + // Publish only after the archive and complete synopsis are durable. An existing + // sidecar is left alone: its identity will be checked by every new snapshot. + File.Move(temporary, path + Extension); + } + catch (IOException ex) + { + Logger.SwallowException(ex); + } + catch (UnauthorizedAccessException ex) + { + Logger.SwallowException(ex); + } + finally + { + Delete(temporary, false); + } + } + + internal static void Delete(string path, bool appendExtension = true) + { + if (string.IsNullOrEmpty(path)) + return; + + try + { + File.Delete(path + (appendExtension ? Extension : "")); + } + catch (IOException ex) + { + Logger.SwallowException(ex); + } + catch (UnauthorizedAccessException ex) + { + Logger.SwallowException(ex); + } + } + + internal static void Move(string oldPath, string newPath) + { + if (string.IsNullOrEmpty(oldPath) || oldPath == newPath) + return; + + try + { + File.Move(oldPath + Extension, newPath + Extension); + } + catch (IOException ex) + { + Logger.SwallowException(ex); + } + catch (UnauthorizedAccessException ex) + { + Logger.SwallowException(ex); + } + } +} + +// Does not own the input stream. A disabled/oversized collector leaves no index. +internal sealed class PointIDCollectingStream : TreeStream + where TKey : SnapTypeBase, new() + where TValue : SnapTypeBase, new() +{ + private readonly TreeStream m_source; + private HashSet m_ids; + private bool m_complete; + private readonly ArchiveTimeBucketBloomIndex.Builder m_bloom; + + internal PointIDCollectingStream(TreeStream source, TimeBucketBloomFilterSettings bloom = null, bool enablePointIDIndex = true) + { + m_source = source ?? throw new ArgumentNullException(nameof(source)); + + if (new TKey() is not TimestampPointIDBase) + return; + + if (enablePointIDIndex) + m_ids = []; + + if (bloom is { Enabled: true }) + m_bloom = new ArchiveTimeBucketBloomIndex.Builder(bloom); + } + + public override bool IsAlwaysSequential => m_source.IsAlwaysSequential; + + public override bool NeverContainsDuplicates => m_source.NeverContainsDuplicates; + + protected override bool ReadNext(TKey key, TValue value) + { + if (!m_source.Read(key, value)) + { + m_complete = true; + return false; + } + + if (m_ids is not null) + { + m_ids.Add(((TimestampPointIDBase)(object)key).PointID); + + if (m_ids.Count > ArchivePointIDIndex.MaximumIDs) + m_ids = null; + } + + if (m_bloom is null) + return true; + + TimestampPointIDBase timeKey = (TimestampPointIDBase)(object)key; + m_bloom.Add(timeKey.Timestamp, timeKey.PointID); + + return true; + } + + internal void Publish(string path, Guid archive, uint sequence) + { + if (m_complete && m_ids is not null) + ArchivePointIDIndex.Save(path, archive, sequence, new TKey().GenericTypeGuid, new TValue().GenericTypeGuid, m_ids); + + if (m_complete && m_bloom is not null) + m_bloom.Complete()?.Save(path, archive, sequence, new TKey().GenericTypeGuid, new TValue().GenericTypeGuid); + } +} \ No newline at end of file diff --git a/Source/Libraries/GSF.SortedTreeStore/Snap/Storage/ArchiveTimeBucketBloomIndex.cs b/Source/Libraries/GSF.SortedTreeStore/Snap/Storage/ArchiveTimeBucketBloomIndex.cs new file mode 100644 index 0000000000..2c7569b60d --- /dev/null +++ b/Source/Libraries/GSF.SortedTreeStore/Snap/Storage/ArchiveTimeBucketBloomIndex.cs @@ -0,0 +1,392 @@ +//****************************************************************************************************** +// ArchiveTimeBucketBloomIndex.cs - Gbtc +// +// Copyright © 2026, Grid Protection Alliance. All Rights Reserved. +// +// Licensed to the Grid Protection Alliance (GPA) under one or more contributor license agreements. See +// the NOTICE file distributed with this work for additional information regarding copyright ownership. +// The GPA licenses this file to you under the MIT License (MIT), the "License"; you may not use this +// file except in compliance with the License. You may obtain a copy of the License at: +// +// http://opensource.org/licenses/MIT +// +// Unless agreed to in writing, the subject software distributed under the License is distributed on an +// "AS-IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. Refer to the +// License for the specific language governing permissions and limitations. +// +// Code Modification History: +// ---------------------------------------------------------------------------------------------------- +// 09/21/2026 - J. Ritchie Carroll +// Generated original version of source code. +// +//****************************************************************************************************** + +using System; +using System.Collections.Generic; +using System.IO; +using System.IO.Compression; +using System.Linq; +using System.Security.Cryptography; +using GSF.Diagnostics; + +namespace GSF.Snap.Storage; + +// Versioned independently of .pidx. Omitted buckets are empty only in a complete, +// identity-checked index. A disabled or capped builder never publishes a partial index. +internal sealed class ArchiveTimeBucketBloomIndex +{ + internal const string Extension = ".bidx"; + internal const int MaximumBytes = 64 * 1024 * 1024; + internal const int MaximumBuckets = 100000; + private const int FileFormatSignature = 0x314D4C42; // ASCII "BLM1" (bloom, version 1), little endian + private readonly ulong m_width; + private readonly Bucket[] m_buckets; + + internal sealed class Bucket + { + internal ulong Ordinal; + internal int Probes; + internal byte[] Bits; + } + + private ArchiveTimeBucketBloomIndex(ulong width, Bucket[] buckets) + { + m_width = width; + m_buckets = buckets; + } + + internal bool MayContain(ulong id, ulong lower, ulong upper) + { + if (lower > upper) + return false; + + ulong first = lower / m_width, last = upper / m_width; + int lo = 0, hi = m_buckets.Length; + + while (lo < hi) + { + int mid = lo + (hi - lo) / 2; + + if (m_buckets[mid].Ordinal < first) + lo = mid + 1; + else + hi = mid; + } + + for (int i = lo; i < m_buckets.Length && m_buckets[i].Ordinal <= last; i++) + { + if (Contains(m_buckets[i], id)) + return true; + } + + return false; + } + + // SplitMix64 finalizer, two independent seeds, double hashing. All arithmetic is + // explicitly unchecked so serialized indexes behave identically on both runtimes. + private static ulong Mix(ulong value) + { + unchecked + { + value = (value ^ (value >> 30)) * 0xbf58476d1ce4e5b9UL; + value = (value ^ (value >> 27)) * 0x94d049bb133111ebUL; + return value ^ (value >> 31); + } + } + + private static bool Contains(Bucket bucket, ulong id) + { + ulong a = Mix(id ^ 0x9e3779b97f4a7c15UL), b = Mix(id ^ 0xd1b54a32d192ed03UL) | 1; + ulong mask = (ulong)bucket.Bits.Length * 8 - 1; + + for (int k = 0; k < bucket.Probes; k++) + { + ulong bit = unchecked(a + (ulong)k * b) & mask; + + if ((bucket.Bits[(int)(bit >> 3)] & (1 << (int)(bit & 7))) == 0) + return false; + } + + return true; + } + + internal sealed class Builder + { + private readonly TimeBucketBloomFilterSettings m_settings; + private readonly List m_buckets = []; + private readonly HashSet m_ids = []; + private ulong m_ordinal; + private int m_bytes = 80; + private bool m_disabled; + + internal Builder(TimeBucketBloomFilterSettings settings) + { + m_settings = settings; + } + + internal void Add(ulong timestamp, ulong id) + { + if (m_disabled) + return; + + ulong ordinal = timestamp / ((ulong)m_settings.BucketDurationSeconds * (ulong)TimeSpan.TicksPerSecond); + + if (m_ids.Count != 0 && ordinal != m_ordinal) + { + if (ordinal < m_ordinal) + { + Disable(); + return; + } + + Flush(); + + if (m_disabled) + return; + } + + m_ordinal = ordinal; + m_ids.Add(id); + + if (m_ids.Count > ArchivePointIDIndex.MaximumIDs) + Disable(); + } + + private void Disable() + { + m_disabled = true; + m_ids.Clear(); + m_buckets.Clear(); + } + private void Flush() + { + if (m_ids.Count == 0 || m_disabled) + return; + + double desired = -m_ids.Count * Math.Log(m_settings.FalsePositiveProbability) / (Math.Log(2) * Math.Log(2)); + int bytes = 8; + + while (bytes * 8.0D < desired && bytes < MaximumBytes) + bytes *= 2; + + if (bytes > m_settings.MaximumIndexSizeMiB * 1024 * 1024 - m_bytes - 16 || m_buckets.Count == MaximumBuckets) + { + Disable(); + return; + } + + Bucket bucket = new() + { + Ordinal = m_ordinal, + Bits = new byte[bytes], + Probes = Math.Max(1, Math.Min(16, (int)Math.Round(bytes * 8.0 / m_ids.Count * Math.Log(2)))) + }; + + ulong mask = (ulong)bytes * 8 - 1; + + foreach (ulong id in m_ids) + { + ulong a = Mix(id ^ 0x9e3779b97f4a7c15UL), b = Mix(id ^ 0xd1b54a32d192ed03UL) | 1; + + for (int k = 0; k < bucket.Probes; k++) + { + ulong bit = unchecked(a + (ulong)k * b) & mask; + bucket.Bits[(int)(bit >> 3)] |= (byte)(1 << (int)(bit & 7)); + } + } + + m_buckets.Add(bucket); + m_bytes += bytes + 16; + m_ids.Clear(); + } + + internal ArchiveTimeBucketBloomIndex Complete() + { + Flush(); + return m_disabled ? null : new ArchiveTimeBucketBloomIndex((ulong)m_settings.BucketDurationSeconds * (ulong)TimeSpan.TicksPerSecond, [.. m_buckets]); + } + } + + internal static ArchiveTimeBucketBloomIndex Load(string path, Guid archive, uint sequence, Guid keyType, Guid valueType) + { + if (string.IsNullOrEmpty(path)) + return null; + + try + { + using FileStream file = File.OpenRead(path + Extension); + + if (file.Length is < 32 or > MaximumBytes + 65536L) + return null; + + byte[] expected = new byte[32]; + + if (file.Read(expected, 0, 32) != 32) + return null; + + byte[] payload; + + using (MemoryStream buffer = new()) + using (DeflateStream compressed = new(file, CompressionMode.Decompress)) + { + byte[] chunk = new byte[8192]; + int read; + + while ((read = compressed.Read(chunk, 0, chunk.Length)) != 0) + { + if (buffer.Length + read > MaximumBytes) + return null; + + buffer.Write(chunk, 0, read); + } + + payload = buffer.ToArray(); + } + + using (SHA256 hash = SHA256.Create()) + { + if (!hash.ComputeHash(payload).SequenceEqual(expected)) + return null; + } + + using (BinaryReader reader = new(new MemoryStream(payload))) + { + if (reader.ReadInt32() != FileFormatSignature || new Guid(reader.ReadBytes(16)) != archive || reader.ReadUInt32() != sequence || new Guid(reader.ReadBytes(16)) != keyType || new Guid(reader.ReadBytes(16)) != valueType || reader.ReadInt64() != new FileInfo(path).Length) + return null; + + ulong width = reader.ReadUInt64(); + int count = reader.ReadInt32(); + + if (width == 0 || count < 0 || count > MaximumBuckets || count > (payload.Length - 76) / 24) + return null; + + Bucket[] buckets = new Bucket[count]; + + for (int i = 0; i < count; i++) + { + ulong ordinal = reader.ReadUInt64(); + int probes = reader.ReadInt32(), bytes = reader.ReadInt32(); + + if (probes < 1 || probes > 16 || bytes < 8 || (bytes & (bytes - 1)) != 0 || bytes > reader.BaseStream.Length - reader.BaseStream.Position || ordinal > ulong.MaxValue / width || (i > 0 && ordinal <= buckets[i - 1].Ordinal)) + return null; + + buckets[i] = new Bucket + { + Ordinal = ordinal, + Probes = probes, + Bits = reader.ReadBytes(bytes) + }; + } + + return reader.BaseStream.Position != reader.BaseStream.Length ? null : new ArchiveTimeBucketBloomIndex(width, buckets); + } + } + catch (InvalidDataException) + { + + return null; + } + catch (IOException) + { + return null; + } + catch (UnauthorizedAccessException) + { + return null; + } + catch (ArgumentException) + { + return null; + } + } + + internal void Save(string path, Guid archive, uint sequence, Guid keyType, Guid valueType) + { + string temporary = $"{path}{Extension}.{Guid.NewGuid():N}.tmp"; + + try + { + byte[] payload; + + using (MemoryStream buffer = new()) + { + using (BinaryWriter writer = new(buffer, System.Text.Encoding.UTF8, true)) + { + writer.Write(FileFormatSignature); + writer.Write(archive.ToByteArray()); + writer.Write(sequence); + writer.Write(keyType.ToByteArray()); + writer.Write(valueType.ToByteArray()); + writer.Write(new FileInfo(path).Length); + writer.Write(m_width); + writer.Write(m_buckets.Length); + + foreach (Bucket bucket in m_buckets) + { + writer.Write(bucket.Ordinal); + writer.Write(bucket.Probes); + writer.Write(bucket.Bits.Length); + writer.Write(bucket.Bits); + } + } + + payload = buffer.ToArray(); + } + + using (FileStream file = new(temporary, FileMode.CreateNew, FileAccess.Write, FileShare.None)) + { + using (SHA256 hash = SHA256.Create()) + { + byte[] checksum = hash.ComputeHash(payload); + file.Write(checksum, 0, checksum.Length); + } + + using (DeflateStream compressed = new(file, CompressionLevel.Optimal, true)) + { + compressed.Write(payload, 0, payload.Length); + } + + file.Flush(true); + } + + File.Move(temporary, path + Extension); + } + catch (IOException ex) + { + Logger.SwallowException(ex); + } + catch (UnauthorizedAccessException ex) + { + Logger.SwallowException(ex); + } + finally + { + ArchivePointIDIndex.Delete(temporary, false); + } + } + + internal static void Delete(string path) + { + if (!string.IsNullOrEmpty(path)) + ArchivePointIDIndex.Delete(path + Extension, false); + } + + internal static void Move(string oldPath, string newPath) + { + if (string.IsNullOrEmpty(oldPath) || oldPath == newPath) + return; + + try + { + File.Move(oldPath + Extension, newPath + Extension); + } + catch (IOException ex) + { + Logger.SwallowException(ex); + } + catch (UnauthorizedAccessException ex) + { + Logger.SwallowException(ex); + } + } +} \ No newline at end of file diff --git a/Source/Libraries/GSF.SortedTreeStore/Snap/Storage/SortedTreeFile.cs b/Source/Libraries/GSF.SortedTreeStore/Snap/Storage/SortedTreeFile.cs index aa87931341..5cad115539 100755 --- a/Source/Libraries/GSF.SortedTreeStore/Snap/Storage/SortedTreeFile.cs +++ b/Source/Libraries/GSF.SortedTreeStore/Snap/Storage/SortedTreeFile.cs @@ -28,355 +28,355 @@ using GSF.Snap.Tree; using GSF.IO.FileStructure; -namespace GSF.Snap.Storage +namespace GSF.Snap.Storage; + +/// +/// Represents an individual self-contained archive file. +/// +public class SortedTreeFile + : IDisposable { + // {63AB3FEA-14CD-4ECA-939B-0DD23742E170} /// - /// Represents a individual self-contained archive file. + /// The main type of the archive file. /// - /// - /// - public class SortedTreeFile - : IDisposable - { - // {63AB3FEA-14CD-4ECA-939B-0DD23742E170} - /// - /// The main type of the archive file. - /// - internal static readonly Guid FileType = new Guid(0x63ab3fea, 0x14cd, 0x4eca, 0x93, 0x9b, 0x0d, 0xd2, 0x37, 0x42, 0xe1, 0x70); + internal static readonly Guid FileType = new(0x63ab3fea, 0x14cd, 0x4eca, 0x93, 0x9b, 0x0d, 0xd2, 0x37, 0x42, 0xe1, 0x70); - // {E0FCA590-F46E-4060-8764-DFDCFC74D728} - /// - /// The guid where the primary archive component exists - /// - internal static readonly Guid PrimaryArchiveType = new Guid(0xe0fca590, 0xf46e, 0x4060, 0x87, 0x64, 0xdf, 0xdc, 0xfc, 0x74, 0xd7, 0x28); + // {E0FCA590-F46E-4060-8764-DFDCFC74D728} + /// + /// The guid where the primary archive component exists + /// + internal static readonly Guid PrimaryArchiveType = new(0xe0fca590, 0xf46e, 0x4060, 0x87, 0x64, 0xdf, 0xdc, 0xfc, 0x74, 0xd7, 0x28); - #region [ Members ] + #region [ Members ] - private string m_filePath; - private bool m_disposed; + private TransactionalFileStructure m_fileStructure; + private readonly SortedList m_openedFiles; - private TransactionalFileStructure m_fileStructure; - private readonly SortedList m_openedFiles; + #endregion - #endregion + #region [ Constructors ] - #region [ Constructors ] + private SortedTreeFile() + { + m_openedFiles = new SortedList(); + } - private SortedTreeFile() - { - m_openedFiles = new SortedList(); - } + /// + /// Creates a new in memory archive file. + /// + /// The number of bytes per block in the file. + /// Flags to write to the file + public static SortedTreeFile CreateInMemory(int blockSize = 4096, params Guid[] flags) + { + SortedTreeFile file = new(); - /// - /// Creates a new in memory archive file. - /// - /// The number of bytes per block in the file. - /// Flags to write to the file - public static SortedTreeFile CreateInMemory(int blockSize = 4096, params Guid[] flags) - { - SortedTreeFile file = new SortedTreeFile(); - file.m_filePath = string.Empty; - file.m_fileStructure = TransactionalFileStructure.CreateInMemory(blockSize, flags); - return file; - } + file.FilePath = string.Empty; + file.m_fileStructure = TransactionalFileStructure.CreateInMemory(blockSize, flags); + + return file; + } - /// - /// Creates an archive file. - /// - /// the path for the file. - /// The number of bytes per block in the file. - /// Flags to write to the file - public static SortedTreeFile CreateFile(string file, int blockSize = 4096, params Guid[] flags) - { - SortedTreeFile af = new SortedTreeFile(); - file = Path.GetFullPath(file); - af.m_filePath = file; - af.m_fileStructure = TransactionalFileStructure.CreateFile(file, blockSize, flags); - return af; - } + /// + /// Creates an archive file. + /// + /// the path for the file. + /// The number of bytes per block in the file. + /// Flags to write to the file + public static SortedTreeFile CreateFile(string file, int blockSize = 4096, params Guid[] flags) + { + SortedTreeFile af = new(); + file = Path.GetFullPath(file); - /// - /// Opens an archive file. - /// - /// - /// - /// - public static SortedTreeFile OpenFile(string file, bool isReadOnly) - { - SortedTreeFile af = new SortedTreeFile(); - file = Path.GetFullPath(file); - af.m_filePath = file; - af.m_fileStructure = TransactionalFileStructure.OpenFile(file, isReadOnly); - if (af.m_fileStructure.Snapshot.Header.ArchiveType != FileType) - throw new Exception("Archive type is unknown"); - return af; - } + af.FilePath = file; + af.m_fileStructure = TransactionalFileStructure.CreateFile(file, blockSize, flags); + + return af; + } - #endregion + /// + /// Opens an archive file. + /// + /// + /// + /// + public static SortedTreeFile OpenFile(string file, bool isReadOnly) + { + SortedTreeFile af = new(); + file = Path.GetFullPath(file); + + af.FilePath = file; + af.m_fileStructure = TransactionalFileStructure.OpenFile(file, isReadOnly); + + return af.m_fileStructure.Snapshot.Header.ArchiveType == FileType ? af : throw new Exception("Archive type is unknown"); + } - #region [ Properties ] + #endregion - /// - /// Determines if the archive file has been disposed. - /// - public bool IsDisposed => m_disposed; + #region [ Properties ] - /// - /// Returns the name of the file. Returns if this is a memory archive. - /// This is the name of the entire path. - /// - public string FilePath => m_filePath; + /// + /// Determines if the archive file has been disposed. + /// + public bool IsDisposed { get; private set; } - /// - /// Gets if the file is a memory file. - /// - public bool IsMemoryFile => m_filePath == string.Empty; + /// + /// Returns the name of the file. Returns if this is a memory archive. + /// This is the name of the entire path. + /// + public string FilePath { get; private set; } - /// - /// Gets the name of the file, but only the file, not the entire path. - /// - public string FileName - { - get - { - if (m_filePath == string.Empty) - return string.Empty; - return Path.GetFileName(m_filePath); - } - } + /// + /// Gets if the file is a memory file. + /// + public bool IsMemoryFile => FilePath == string.Empty; - /// - /// Gets the last committed read snapshot on the file system. - /// - /// - public ReadSnapshot Snapshot => m_fileStructure.Snapshot; + /// + /// Gets the name of the file, but only the file, not the entire path. + /// + public string FileName => FilePath == string.Empty ? string.Empty : Path.GetFileName(FilePath); - /// - /// Gets the size of the file. - /// - public long ArchiveSize => m_fileStructure.ArchiveSize; + /// + /// Gets the last committed read snapshot on the file system. + /// + /// + public ReadSnapshot Snapshot => m_fileStructure.Snapshot; - #endregion + /// + /// Gets the size of the file. + /// + public long ArchiveSize => m_fileStructure.ArchiveSize; - #region [ Methods ] + #endregion + #region [ Methods ] - /// - /// Changes the extension of the current file. - /// - /// the new extension - /// If the file should be reopened as readonly - /// If the file should share read privileges. - public void ChangeExtension(string extension, bool isReadOnly, bool isSharingEnabled) - { - m_fileStructure.ChangeExtension(extension, isReadOnly, isSharingEnabled); - m_filePath = m_fileStructure.FileName; - } + /// + /// Changes the extension of the current file. + /// + /// the new extension + /// If the file should be reopened as readonly + /// If the file should share read privileges. + public void ChangeExtension(string extension, bool isReadOnly, bool isSharingEnabled) + { + string oldPath = FilePath; + + m_fileStructure.ChangeExtension(extension, isReadOnly, isSharingEnabled); + FilePath = m_fileStructure.FileName; + + ArchivePointIDIndex.Move(oldPath, FilePath); + ArchiveTimeBucketBloomIndex.Move(oldPath, FilePath); + } - /// - /// Reopens the file with different permissions. - /// - /// If the file should be reopened as readonly - /// If the file should share read privileges. - public void ChangeShareMode(bool isReadOnly, bool isSharingEnabled) - { - m_fileStructure.ChangeShareMode(isReadOnly, isSharingEnabled); - } + /// + /// Reopens the file with different permissions. + /// + /// If the file should be reopened as readonly + /// If the file should share read privileges. + public void ChangeShareMode(bool isReadOnly, bool isSharingEnabled) + { + m_fileStructure.ChangeShareMode(isReadOnly, isSharingEnabled); + } - /// - /// Opens the default table for this TKey and TValue. - /// - /// The key - /// The value - /// - /// Every Key and Value have their uniquely mapped file, therefore a different file is opened if TKey and TValue are different. - /// - /// null if table does not exist - public SortedTreeTable OpenTable() - where TKey : SnapTypeBase, new() - where TValue : SnapTypeBase, new() - { - return OpenTable(GetFileName()); - } + /// + /// Opens the default table for this TKey and TValue. + /// + /// The key + /// The value + /// + /// Every Key and Value have their uniquely mapped file, therefore a different file is opened if TKey and TValue are different. + /// + /// null if table does not exist + public SortedTreeTable OpenTable() + where TKey : SnapTypeBase, new() + where TValue : SnapTypeBase, new() + { + return OpenTable(GetFileName()); + } - /// - /// Opens the default table for this TKey and TValue. - /// - /// The key - /// The value - /// the name of an internal table - /// - /// Every Key and Value have their uniquely mapped file, therefore a different file is opened if TKey and TValue are different. - /// - /// null if table does not exist - public SortedTreeTable OpenTable(string tableName) - where TKey : SnapTypeBase, new() - where TValue : SnapTypeBase, new() - { - return OpenTable(GetFileName(tableName)); - } + /// + /// Opens the default table for this TKey and TValue. + /// + /// The key + /// The value + /// the name of an internal table + /// + /// Every Key and Value have their uniquely mapped file, therefore a different file is opened if TKey and TValue are different. + /// + /// null if table does not exist + public SortedTreeTable OpenTable(string tableName) + where TKey : SnapTypeBase, new() + where TValue : SnapTypeBase, new() + { + return OpenTable(GetFileName(tableName)); + } - /// - /// Opens the table for the provided file name. - /// - /// - /// - /// the filename to open - /// null if table does not exist - private SortedTreeTable OpenTable(SubFileName fileName) - where TKey : SnapTypeBase, new() - where TValue : SnapTypeBase, new() + /// + /// Opens the table for the provided file name. + /// + /// + /// + /// the filename to open + /// null if table does not exist + private SortedTreeTable OpenTable(SubFileName fileName) + where TKey : SnapTypeBase, new() + where TValue : SnapTypeBase, new() + { + if (!m_openedFiles.ContainsKey(fileName)) { - if (!m_openedFiles.ContainsKey(fileName)) - { - if (!m_fileStructure.Snapshot.Header.ContainsSubFile(fileName)) - return null; - m_openedFiles.Add(fileName, new SortedTreeTable(m_fileStructure, fileName, this)); - } - return (SortedTreeTable)m_openedFiles[fileName]; + if (!m_fileStructure.Snapshot.Header.ContainsSubFile(fileName)) + return null; + + m_openedFiles.Add(fileName, new SortedTreeTable(m_fileStructure, fileName, this)); } + + return (SortedTreeTable)m_openedFiles[fileName]; + } - /// - /// Opens the default table for this TKey and TValue. If it does not exists, - /// it will be created with the provided compression method. - /// - /// - /// - /// The method of compression to utilize in this table. - /// the maximum desired block size for a SortedTree. Must be at least 1024. - /// the name of an internal table - /// - public SortedTreeTable OpenOrCreateTable(EncodingDefinition storageMethod, string tableName, int maxSortedTreeBlockSize = 4096) - where TKey : SnapTypeBase, new() - where TValue : SnapTypeBase, new() - { - if (storageMethod is null) - throw new ArgumentNullException("storageMethod"); + /// + /// Opens the default table for this TKey and TValue. If it does not exists, + /// it will be created with the provided compression method. + /// + /// + /// + /// The method of compression to utilize in this table. + /// the maximum desired block size for a SortedTree. Must be at least 1024. + /// the name of an internal table + /// + public SortedTreeTable OpenOrCreateTable(EncodingDefinition storageMethod, string tableName, int maxSortedTreeBlockSize = 4096) + where TKey : SnapTypeBase, new() + where TValue : SnapTypeBase, new() + { + if (storageMethod is null) + throw new ArgumentNullException(nameof(storageMethod)); - SubFileName fileName = GetFileName(tableName); - return OpenOrCreateTable(storageMethod, fileName, maxSortedTreeBlockSize); - } + SubFileName fileName = GetFileName(tableName); + + return OpenOrCreateTable(storageMethod, fileName, maxSortedTreeBlockSize); + } - /// - /// Opens the default table for this TKey and TValue. If it does not exists, - /// it will be created with the provided compression method. - /// - /// - /// - /// The method of compression to utilize in this table. - /// the maximum desired block size for a SortedTree. Must be at least 1024. - /// - public SortedTreeTable OpenOrCreateTable(EncodingDefinition storageMethod, int maxSortedTreeBlockSize = 4096) - where TKey : SnapTypeBase, new() - where TValue : SnapTypeBase, new() - { - if (storageMethod is null) - throw new ArgumentNullException("storageMethod"); + /// + /// Opens the default table for this TKey and TValue. If it does not exists, + /// it will be created with the provided compression method. + /// + /// + /// + /// The method of compression to utilize in this table. + /// the maximum desired block size for a SortedTree. Must be at least 1024. + /// + public SortedTreeTable OpenOrCreateTable(EncodingDefinition storageMethod, int maxSortedTreeBlockSize = 4096) + where TKey : SnapTypeBase, new() + where TValue : SnapTypeBase, new() + { + if (storageMethod is null) + throw new ArgumentNullException(nameof(storageMethod)); - SubFileName fileName = GetFileName(); - return OpenOrCreateTable(storageMethod, fileName, maxSortedTreeBlockSize); - } + SubFileName fileName = GetFileName(); + + return OpenOrCreateTable(storageMethod, fileName, maxSortedTreeBlockSize); + } - private SortedTreeTable OpenOrCreateTable(EncodingDefinition storageMethod, SubFileName fileName, int maxSortedTreeBlockSize) - where TKey : SnapTypeBase, new() - where TValue : SnapTypeBase, new() + private SortedTreeTable OpenOrCreateTable(EncodingDefinition storageMethod, SubFileName fileName, int maxSortedTreeBlockSize) + where TKey : SnapTypeBase, new() + where TValue : SnapTypeBase, new() + { + if (!m_openedFiles.ContainsKey(fileName)) { - if (!m_openedFiles.ContainsKey(fileName)) - { - if (!m_fileStructure.Snapshot.Header.ContainsSubFile(fileName)) - CreateArchiveFile(fileName, storageMethod, maxSortedTreeBlockSize); - m_openedFiles.Add(fileName, new SortedTreeTable(m_fileStructure, fileName, this)); - } - return (SortedTreeTable)m_openedFiles[fileName]; + if (!m_fileStructure.Snapshot.Header.ContainsSubFile(fileName)) + CreateArchiveFile(fileName, storageMethod, maxSortedTreeBlockSize); + m_openedFiles.Add(fileName, new SortedTreeTable(m_fileStructure, fileName, this)); } + + return (SortedTreeTable)m_openedFiles[fileName]; + } - /// - /// Helper method. Creates the for the default table. - /// - /// - /// - /// - private SubFileName GetFileName() - where TKey : SnapTypeBase, new() - where TValue : SnapTypeBase, new() - { - Guid keyType = new TKey().GenericTypeGuid; - Guid valueType = new TValue().GenericTypeGuid; - return SubFileName.Create(PrimaryArchiveType, keyType, valueType); - } + /// + /// Helper method. Creates the for the default table. + /// + /// + /// + /// + private SubFileName GetFileName() + where TKey : SnapTypeBase, new() + where TValue : SnapTypeBase, new() + { + Guid keyType = new TKey().GenericTypeGuid; + Guid valueType = new TValue().GenericTypeGuid; + + return SubFileName.Create(PrimaryArchiveType, keyType, valueType); + } - private SubFileName GetFileName(string fileName) - where TKey : SnapTypeBase, new() - where TValue : SnapTypeBase, new() - { - Guid keyType = new TKey().GenericTypeGuid; - Guid valueType = new TValue().GenericTypeGuid; - return SubFileName.Create(fileName, keyType, valueType); - } + private SubFileName GetFileName(string fileName) + where TKey : SnapTypeBase, new() + where TValue : SnapTypeBase, new() + { + Guid keyType = new TKey().GenericTypeGuid; + Guid valueType = new TValue().GenericTypeGuid; + + return SubFileName.Create(fileName, keyType, valueType); + } - private void CreateArchiveFile(SubFileName fileName, EncodingDefinition storageMethod, int maxSortedTreeBlockSize) - where TKey : SnapTypeBase, new() - where TValue : SnapTypeBase, new() + private void CreateArchiveFile(SubFileName fileName, EncodingDefinition storageMethod, int maxSortedTreeBlockSize) + where TKey : SnapTypeBase, new() + where TValue : SnapTypeBase, new() + { + if (maxSortedTreeBlockSize < 1024) + throw new ArgumentOutOfRangeException(nameof(maxSortedTreeBlockSize), "Must be greater than 1024"); + if (storageMethod is null) + throw new ArgumentNullException(nameof(storageMethod)); + + using TransactionalEdit trans = m_fileStructure.BeginEdit(); + using (SubFileStream fs = trans.CreateFile(fileName)) + using (BinaryStream bs = new(fs)) { - if (maxSortedTreeBlockSize < 1024) - throw new ArgumentOutOfRangeException(nameof(maxSortedTreeBlockSize), "Must be greater than 1024"); - if (storageMethod is null) - throw new ArgumentNullException("storageMethod"); - - using (TransactionalEdit trans = m_fileStructure.BeginEdit()) - { - using (SubFileStream fs = trans.CreateFile(fileName)) - using (BinaryStream bs = new BinaryStream(fs)) - { - int blockSize = m_fileStructure.Snapshot.Header.DataBlockSize; - - while (blockSize > maxSortedTreeBlockSize) - { - blockSize >>= 2; - } - - SortedTree tree = SortedTree.Create(bs, blockSize, storageMethod); - tree.Flush(); - } - trans.ArchiveType = FileType; - trans.CommitAndDispose(); - } - } + int blockSize = m_fileStructure.Snapshot.Header.DataBlockSize; + while (blockSize > maxSortedTreeBlockSize) + blockSize >>= 2; - /// - /// Closes the archive file. If there is a current transaction, - /// that transaction is immediately rolled back and disposed. - /// - public void Dispose() - { - if (!m_disposed) - { - foreach (IDisposable d in m_openedFiles.Values) - { - d.Dispose(); - } - m_openedFiles.Clear(); - m_fileStructure.Dispose(); - m_disposed = true; - } - } - - /// - /// Closes and deletes the Archive File. Also calls dispose. - /// If this is a memory archive, it will release the memory space to the buffer pool. - /// - public void Delete() - { - Dispose(); - if (m_filePath != string.Empty) - { - File.Delete(m_filePath); - } + SortedTree tree = SortedTree.Create(bs, blockSize, storageMethod); + + tree.Flush(); } + + trans.ArchiveType = FileType; + trans.CommitAndDispose(); + } - #endregion + /// + /// Closes the archive file. If there is a current transaction, + /// that transaction is immediately rolled back and disposed. + /// + public void Dispose() + { + if (IsDisposed) + return; + + foreach (IDisposable d in m_openedFiles.Values) + d.Dispose(); + + m_openedFiles.Clear(); + m_fileStructure.Dispose(); + IsDisposed = true; + } + /// + /// Closes and deletes the Archive File. Also calls Dispose. + /// If this is a memory archive, it will release the memory space to the buffer pool. + /// + public void Delete() + { + Dispose(); + + if (FilePath == string.Empty) + return; + + File.Delete(FilePath); + + ArchivePointIDIndex.Delete(FilePath); + ArchiveTimeBucketBloomIndex.Delete(FilePath); } + + #endregion } \ No newline at end of file diff --git a/Source/Libraries/GSF.SortedTreeStore/Snap/Storage/SortedTreeFileSimpleWriter.cs b/Source/Libraries/GSF.SortedTreeStore/Snap/Storage/SortedTreeFileSimpleWriter.cs index b761945797..74be76480d 100755 --- a/Source/Libraries/GSF.SortedTreeStore/Snap/Storage/SortedTreeFileSimpleWriter.cs +++ b/Source/Libraries/GSF.SortedTreeStore/Snap/Storage/SortedTreeFileSimpleWriter.cs @@ -30,110 +30,110 @@ using GSF.Snap.Services.Reader; using GSF.Snap.Tree.Specialized; -namespace GSF.Snap.Storage +namespace GSF.Snap.Storage; + +/// +/// Provides a simple interface for writing a to disk. +/// +/// > +/// This class is intended for use in scenarios where the data is already sorted and can be written sequentially to disk. +/// +public static class SortedTreeFileSimpleWriter + where TKey : SnapTypeBase, new() + where TValue : SnapTypeBase, new() { /// - /// Will write a file. + /// Creates a new archive file with the supplied data. /// - /// - /// - public static class SortedTreeFileSimpleWriter - where TKey : SnapTypeBase, new() - where TValue : SnapTypeBase, new() + public static void Create(string pendingFileName, string completeFileName, int blockSize, Action archiveIdCallback, EncodingDefinition treeNodeType, TreeStream treeStream, params Guid[] flags) { - /// - /// Creates a new arhive file with the supplied data. - /// - /// - /// - /// - /// - /// - /// - public static void Create(string pendingFileName, string completeFileName, int blockSize, Action archiveIdCallback, EncodingDefinition treeNodeType, TreeStream treeStream, params Guid[] flags) - { - using (SimplifiedFileWriter writer = new SimplifiedFileWriter(pendingFileName, completeFileName, blockSize, flags)) - { - if (archiveIdCallback != null) - archiveIdCallback(writer.ArchiveId); - - using (ISupportsBinaryStream file = writer.CreateFile(GetFileName())) - using (BinaryStream bs = new BinaryStream(file)) - { - SequentialSortedTreeWriter.Create(bs, blockSize - 32, treeNodeType, treeStream); - } - writer.Commit(); - } - } + CreateWithBloomFilter(pendingFileName, completeFileName, blockSize, archiveIdCallback, treeNodeType, treeStream, new TimeBucketBloomFilterSettings(), flags); + } - public static void CreateNonSequential(string pendingFileName, string completeFileName, int blockSize, Action archiveIdCallback, EncodingDefinition treeNodeType, TreeStream treeStream, params Guid[] flags) - { - SortedPointBuffer m_queue; - m_queue = new SortedPointBuffer(100000, true); - m_queue.IsReadingMode = false; + /// + /// Writes a finalized archive with optional time-bucket Bloom indexing. + /// + public static void CreateWithBloomFilter(string pendingFileName, string completeFileName, int blockSize, Action archiveIdCallback, EncodingDefinition treeNodeType, TreeStream treeStream, TimeBucketBloomFilterSettings bloomFilter, params Guid[] flags) + { + CreateWithIndexes(pendingFileName, completeFileName, blockSize, archiveIdCallback, treeNodeType, treeStream, bloomFilter, true, flags); + } + + /// + /// Writes a finalized archive with independently controlled presence and Bloom indexes. + /// + public static void CreateWithIndexes(string pendingFileName, string completeFileName, int blockSize, Action archiveIdCallback, EncodingDefinition treeNodeType, TreeStream treeStream, TimeBucketBloomFilterSettings bloomFilter, bool enablePointIDIndex, params Guid[] flags) + { + PointIDCollectingStream indexedStream = new(treeStream, bloomFilter, enablePointIDIndex); + using SimplifiedFileWriter writer = new(pendingFileName, completeFileName, blockSize, flags); + + archiveIdCallback?.Invoke(writer.ArchiveId); - TKey key = new TKey(); - TValue value = new TValue(); + using (ISupportsBinaryStream file = writer.CreateFile(GetFileName())) + using (BinaryStream bs = new(file)) + { + SequentialSortedTreeWriter.Create(bs, blockSize - 32, treeNodeType, indexedStream); + } + + writer.Commit(); + indexedStream.Publish(completeFileName, writer.ArchiveId, writer.SnapshotSequenceNumber); + } - List> pendingFiles = new List>(); + public static void CreateNonSequential(string pendingFileName, string completeFileName, int blockSize, Action archiveIdCallback, EncodingDefinition treeNodeType, TreeStream treeStream, params Guid[] flags) + { + SortedPointBuffer queue = new(100000, true); + queue.IsReadingMode = false; - try - { - while (treeStream.Read(key, value)) - { - if (m_queue.IsFull) - { - pendingFiles.Add(CreateMemoryFile(treeNodeType, m_queue)); - } - m_queue.TryEnqueue(key, value); - } - - if (m_queue.Count > 0) - { - pendingFiles.Add(CreateMemoryFile(treeNodeType, m_queue)); - } - - using (UnionTreeStream reader = new UnionTreeStream(pendingFiles.Select(x => new ArchiveTreeStreamWrapper(x)), false)) - { - Create(pendingFileName, completeFileName, blockSize, archiveIdCallback, treeNodeType, reader, flags); - } + TKey key = new(); + TValue value = new(); - } - finally - { - pendingFiles.ForEach(x => x.Dispose()); - } - } + List> pendingFiles = new(); - private static SortedTreeTable CreateMemoryFile(EncodingDefinition treeNodeType, SortedPointBuffer buffer) + try { - buffer.IsReadingMode = true; - - SortedTreeFile file = SortedTreeFile.CreateInMemory(4096); - SortedTreeTable table = file.OpenOrCreateTable(treeNodeType); - using (SortedTreeTableEditor edit = table.BeginEdit()) + while (treeStream.Read(key, value)) { - edit.AddPoints(buffer); - edit.Commit(); + if (queue.IsFull) + pendingFiles.Add(CreateMemoryFile(treeNodeType, queue)); + + queue.TryEnqueue(key, value); } - buffer.IsReadingMode = false; - return table; + if (queue.Count > 0) + pendingFiles.Add(CreateMemoryFile(treeNodeType, queue)); + + using UnionTreeStream reader = new(pendingFiles.Select(x => new ArchiveTreeStreamWrapper(x)), false); + Create(pendingFileName, completeFileName, blockSize, archiveIdCallback, treeNodeType, reader, flags); + } + finally + { + pendingFiles.ForEach(x => x.Dispose()); } + } - /// - /// Helper method. Creates the for the default table. - /// - /// - /// - /// - private static SubFileName GetFileName() + private static SortedTreeTable CreateMemoryFile(EncodingDefinition treeNodeType, SortedPointBuffer buffer) + { + buffer.IsReadingMode = true; + + SortedTreeFile file = SortedTreeFile.CreateInMemory(); + SortedTreeTable table = file.OpenOrCreateTable(treeNodeType); + + using (SortedTreeTableEditor edit = table.BeginEdit()) { - Guid keyType = new TKey().GenericTypeGuid; - Guid valueType = new TValue().GenericTypeGuid; - return SubFileName.Create(SortedTreeFile.PrimaryArchiveType, keyType, valueType); + edit.AddPoints(buffer); + edit.Commit(); } + buffer.IsReadingMode = false; + + return table; + } + // Helper method. Creates the for the default table. + private static SubFileName GetFileName() + { + Guid keyType = new TKey().GenericTypeGuid; + Guid valueType = new TValue().GenericTypeGuid; + + return SubFileName.Create(SortedTreeFile.PrimaryArchiveType, keyType, valueType); } -} +} \ No newline at end of file diff --git a/Source/Libraries/GSF.SortedTreeStore/Snap/Storage/SortedTreeTableSnapshotInfo`2.cs b/Source/Libraries/GSF.SortedTreeStore/Snap/Storage/SortedTreeTableSnapshotInfo`2.cs index b663fd974c..e43fdc84f5 100755 --- a/Source/Libraries/GSF.SortedTreeStore/Snap/Storage/SortedTreeTableSnapshotInfo`2.cs +++ b/Source/Libraries/GSF.SortedTreeStore/Snap/Storage/SortedTreeTableSnapshotInfo`2.cs @@ -21,48 +21,85 @@ // //****************************************************************************************************** +using System; using GSF.IO.FileStructure; -namespace GSF.Snap.Storage +namespace GSF.Snap.Storage; + +/// +/// Acquires a read transaction on the current archive partition. This will allow all user created +/// transactions to have snapshot isolation of the entire data set. +/// +public class SortedTreeTableSnapshotInfo + where TKey : SnapTypeBase, new() + where TValue : SnapTypeBase, new() { - /// - /// Aquires a read transaction on the current archive partition. This will allow all user created - /// transactions to have snapshot isolation of the entire data set. - /// - public class SortedTreeTableSnapshotInfo - where TKey : SnapTypeBase, new() - where TValue : SnapTypeBase, new() - { - #region [ Members ] + #region [ Members ] + + private readonly TransactionalFileStructure m_fileStructure; + private readonly Lazy m_pointIDs; + private readonly Lazy m_bloom; + private readonly ReadSnapshot m_currentTransaction; + private readonly SubFileName m_fileName; + + #endregion - private readonly TransactionalFileStructure m_fileStructure; - private readonly ReadSnapshot m_currentTransaction; - private readonly SubFileName m_fileName; + #region [ Constructors ] - #endregion + internal SortedTreeTableSnapshotInfo(TransactionalFileStructure fileStructure, SubFileName fileName) + { + m_fileName = fileName; + m_fileStructure = fileStructure; + m_currentTransaction = m_fileStructure.Snapshot; - #region [ Constructors ] + m_pointIDs = new Lazy(() => + { + Guid keyType = new TKey().GenericTypeGuid; + Guid valueType = new TValue().GenericTypeGuid; + + return m_fileName == SubFileName.Create(SortedTreeFile.PrimaryArchiveType, keyType, valueType) ? + ArchivePointIDIndex.Load(m_fileStructure.FileName, m_currentTransaction.Header.ArchiveId, m_currentTransaction.Header.SnapshotSequenceNumber, keyType, valueType) : + null; + }); - internal SortedTreeTableSnapshotInfo(TransactionalFileStructure fileStructure, SubFileName fileName) + m_bloom = new Lazy(() => { - m_fileName = fileName; - m_fileStructure = fileStructure; - m_currentTransaction = m_fileStructure.Snapshot; - } + Guid keyType = new TKey().GenericTypeGuid; + Guid valueType = new TValue().GenericTypeGuid; + + return m_fileName == SubFileName.Create(SortedTreeFile.PrimaryArchiveType, keyType, valueType) ? + ArchiveTimeBucketBloomIndex.Load(m_fileStructure.FileName, m_currentTransaction.Header.ArchiveId, m_currentTransaction.Header.SnapshotSequenceNumber, keyType, valueType) : + null; + }); + } - #endregion + #endregion - #region [ Methods ] + #region [ Properties ] - /// - /// Opens an instance of the archive file to allow for concurrent reading of a snapshot. - /// - /// - public SortedTreeTableReadSnapshot CreateReadSnapshot() - { - return new SortedTreeTableReadSnapshot(m_currentTransaction, m_fileName); - } + internal bool HasTimeBucketBloomIndex => m_bloom.Value is not null; + + #endregion - #endregion + #region [ Methods ] + + /// + /// Opens an instance of the archive file to allow for concurrent reading of a snapshot. + /// + public SortedTreeTableReadSnapshot CreateReadSnapshot() + { + return new SortedTreeTableReadSnapshot(m_currentTransaction, m_fileName); } + + internal bool MayContainPointID(ulong id, ulong lower, ulong upper) + { + return m_bloom.Value is null || m_bloom.Value.MayContain(id, lower, upper); + } + + internal bool MayContainPointID(ulong id) + { + return m_pointIDs.Value is null || Array.BinarySearch(m_pointIDs.Value, id) >= 0; + } + + #endregion } \ No newline at end of file diff --git a/Source/Libraries/GSF.SortedTreeStore/Snap/Storage/TimeBucketBloomFilterSettings.cs b/Source/Libraries/GSF.SortedTreeStore/Snap/Storage/TimeBucketBloomFilterSettings.cs new file mode 100644 index 0000000000..cdf68ba05b --- /dev/null +++ b/Source/Libraries/GSF.SortedTreeStore/Snap/Storage/TimeBucketBloomFilterSettings.cs @@ -0,0 +1,72 @@ +//****************************************************************************************************** +// TimeBucketBloomFilterSettings.cs - Gbtc +// +// Copyright © 2026, Grid Protection Alliance. All Rights Reserved. +// +// Licensed to the Grid Protection Alliance (GPA) under one or more contributor license agreements. See +// the NOTICE file distributed with this work for additional information regarding copyright ownership. +// The GPA licenses this file to you under the MIT License (MIT), the "License"; you may not use this +// file except in compliance with the License. You may obtain a copy of the License at: +// +// http://opensource.org/licenses/MIT +// +// Unless agreed to in writing, the subject software distributed under the License is distributed on an +// "AS-IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. Refer to the +// License for the specific language governing permissions and limitations. +// +// Code Modification History: +// ---------------------------------------------------------------------------------------------------- +// 09/22/2026 - J. Ritchie Carroll +// Generated original version of source code. +// +//****************************************************************************************************** + +using System; + +namespace GSF.Snap.Storage; + +/// Immutable options for optional time-bucket PointID Bloom sidecars. +public sealed class TimeBucketBloomFilterSettings +{ + /// + /// Whether finalized archives should receive a Bloom sidecar. + /// + public bool Enabled { get; } + + /// + /// Bucket width in seconds, from 1 through 86400. + /// + public int BucketDurationSeconds { get; } + + /// + /// Target false-positive probability per bucket, from 0.0001 through 0.25. + /// + public double FalsePositiveProbability { get; } + + /// + /// Maximum uncompressed index payload per archive in MiB (1 through 64). + /// + public int MaximumIndexSizeMiB { get; } + + /// Creates validated settings. Disabled by default; existing indexes remain readable. + /// Whether new finalized files should receive Bloom indexes. + /// Time bucket width in seconds, 1 through 86400. + /// Target per-bucket probability, 0.0001 through 0.25. + /// Maximum uncompressed index payload per archive, 1 through 64 MiB. + public TimeBucketBloomFilterSettings(bool enabled = false, int bucketDurationSeconds = 900, double falsePositiveProbability = 0.01D, int maximumIndexSizeMiB = 64) + { + if (bucketDurationSeconds is < 1 or > 86400) + throw new ArgumentOutOfRangeException(nameof(bucketDurationSeconds)); + + if (double.IsNaN(falsePositiveProbability) || falsePositiveProbability < 0.0001D || falsePositiveProbability > 0.25D) + throw new ArgumentOutOfRangeException(nameof(falsePositiveProbability)); + + if (maximumIndexSizeMiB is < 1 or > 64) + throw new ArgumentOutOfRangeException(nameof(maximumIndexSizeMiB)); + + MaximumIndexSizeMiB = maximumIndexSizeMiB; + Enabled = enabled; + BucketDurationSeconds = bucketDurationSeconds; + FalsePositiveProbability = falsePositiveProbability; + } +} diff --git a/Source/Libraries/GSF.SortedTreeStore/Snap/Tree/GenericEncodedNodeScanner`2.cs b/Source/Libraries/GSF.SortedTreeStore/Snap/Tree/GenericEncodedNodeScanner`2.cs index d444b25641..9c293f6b28 100755 --- a/Source/Libraries/GSF.SortedTreeStore/Snap/Tree/GenericEncodedNodeScanner`2.cs +++ b/Source/Libraries/GSF.SortedTreeStore/Snap/Tree/GenericEncodedNodeScanner`2.cs @@ -26,140 +26,142 @@ using GSF.Snap.Encoding; using GSF.Snap.Filters; -namespace GSF.Snap.Tree +namespace GSF.Snap.Tree; + +/// +/// Base class for reading from a node that is encoded and must be read sequentially through the node. +/// +public unsafe class GenericEncodedNodeScanner + : SortedTreeScannerBase + where TKey : SnapTypeBase, new() + where TValue : SnapTypeBase, new() { + private readonly PairEncodingBase m_encoding; + private readonly TKey m_prevKey; + private readonly TValue m_prevValue; + private int m_nextOffset; + private readonly TKey m_tmpKey; + private readonly TValue m_tmpValue; + /// - /// Base class for reading from a node that is encoded and must be read sequentially through the node. + /// Creates a new class /// - /// - /// - public unsafe class GenericEncodedNodeScanner - : SortedTreeScannerBase - where TKey : SnapTypeBase, new() - where TValue : SnapTypeBase, new() + public GenericEncodedNodeScanner(PairEncodingBase encoding, byte level, int blockSize, BinaryStreamPointerBase stream, Func lookupKey) + : base(level, blockSize, stream, lookupKey) { - private readonly PairEncodingBase m_encoding; - private readonly TKey m_prevKey; - private readonly TValue m_prevValue; - private int m_nextOffset; - private readonly TKey m_tmpKey; - private readonly TValue m_tmpValue; - - /// - /// Creates a new class - /// - /// - /// - /// - /// - public GenericEncodedNodeScanner(PairEncodingBase encoding, byte level, int blockSize, BinaryStreamPointerBase stream, Func lookupKey) - : base(level, blockSize, stream, lookupKey) - { - m_encoding = encoding; - m_nextOffset = 0; - m_prevKey = new TKey(); - m_prevValue = new TValue(); - m_prevKey.Clear(); - m_prevValue.Clear(); - m_tmpKey = new TKey(); - m_tmpValue = new TValue(); - } + m_encoding = encoding; + m_nextOffset = 0; + m_prevKey = new TKey(); + m_prevValue = new TValue(); + m_prevKey.Clear(); + m_prevValue.Clear(); + m_tmpKey = new TKey(); + m_tmpValue = new TValue(); + } - protected override void InternalPeek(TKey key, TValue value) - { - byte* stream = Pointer + m_nextOffset; - m_encoding.Decode(stream, m_prevKey, m_prevValue, key, value, out _); - } + protected override void InternalPeek(TKey key, TValue value) + { + byte* stream = Pointer + m_nextOffset; + m_encoding.Decode(stream, m_prevKey, m_prevValue, key, value, out _); + } - protected override void InternalRead(TKey key, TValue value) - { - byte* stream = Pointer + m_nextOffset; - int length = m_encoding.Decode(stream, m_prevKey, m_prevValue, key, value, out _); - key.CopyTo(m_prevKey); - value.CopyTo(m_prevValue); - m_nextOffset += length; - IndexOfNextKeyValue++; - } + protected override void InternalRead(TKey key, TValue value) + { + byte* stream = Pointer + m_nextOffset; + int length = m_encoding.Decode(stream, m_prevKey, m_prevValue, key, value, out _); + + key.CopyTo(m_prevKey); + value.CopyTo(m_prevValue); + + m_nextOffset += length; + IndexOfNextKeyValue++; + } - protected override bool InternalRead(TKey key, TValue value, MatchFilterBase filter) - { - TryAgain: - byte* stream = Pointer + m_nextOffset; - int length = m_encoding.Decode(stream, m_prevKey, m_prevValue, key, value, out _); - key.CopyTo(m_prevKey); - value.CopyTo(m_prevValue); - m_nextOffset += length; - IndexOfNextKeyValue++; - - if (filter.Contains(key, value)) - return true; - if (IndexOfNextKeyValue >= RecordCount) - return false; - - goto TryAgain; - } + protected override bool InternalRead(TKey key, TValue value, MatchFilterBase filter) + { + TryAgain: + byte* stream = Pointer + m_nextOffset; + int length = m_encoding.Decode(stream, m_prevKey, m_prevValue, key, value, out _); + + key.CopyTo(m_prevKey); + value.CopyTo(m_prevValue); + + m_nextOffset += length; + IndexOfNextKeyValue++; + + if (filter.Contains(key, value)) + return true; + + if (IndexOfNextKeyValue >= RecordCount) + return false; - protected override bool InternalReadWhile(TKey key, TValue value, TKey upperBounds) - { - byte* stream = Pointer + m_nextOffset; - int length = m_encoding.Decode(stream, m_prevKey, m_prevValue, key, value, out _); - - if (key.IsLessThan(upperBounds)) - { - key.CopyTo(m_prevKey); - value.CopyTo(m_prevValue); - m_nextOffset += length; - IndexOfNextKeyValue++; - return true; - } + goto TryAgain; + } + + protected override bool InternalReadWhile(TKey key, TValue value, TKey upperBounds) + { + byte* stream = Pointer + m_nextOffset; + int length = m_encoding.Decode(stream, m_prevKey, m_prevValue, key, value, out _); + + if (!key.IsLessThan(upperBounds)) return false; - } + + key.CopyTo(m_prevKey); + value.CopyTo(m_prevValue); + + m_nextOffset += length; + IndexOfNextKeyValue++; + + return true; + } - protected override bool InternalReadWhile(TKey key, TValue value, TKey upperBounds, MatchFilterBase filter) - { - TryAgain: - - byte* stream = Pointer + m_nextOffset; - int length = m_encoding.Decode(stream, m_prevKey, m_prevValue, key, value, out _); - - if (key.IsLessThan(upperBounds)) - { - key.CopyTo(m_prevKey); - value.CopyTo(m_prevValue); - m_nextOffset += length; - IndexOfNextKeyValue++; - - if (filter.Contains(key, value)) - return true; - if (IndexOfNextKeyValue >= RecordCount) - return false; - goto TryAgain; - } + protected override bool InternalReadWhile(TKey key, TValue value, TKey upperBounds, MatchFilterBase filter) + { + TryAgain: + + byte* stream = Pointer + m_nextOffset; + int length = m_encoding.Decode(stream, m_prevKey, m_prevValue, key, value, out _); + + if (!key.IsLessThan(upperBounds)) return false; - } + + key.CopyTo(m_prevKey); + value.CopyTo(m_prevValue); + + m_nextOffset += length; + IndexOfNextKeyValue++; + + if (filter.Contains(key, value)) + return true; + + if (IndexOfNextKeyValue >= RecordCount) + return false; + + goto TryAgain; + } - /// - /// Using advance to the search location of the provided - /// - /// the key to advance to - protected override void FindKey(TKey key) - { - OnNoadReload(); - while (IndexOfNextKeyValue < RecordCount && InternalReadWhile(m_tmpKey, m_tmpValue, key)) - { - } - } + /// + /// Using advance to the search location of the provided + /// + /// the key to advance to + protected override void FindKey(TKey key) + { + OnNodeReload(); - /// - /// Occurs when a node's data is reset. - /// Derived classes can override this - /// method if fields need to be reset when a node is loaded. - /// - protected override void OnNoadReload() + while (IndexOfNextKeyValue < RecordCount && InternalReadWhile(m_tmpKey, m_tmpValue, key)) { - m_nextOffset = 0; - m_prevKey.Clear(); - m_prevValue.Clear(); } } + + /// + /// Occurs when a node's data is reset. + /// Derived classes can override this + /// method if fields need to be reset when a node is loaded. + /// + protected override void OnNodeReload() + { + m_nextOffset = 0; + m_prevKey.Clear(); + m_prevValue.Clear(); + } } \ No newline at end of file diff --git a/Source/Libraries/GSF.SortedTreeStore/Snap/Tree/SortedTreeScannerBase'2.cs b/Source/Libraries/GSF.SortedTreeStore/Snap/Tree/SortedTreeScannerBase'2.cs index 6dcb7f769d..944771f8fe 100755 --- a/Source/Libraries/GSF.SortedTreeStore/Snap/Tree/SortedTreeScannerBase'2.cs +++ b/Source/Libraries/GSF.SortedTreeStore/Snap/Tree/SortedTreeScannerBase'2.cs @@ -22,6 +22,7 @@ //****************************************************************************************************** using System; +using System.Collections.Generic; using GSF.IO; using GSF.Snap.Filters; @@ -48,51 +49,41 @@ public abstract unsafe class SortedTreeScannerBase private const int IndexSize = sizeof(uint); protected int KeySize; protected int ValueSize; - protected TKey UpperKey = new TKey(); - protected TKey LowerKey = new TKey(); + protected TKey UpperKey = new(); + protected TKey LowerKey = new(); private readonly Func m_lookupKey; private readonly TKey m_tempKey; - //private TKey m_lowerKey; - //private TKey m_upperKey; + private readonly List m_reverseKeys = []; + private readonly List m_reverseValues = []; + private readonly byte m_level; + private readonly int m_blockSize; // JRC: 09/22/2026 -- Changed to private from protected, saw no other external uses + + /// + /// Gets the methods for the key type. + /// protected SnapTypeCustomMethods KeyMethods; /// /// The index of the current node. /// - protected uint NodeIndex - { - get; - private set; - } + protected uint NodeIndex { get; private set; } /// /// The number of records in the current node. /// - protected ushort RecordCount - { - get; - private set; - } + protected ushort RecordCount { get; private set; } /// /// The node index of the previous sibling. /// uint.MaxValue means there is no sibling to the right. /// - protected uint LeftSiblingNodeIndex - { - get; - private set; - } + protected uint LeftSiblingNodeIndex { get; private set; } /// /// The node index of the next sibling. /// uint.MaxValue means there is no sibling to the right. /// - protected uint RightSiblingNodeIndex - { - get; - private set; - } + protected uint RightSiblingNodeIndex { get; private set; } /// /// Gets the byte offset of the upper bounds key @@ -111,8 +102,9 @@ protected uint RightSiblingNodeIndex /// protected long PointerVersion { get; private set; } - private readonly byte m_level; - protected readonly int m_blockSize; + /// + /// The stream that is being read from. + /// protected readonly BinaryStreamPointerBase Stream; /// @@ -124,23 +116,24 @@ protected uint RightSiblingNodeIndex /// /// The number of bytes in the header of any given node. /// - protected int HeaderSize { get; private set; } - //protected int OffsetOfUpperBounds; + protected int HeaderSize { get; } + + /// + /// Gets if the stream will never return duplicate keys. Do not return true unless it is Guaranteed that + /// the data read from this stream will never contain duplicates. + /// + public override bool NeverContainsDuplicates => true; protected SortedTreeScannerBase(byte level, int blockSize, BinaryStreamPointerBase stream, Func lookupKey) { m_tempKey = new TKey(); - //m_lowerKey = new TKey(); - //m_upperKey = new TKey(); m_lookupKey = lookupKey; m_level = level; - //m_currentNode = new Node(stream, blockSize); KeyMethods = m_tempKey.CreateValueMethods(); KeySize = new TKey().Size; ValueSize = new TValue().Size; - //OffsetOfUpperBounds = OffsetOfLowerBounds + KeySize; HeaderSize = OffsetOfLowerBounds + 2 * KeySize; m_blockSize = blockSize; Stream = stream; @@ -155,11 +148,64 @@ protected SortedTreeScannerBase(byte level, int blockSize, BinaryStreamPointerBa /// public override bool IsAlwaysSequential => true; - /// - /// Gets if the stream will never return duplicate keys. Do not return true unless it is Guaranteed that - /// the data read from this stream will never contain duplicates. - /// - public override bool NeverContainsDuplicates => true; + // Visit from an indexed seek without setting the stream's EOS/disposed state. Nearest + // queries reposition the same snapshot repeatedly, including after the last stored key. + internal void VisitForward(TKey start, Func visitor) + { + SeekToKey(start); + TKey key = new(); + TValue value = new(); + + while (true) + { + while (IndexOfNextKeyValue < RecordCount) + { + InternalRead(key, value); + if (!visitor(key, value)) + return; + } + + if (RightSiblingNodeIndex == uint.MaxValue) + return; + + LoadNode(RightSiblingNodeIndex); + } + } + + // Compressed leaves must be decoded forwards. Decode one leaf once, then visit its + // buffered records backwards; memory is bounded by the largest leaf encountered. + internal void VisitBackward(TKey exclusiveStart, Func visitor) + { + SeekToKey(exclusiveStart); + int count = IndexOfNextKeyValue; + LoadNode(NodeIndex); + + while (true) + { + for (int i = 0; i < count; i++) + { + if (i == m_reverseKeys.Count) + { + m_reverseKeys.Add(new TKey()); + m_reverseValues.Add(new TValue()); + } + + InternalRead(m_reverseKeys[i], m_reverseValues[i]); + } + + for (int i = count - 1; i >= 0; i--) + { + if (!visitor(m_reverseKeys[i], m_reverseValues[i])) + return; + } + + if (LeftSiblingNodeIndex == uint.MaxValue) + return; + + LoadNode(LeftSiblingNodeIndex); + count = RecordCount; + } + } protected abstract void InternalPeek(TKey key, TValue value); @@ -180,7 +226,7 @@ protected SortedTreeScannerBase(byte level, int blockSize, BinaryStreamPointerBa #region [ Peek ] /// - /// Reads the next point, but doees not advance the position of the stream. + /// Reads the next point, but does not advance the position of the stream. /// /// the key to write the results to /// the value to write the results to @@ -192,13 +238,14 @@ public bool Peek(TKey key, TValue value) { if (Stream.PointerVersion == PointerVersion) { - //A light weight function that can be called quickly since 99% of the time, this logic statement will return successfully. + // A lightweight function that can be called quickly since 99% of the time, this logic statement will return successfully. if (IndexOfNextKeyValue < RecordCount) { InternalPeek(key, value); return true; } } + return PeekCatchAll(key, value); } @@ -212,18 +259,22 @@ protected bool PeekCatchAll(TKey key, TValue value) { key.Clear(); value.Clear(); + Dispose(); + return false; } + LoadNode(RightSiblingNodeIndex); } + //if the pointer data is no longer valid, refresh the pointer if (Stream.PointerVersion != PointerVersion) - { RefreshPointer(); - } + //Reads the next key in the sequence. InternalPeek(key, value); + return true; } @@ -251,7 +302,7 @@ public virtual bool ReadWhile(TKey key, TValue value, TKey upperBounds) { if (Stream.PointerVersion == PointerVersion) { - //A light weight function that can be called quickly since 99% of the time, this logic statement will return successfully. + // A lightweight function that can be called quickly since 99% of the time, this logic statement will return successfully. if (IndexOfNextKeyValue < RecordCount) { if (UpperKey.IsLessThan(upperBounds)) @@ -259,38 +310,44 @@ public virtual bool ReadWhile(TKey key, TValue value, TKey upperBounds) InternalRead(key, value); return true; } + return InternalReadWhile(key, value, upperBounds); } } + return ReadWhileCatchAll(key, value, upperBounds); } protected bool ReadWhileCatchAll(TKey key, TValue value, TKey upperBounds) { - //If there are no more records in the current node. + // If there are no more records in the current node. if (IndexOfNextKeyValue >= RecordCount) { - //If the last leaf node, return false + // If the last leaf node, return false if (RightSiblingNodeIndex == uint.MaxValue) { key.Clear(); value.Clear(); + Dispose(); + return false; } + LoadNode(RightSiblingNodeIndex); } - //if the pointer data is no longer valid, refresh the pointer + + // If the pointer data is no longer valid, refresh the pointer if (Stream.PointerVersion != PointerVersion) - { RefreshPointer(); - } - //Reads the next key in the sequence. + + // Reads the next key in the sequence. if (UpperKey.IsLessThan(upperBounds)) { InternalRead(key, value); return true; } + return InternalReadWhile(key, value, upperBounds); } @@ -315,42 +372,36 @@ protected bool ReadWhileCatchAll(TKey key, TValue value, TKey upperBounds) public virtual bool ReadWhile(TKey key, TValue value, TKey upperBounds, MatchFilterBase filter) { if (Stream.PointerVersion == PointerVersion && IndexOfNextKeyValue < RecordCount) - { - if (UpperKey.IsLessThan(upperBounds)) - { - return InternalRead(key, value, filter); - } - return InternalReadWhile(key, value, upperBounds, filter); - } + return UpperKey.IsLessThan(upperBounds) ? InternalRead(key, value, filter) : InternalReadWhile(key, value, upperBounds, filter); + return ReadWhileCatchAll(key, value, upperBounds, filter); } protected bool ReadWhileCatchAll(TKey key, TValue value, TKey upperBounds, MatchFilterBase filter) { - //If there are no more records in the current node. + // If there are no more records in the current node. if (IndexOfNextKeyValue >= RecordCount) { - //If the last leaf node, return false + // If the last leaf node, return false if (RightSiblingNodeIndex == uint.MaxValue) { key.Clear(); value.Clear(); + Dispose(); + return false; } + LoadNode(RightSiblingNodeIndex); } - //if the pointer data is no longer valid, refresh the pointer + + // If the pointer data is no longer valid, refresh the pointer if (Stream.PointerVersion != PointerVersion) - { RefreshPointer(); - } - //Reads the next key in the sequence. - if (UpperKey.IsLessThan(upperBounds)) - { - return InternalRead(key, value, filter); - } - return InternalReadWhile(key, value, upperBounds, filter); + + // Reads the next key in the sequence. + return UpperKey.IsLessThan(upperBounds) ? InternalRead(key, value, filter) : InternalReadWhile(key, value, upperBounds, filter); } #endregion @@ -366,13 +417,14 @@ protected override bool ReadNext(TKey key, TValue value) { if (Stream.PointerVersion == PointerVersion) { - //A light weight function that can be called quickly since 99% of the time, this logic statement will return successfully. + // A lightweight function that can be called quickly since 99% of the time, this logic statement will return successfully. if (IndexOfNextKeyValue < RecordCount) { InternalRead(key, value); return true; } } + return ReadCatchAll(key, value); } @@ -383,60 +435,64 @@ protected override bool ReadNext(TKey key, TValue value) /// Note: This functionality should be used only to inspect the contests of a file, and not be /// used to attempt supporting reverse readings of files. /// - /// - /// - /// public bool ReadBackwardish(TKey key, TValue value) { - //If there are no more records in the current node. + // If there are no more records in the current node. if (IndexOfNextKeyValue >= RecordCount) { - //If the last leaf node, return false + // If the last leaf node, return false if (LeftSiblingNodeIndex == uint.MaxValue) { key.Clear(); value.Clear(); + Dispose(); + return false; } + LoadNode(LeftSiblingNodeIndex); } - //if the pointer data is no longer valid, refresh the pointer + + // If the pointer data is no longer valid, refresh the pointer if (Stream.PointerVersion != PointerVersion) - { RefreshPointer(); - } - //Reads the next key in the sequence. + + // Reads the next key in the sequence. InternalRead(key, value); + return true; } /// - /// A catch all read function. That can be called if overriding in a derived class. + /// A catch-all read function. That can be called if overriding in a derived class. /// - /// protected bool ReadCatchAll(TKey key, TValue value) { - //If there are no more records in the current node. + // If there are no more records in the current node. if (IndexOfNextKeyValue >= RecordCount) { - //If the last leaf node, return false + // If the last leaf node, return false if (RightSiblingNodeIndex == uint.MaxValue) { key.Clear(); value.Clear(); + Dispose(); + return false; } + LoadNode(RightSiblingNodeIndex); } - //if the pointer data is no longer valid, refresh the pointer + + // If the pointer data is no longer valid, refresh the pointer if (Stream.PointerVersion != PointerVersion) - { RefreshPointer(); - } - //Reads the next key in the sequence. + + // Reads the next key in the sequence. InternalRead(key, value); + return true; } @@ -465,26 +521,29 @@ public override void SeekToKey(TKey key) /// Loads the header data for the provided node. /// /// the node index - /// occurs when - /// is equal to uint.MaxValue + /// occurs when is equal to uint.MaxValue private void LoadNode(uint index) { if (index == uint.MaxValue) - throw new ArgumentNullException("index", "Cannot be uint.MaxValue. Which is null."); + throw new ArgumentNullException(nameof(index), "Cannot be uint.MaxValue. Which is null."); + NodeIndex = index; RefreshPointer(); byte* ptr = Pointer - HeaderSize; + if (ptr[OffsetOfNodeLevel] != m_level) throw new Exception("This node is not supposed to access the underlying node level."); + RecordCount = *(ushort*)(ptr + OffsetOfRecordCount); LeftSiblingNodeIndex = *(uint*)(ptr + OffsetOfLeftSibling); RightSiblingNodeIndex = *(uint*)(ptr + OffsetOfRightSibling); LowerKey.Read(ptr + OffsetOfLowerBounds); UpperKey.Read(ptr + OffsetOfUpperBounds); IndexOfNextKeyValue = 0; - OnNoadReload(); + + OnNodeReload(); } /// @@ -500,7 +559,6 @@ private void RefreshPointer() /// Gets the block index when seeking for the provided key. /// /// the key to start the search from. - /// protected uint FindLeafNodeAddress(TKey key) { return m_lookupKey(key, m_level); @@ -511,9 +569,8 @@ protected uint FindLeafNodeAddress(TKey key) /// Derived classes can override this /// method if fields need to be reset when a node is loaded. /// - protected virtual void OnNoadReload() + protected virtual void OnNodeReload() { } - } } \ No newline at end of file diff --git a/Source/Libraries/Tests/GSF.SortedTreeStore.Test/GSF.SortedTreeStore.Test.csproj b/Source/Libraries/Tests/GSF.SortedTreeStore.Test/GSF.SortedTreeStore.Test.csproj index ea806c4c89..ec36aed4b6 100755 --- a/Source/Libraries/Tests/GSF.SortedTreeStore.Test/GSF.SortedTreeStore.Test.csproj +++ b/Source/Libraries/Tests/GSF.SortedTreeStore.Test/GSF.SortedTreeStore.Test.csproj @@ -115,6 +115,7 @@ + diff --git a/Source/Libraries/Tests/GSF.SortedTreeStore.Test/SortedTreeStore/Services/NearestTimestampTest.cs b/Source/Libraries/Tests/GSF.SortedTreeStore.Test/SortedTreeStore/Services/NearestTimestampTest.cs new file mode 100644 index 0000000000..9a50c769af --- /dev/null +++ b/Source/Libraries/Tests/GSF.SortedTreeStore.Test/SortedTreeStore/Services/NearestTimestampTest.cs @@ -0,0 +1,1091 @@ +//****************************************************************************************************** +// NearestTimestampTest.cs - Gbtc +// +// Copyright © 2026, Grid Protection Alliance. All Rights Reserved. +// +// Licensed to the Grid Protection Alliance (GPA) under one or more contributor license agreements. See +// the NOTICE file distributed with this work for additional information regarding copyright ownership. +// The GPA licenses this file to you under the MIT License (MIT), the "License"; you may +// not use this file except in compliance with the License. You may obtain a copy of the License at: +// +// http://opensource.org/licenses/MIT +// +// Unless agreed to in writing, the subject software distributed under the License is distributed on an +// "AS-IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. Refer to the +// License for the specific language governing permissions and limitations. +// +//****************************************************************************************************** +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using NUnit.Framework; +using GSF.IO; +using GSF.Snap; +using GSF.Snap.Filters; +using GSF.Snap.Services; +using GSF.Snap.Services.Configuration; +using GSF.Snap.Services.Reader; +using GSF.Snap.Services.Writer; +using GSF.Snap.Storage; +using openHistorian.Snap; +using openHistorian.Snap.Definitions; + +namespace GSF.UnitTests.SortedTreeStore.Services +{ + [TestFixture] + public class NearestTimestampTest + { + private sealed class Sample + { + public ulong Time; + public ulong ID; + public ulong Entry; + public ulong Value; + public override string ToString() => $"{Time}:{ID}:{Entry}:{Value}"; + } + + private static void AddArchive(ArchiveList archives, IEnumerable data, bool compressed) + { + SortedTreeFile file = SortedTreeFile.CreateInMemory(); + SortedTreeTable table = file.OpenOrCreateTable( + compressed ? HistorianFileEncodingDefinition.TypeGuid : EncodingDefinition.FixedSizeCombinedEncoding); + + using (SortedTreeTableEditor edit = table.BeginEdit()) + { + foreach (Sample sample in data.OrderBy(s => s.Time).ThenBy(s => s.ID).ThenBy(s => s.Entry)) + edit.AddPoint(new HistorianKey { Timestamp = sample.Time, PointID = sample.ID, EntryNumber = sample.Entry }, + new HistorianValue { Value1 = sample.Value, Value2 = sample.Time, Value3 = sample.ID }); + edit.Commit(); + } + + using (ArchiveListEditor editor = archives.AcquireEditLock()) + editor.Add(table); + } + + private static List Read(ArchiveList archives, + SeekFilterBase filter, MatchFilterBase match = null, + SortedTreeEngineReaderOptions options = null, bool verifyGeneratedPayload = true) + { + List result = new List(); + using SequentialReaderStream reader = new SequentialReaderStream(archives, options, filter, match); + HistorianKey key = new HistorianKey(); + HistorianValue value = new HistorianValue(); + + while (reader.Read(key, value)) + { + if (verifyGeneratedPayload) Assert.AreEqual(key.Timestamp, value.Value2, "Timestamp/value pairing"); + if (verifyGeneratedPayload) Assert.AreEqual(key.PointID, value.Value3, "Quality/value pairing"); + result.Add(new Sample { Time = key.Timestamp, ID = key.PointID, Entry = key.EntryNumber, Value = value.Value1 }); + } + + return result; + } + + // Independent full-resolution oracle: assign each sample to its closest grid target + // (shared midpoint goes to earlier target), then choose the nearest sample per ID. + private static List Expected(List samples, ulong first, ulong last, ulong step, ulong radius, ulong[] ids, bool atOrAfter = false) + { + List result = new List(); + + for (ulong target = first; target <= last; target += step) + { + foreach (ulong id in ids.Distinct()) + { + Sample sample = samples.Where(s => s.ID == id && s.Time >= first && s.Time <= last) + .Where(s => !atOrAfter || s.Time >= target) + .Where(s => (s.Time >= target ? s.Time - target : target - s.Time) <= radius) + .Where(s => target == first || s.Time > target - step / 2 || step % 2 != 0) + .OrderBy(s => s.Time >= target ? s.Time - target : target - s.Time) + .ThenBy(s => s.Time).ThenBy(s => s.Entry).FirstOrDefault(); + + if (sample != null) + result.Add(sample); + } + + if (last - target < step) + break; + } + + return result.OrderBy(s => s.Time).ThenBy(s => s.ID).ThenBy(s => s.Entry).ToList(); + } + + [TestCase(false, false)] + [TestCase(true, false)] + [TestCase(false, true)] + [TestCase(true, true)] + public void SparseOverlappingArchivesMatchFullResolutionOracle(bool compressed, bool atOrAfter) + { + Random random = new Random(47291); + List samples = new List(); + + for (ulong t = 7; t < 20000; t += (ulong)random.Next(1, 14)) + { + for (ulong id = 1; id <= 6; id++) + { + if (random.Next(4) != 0 && !(id == 3 && t > 5000 && t < 15000)) + samples.Add(new Sample { Time = t, ID = id, Value = t * 10 + id }); + } + } + + samples.Add(new Sample { Time = 9950, ID = 8, Value = 11 }); + samples.Add(new Sample { Time = 10050, ID = 8, Value = 12 }); + samples.Add(new Sample { Time = 9950, ID = 8, Entry = 1, Value = 13 }); + + ulong[] ids = { 1, 2, 3, 6, 8, 999, 1 }; + + using ArchiveList archives = new ArchiveList(); + + AddArchive(archives, samples.Where((s, i) => i % 2 == 0), compressed); + AddArchive(archives, samples.Where((s, i) => i % 2 != 0 || i % 11 == 0), compressed); + + NearestTimestampSeekFilter filter = new NearestTimestampSeekFilter(0, 20000, 100, ids, + searchMode: atOrAfter ? TimestampSearchMode.AtOrAfter : TimestampSearchMode.Nearest); + + List expected = Expected(samples, 0, 20000, 100, 50, ids, atOrAfter); + CollectionAssert.AreEqual(expected.Select(s => s.ToString()).ToArray(), Read(archives, filter).Select(s => s.ToString()).ToArray()); + + // A filter can be reused and a scanner can seek again after reaching file boundaries. + CollectionAssert.AreEqual(expected.Select(s => s.ToString()).ToArray(), Read(archives, filter).Select(s => s.ToString()).ToArray()); + } + + [TestCase(false)] + [TestCase(true)] + public void ExactMatchesUseOneSeekPerTargetAndStillResolveArchiveTies(bool compressed) + { + using ArchiveList archives = new ArchiveList(); + List data = new List(); + + for (ulong time = 0; time <= 20; time += 10) + for (ulong id = 1; id <= 2; id++) + data.Add(new Sample { Time = time, ID = id, Entry = 1, Value = time + id }); + + AddArchive(archives, data, compressed); + NearestTimestampSeekFilter filter = new NearestTimestampSeekFilter(0, 20, 10, new ulong[] { 1, 2 }); + + long seeks = Stats.SeeksRequested; + + Assert.AreEqual(6, Read(archives, filter).Count); + Assert.AreEqual(3L, Stats.SeeksRequested - seeks, "Exact timestamps must not trigger backward seeks."); + + AddArchive(archives, data.Select(s => new Sample { Time = s.Time, ID = s.ID, Entry = 0, Value = s.Value + 100 }), compressed); + + List result = Read(archives, filter); + + Assert.AreEqual(6, result.Count); + Assert.IsTrue(result.All(s => s.Entry == 0 && s.Value >= 100), "Other archives still participate after an exact match."); + } + + [Test] + public void PartialExactMatchesStillSearchBackwardsAndForwardOnlyDoesNot() + { + using ArchiveList archives = new ArchiveList(); + + AddArchive(archives, new[] { + new Sample { Time = 90, ID = 1 }, new Sample { Time = 110, ID = 1 }, new Sample { Time = 130, ID = 1 }, + new Sample { Time = 109, ID = 2 }, new Sample { Time = 112, ID = 2 }, new Sample { Time = 129, ID = 2 } + }, true); + + NearestTimestampSeekFilter nearest = new NearestTimestampSeekFilter(90, 130, 20, new ulong[] { 1, 2 }); + + CollectionAssert.AreEqual(new ulong[] { 109, 129 }, Read(archives, nearest).Where(s => s.ID == 2).Select(s => s.Time).ToArray()); + + NearestTimestampSeekFilter forward = new NearestTimestampSeekFilter(90, 130, 20, new ulong[] { 1, 2 }, searchMode: TimestampSearchMode.AtOrAfter); + long seeks = Stats.SeeksRequested; + + CollectionAssert.AreEqual(new ulong[] { 112 }, Read(archives, forward).Where(s => s.ID == 2).Select(s => s.Time).ToArray()); + Assert.AreEqual(3L, Stats.SeeksRequested - seeks, "Forward-only performs no backward seeks, even when IDs are missing."); + Assert.AreEqual(0, Read(archives, TimestampPointIDSeekFilter.FindNearestKey(130, 2, 10, TimestampSearchMode.AtOrAfter)).Count); + Assert.AreEqual(0, Read(archives, TimestampPointIDSeekFilter.FindNearestKey(110, 2, 1, TimestampSearchMode.AtOrAfter)).Count); + } + + [TestCase(false)] + [TestCase(true)] + public void SearchModeRoundTripsWithoutChangingVersionOneNearestPayloads(bool singleTarget) + { + foreach (TimestampSearchMode mode in new[] { TimestampSearchMode.Nearest, TimestampSearchMode.AtOrAfter }) + { + NearestTimestampSeekFilter filter = singleTarget + ? new NearestTimestampSeekFilter(ulong.MaxValue - 10, 7, 15, mode) + : new NearestTimestampSeekFilter(ulong.MaxValue - 20, ulong.MaxValue, 10, new ulong[] { 7 }, searchMode: mode); + using MemoryStream memory = new MemoryStream(); + using BinaryStreamWrapper stream = new BinaryStreamWrapper(memory, false); + + filter.Save(stream); + + byte[] bytes = memory.ToArray(); + + Assert.AreEqual(mode == TimestampSearchMode.Nearest ? 1 : 2, bytes[0]); + + if (mode == TimestampSearchMode.Nearest) + { + Assert.AreEqual(singleTarget ? 26 : 46, bytes.Length, "Version-one layout remains unchanged."); + Assert.AreEqual(singleTarget ? ulong.MaxValue - 10 : ulong.MaxValue - 20, BitConverter.ToUInt64(bytes, 2)); + } + + memory.Position = 0; + NearestTimestampSeekFilter copy = (NearestTimestampSeekFilter)Library.Filters.GetSeekFilter(filter.FilterType, stream); + + Assert.AreEqual(mode, copy.SearchMode); + + while (filter.NextWindow()) + { + Assert.IsTrue(copy.NextWindow()); + Assert.AreEqual(filter.TargetTimestamp, copy.TargetTimestamp); + Assert.AreEqual(filter.StartOfFrame.Timestamp, copy.StartOfFrame.Timestamp); + Assert.AreEqual(filter.EndOfFrame.Timestamp, copy.EndOfFrame.Timestamp); + + if (mode == TimestampSearchMode.AtOrAfter) + Assert.AreEqual(copy.TargetTimestamp, copy.StartOfFrame.Timestamp); + } + + Assert.IsFalse(copy.NextWindow()); + + copy.Reset(); + + Assert.IsTrue(copy.NextWindow()); + } + + Assert.Throws(() => new NearestTimestampSeekFilter(0, 10, 10, new ulong[] { 1 }, searchMode: (TimestampSearchMode)99)); + + using (MemoryStream memory = new MemoryStream(new byte[] { 2, 0, 99 })) + using (BinaryStreamWrapper stream = new BinaryStreamWrapper(memory, false)) + Assert.Throws(() => NearestTimestampSeekFilter.Load(stream)); + } + [Test] + public void BoundariesTiesAndDuplicateEntries() + { + List data = new List + { + new Sample { Time = 0, ID = 1, Value = 10 }, + new Sample { Time = 9, ID = 1, Value = 11 }, + new Sample { Time = 11, ID = 1, Value = 12 }, + new Sample { Time = 9, ID = 1, Entry = 1, Value = 13 }, + new Sample { Time = 15, ID = 2, Value = 14 }, + new Sample { Time = 20, ID = 1, Value = 15 } + }; + + using ArchiveList archives = new ArchiveList(); + + AddArchive(archives, data, true); + + List actual = Read(archives, new NearestTimestampSeekFilter(0, 20, 10, new ulong[] { 1, 2 })); + CollectionAssert.AreEqual(new ulong[] { 0, 9, 15, 20 }, actual.Select(s => s.Time).ToArray()); + + Assert.AreEqual(0UL, actual[1].Entry); + Assert.AreEqual(11UL, actual[1].Value); + Assert.AreEqual(3, Read(archives, new NearestTimestampSeekFilter(0, 20, 10, new ulong[] { 1, 2 }), new OnlyIDOne()).Count); + Assert.AreEqual(2, Read(archives, new NearestTimestampSeekFilter(0, 20, 10, new ulong[] { 1 }, 0)).Count); + Assert.AreEqual(0, Read(archives, new NearestTimestampSeekFilter(0, 20, 10, new ulong[0])).Count); + Assert.AreEqual(0, Read(archives, new NearestTimestampSeekFilter(30, 40, 10, new ulong[] { 1 })).Count); + } + + private sealed class OnlyIDOne : MatchFilterBase + { + public override Guid FilterType => Guid.Empty; + public override bool Contains(HistorianKey key, HistorianValue value) => key.PointID == 1; + public override void Save(BinaryStreamBase stream) { throw new NotSupportedException(); } + } + + [Test] + public void SerializationAndUnsignedLimits() + { + NearestTimestampSeekFilter filter = new NearestTimestampSeekFilter(ulong.MaxValue - 20, ulong.MaxValue, 10, new ulong[] { 2, 1, 1 }); + + using (MemoryStream memory = new MemoryStream()) + using (BinaryStreamWrapper stream = new BinaryStreamWrapper(memory, false)) + { + filter.Save(stream); + memory.Position = 0; + + SeekFilterBase copy = new NearestTimestampSeekFilterDefinition().Create(stream); + + for (int i = 0; i < 3; i++) + { + Assert.IsTrue(filter.NextWindow()); + Assert.IsTrue(copy.NextWindow()); + Assert.AreEqual(filter.StartOfFrame.Timestamp, copy.StartOfFrame.Timestamp); + Assert.AreEqual(filter.EndOfFrame.Timestamp, copy.EndOfFrame.Timestamp); + } + + Assert.IsFalse(copy.NextWindow()); + Assert.IsFalse(copy.NextWindow()); + + copy.Reset(); + + Assert.IsTrue(copy.NextWindow()); + } + + Assert.Throws(() => new NearestTimestampSeekFilter(0, 1, 0, new ulong[] { 1 })); + Assert.Throws(() => new NearestTimestampSeekFilter(1, 0, 1, new ulong[] { 1 })); + Assert.Throws(() => new NearestTimestampSeekFilter(0, 10, 10, new ulong[] { 1 }, 6)); + } + + [Test] + public void LimitsAndCancellationDoNotEmitPartiallySearchedWindows() + { + using ArchiveList archives = new ArchiveList(); + + AddArchive(archives, Enumerable.Range(0, 1000).Select(i => new Sample { Time = (ulong)i, ID = 1 }), false); + + NearestTimestampSeekFilter filter() => new NearestTimestampSeekFilter(0, 1000, 100, new ulong[] { 1 }); + + Assert.AreEqual(2, Read(archives, filter(), null, new SortedTreeEngineReaderOptions(maxReturnedCount: 2)).Count); + Assert.AreEqual(0, Read(archives, filter(), null, new SortedTreeEngineReaderOptions(maxScanCount: 1)).Count); + Assert.AreEqual(1, Read(archives, filter(), null, new SortedTreeEngineReaderOptions(maxSeekCount: 1)).Count); + Assert.AreEqual(0, Read(archives, TimestampPointIDSeekFilter.FindNearestKey(1001, 1, 10), + null, new SortedTreeEngineReaderOptions(maxSeekCount: 1)).Count); + + using SequentialReaderStream reader = new SequentialReaderStream(archives, null, filter()); + + reader.CancelReader(); + + Assert.IsFalse(reader.Read(new HistorianKey(), new HistorianValue())); + } + + [Test] + public void SingleTargetSearchesBeforeAndAfterAndClampsUnsignedLimits() + { + using ArchiveList archives = new ArchiveList(); + + AddArchive(archives, new[] { + new Sample { Time = 90, ID = 1, Value = 1 }, + new Sample { Time = 108, ID = 1, Value = 2 }, + new Sample { Time = 112, ID = 1, Value = 3 }, + new Sample { Time = ulong.MaxValue - 1, ID = 2, Value = 4 } + }, false); + + Assert.AreEqual(90UL, Read(archives, TimestampPointIDSeekFilter.FindNearestKey(95, 1, 10)).Single().Time); + Assert.AreEqual(108UL, Read(archives, TimestampPointIDSeekFilter.FindNearestKey(100, 1, 10)).Single().Time); + Assert.AreEqual(108UL, Read(archives, TimestampPointIDSeekFilter.FindNearestKey(110, 1, 10)).Single().Time); + Assert.AreEqual(0, Read(archives, TimestampPointIDSeekFilter.FindNearestKey(100, 1, 7)).Count); + Assert.AreEqual(0, Read(archives, TimestampPointIDSeekFilter.FindNearestKey(0, 1, 10)).Count); + Assert.AreEqual(ulong.MaxValue - 1, Read(archives, TimestampPointIDSeekFilter.FindNearestKey(ulong.MaxValue, 2, ulong.MaxValue)).Single().Time); + + using MemoryStream memory = new MemoryStream(); + using BinaryStreamWrapper stream = new BinaryStreamWrapper(memory, false); + + TimestampPointIDSeekFilter.FindNearestKey(95, 1, 10).Save(stream); + + memory.Position = 0; + + SeekFilterBase restored = Library.Filters.GetSeekFilter(NearestTimestampSeekFilterDefinition.FilterGuid, stream); + + Assert.AreEqual(90UL, Read(archives, restored).Single().Time); + } + + [Test] + [Category("ArchiveIntegration")] + public void OptionalReadOnlyArchiveMatchesFullResolution() + { + string path = Environment.GetEnvironmentVariable("SNAPDB_TEST_ARCHIVE"); + string selection = Environment.GetEnvironmentVariable("SNAPDB_TEST_POINTIDS"); + + if (string.IsNullOrEmpty(path) || string.IsNullOrEmpty(selection)) + Assert.Ignore("Set SNAPDB_TEST_ARCHIVE and SNAPDB_TEST_POINTIDS for a read-only archive comparison."); + + ulong[] ids = selection.Split(',').Select(ulong.Parse).ToArray(); + + using SortedTreeFile file = SortedTreeFile.OpenFile(path, true); + using ArchiveList archives = new ArchiveList(); + + SortedTreeTable table = file.OpenTable(); + ulong first = table.FirstKey.Timestamp; + ulong last = table.LastKey.Timestamp; + ulong interval = Math.Max(2UL, (last - first) / 60); + + using (ArchiveListEditor editor = archives.AcquireEditLock()) + editor.Add(table); + + SelectedIDs match = new SelectedIDs(ids); + Stopwatch timer = Stopwatch.StartNew(); + List full = Read(archives, TimestampSeekFilter.CreateFromRange(first, last), match, null, false); + + timer.Stop(); + + long fullMilliseconds = timer.ElapsedMilliseconds; + + Assert.Greater(full.Count, 0, "The selected archive must contain the specified measurement IDs."); + + timer.Restart(); + + List nearest = Read(archives, new NearestTimestampSeekFilter(first, last, interval, ids), null, null, false); + + timer.Stop(); + + CollectionAssert.AreEqual(Expected(full, first, last, interval, interval / 2, ids).Select(s => s.ToString()).ToArray(), nearest.Select(s => s.ToString()).ToArray()); + + System.Console.WriteLine("Read-only archive: selected source={0}, returned={1}, full={2} ms, nearest={3} ms", full.Count, nearest.Count, fullMilliseconds, timer.ElapsedMilliseconds); + + ulong[] presentIDs = full.Select(s => s.ID).Distinct().ToArray(); + + System.Console.WriteLine("Present IDs: " + string.Join(",", presentIDs)); + + timer.Restart(); + List present = Read(archives, new NearestTimestampSeekFilter(first, last, interval, presentIDs), null, null, false); + + timer.Stop(); + + CollectionAssert.AreEqual(nearest.Select(s => s.ToString()).ToArray(), present.Select(s => s.ToString()).ToArray()); + + System.Console.WriteLine("Read-only archive (present IDs only): {0} ms", timer.ElapsedMilliseconds); + } + + + + private sealed class DiskSamples : TreeStream + { + private readonly IEnumerator m_samples; + internal DiskSamples(IEnumerable samples) { m_samples = samples.GetEnumerator(); } + public override bool IsAlwaysSequential => true; + public override bool NeverContainsDuplicates => true; + + protected override bool ReadNext(HistorianKey key, HistorianValue value) + { + if (!m_samples.MoveNext()) return false; + Sample sample = m_samples.Current; + key.Timestamp = sample.Time; key.PointID = sample.ID; key.EntryNumber = sample.Entry; + value.Value1 = sample.Value; value.Value2 = sample.Time; value.Value3 = sample.ID; + return true; + } + protected override void Dispose(bool disposing) { if (disposing) m_samples.Dispose(); base.Dispose(disposing); } + } + + private static void WriteIndexedArchive(string path, IEnumerable samples) + { + using DiskSamples stream = new DiskSamples(samples); + SortedTreeFileSimpleWriter.Create(path + ".pending", path, 4096, null, HistorianFileEncodingDefinition.TypeGuid, stream); + } + + private static List ReadDiskArchives(string[] paths, out long visits, SortedTreeEngineReaderOptions options = null) + { + using ArchiveList archives = new ArchiveList(); + + foreach (string path in paths) + { + SortedTreeFile file = SortedTreeFile.OpenFile(path, true); + using ArchiveListEditor editor = archives.AcquireEditLock(); + + editor.Add(file.OpenTable()); + } + + long before = Stats.PointsScanned; + List result = Read(archives, new NearestTimestampSeekFilter(0, 999, 100, new ulong[] { 1, 2, 999 }), null, options); + + visits = Stats.PointsScanned - before; + + return result; + } + + [Test] + public void FileIDIndexPrunesMixedRequestsAndFailsOpen() + { + string folder = Path.Combine(Path.GetTempPath(), "nearest-index-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(folder); + string first = Path.Combine(folder, "one.d2"); + string second = Path.Combine(folder, "two.d2i"); + + try + { + WriteIndexedArchive(first, Enumerable.Range(0, 1000).Select(t => new Sample { Time = (ulong)t, ID = 1 })); + WriteIndexedArchive(second, Enumerable.Range(0, 1000).Select(t => new Sample { Time = (ulong)t, ID = 2 })); + + Assert.IsTrue(File.Exists(first + ".pidx")); + Assert.IsFalse(File.Exists(first + ".bidx"), "Bloom generation is opt-in."); + + byte[] index = File.ReadAllBytes(first + ".pidx"); + + List indexed = ReadDiskArchives(new[] { first, second }, out long indexedVisits); + + Assert.AreEqual(20, indexed.Count, "An ID absent in one file remains eligible in another."); + + List bypass = ReadDiskArchives(new[] { first, second }, out long bypassVisits, new SortedTreeEngineReaderOptions(false, false)); + + CollectionAssert.AreEqual(indexed.Select(sample => sample.ToString()).ToArray(), bypass.Select(sample => sample.ToString()).ToArray()); + Assert.Greater(bypassVisits, indexedVisits * 10); + + File.Delete(first + ".pidx"); + File.Delete(second + ".pidx"); + + List fallback = ReadDiskArchives(new[] { first, second }, out long fallbackVisits); + + CollectionAssert.AreEqual(fallback.Select(s => s.ToString()).ToArray(), indexed.Select(s => s.ToString()).ToArray()); + Assert.Less(indexedVisits * 10, fallbackVisits); + + System.Console.WriteLine("File presence index: indexed visits={0}, fallback visits={1}, one-ID sidecar={2} bytes", indexedVisits, fallbackVisits, index.Length); + + // A valid index belonging to a different archive cannot exclude that file's IDs. + File.WriteAllBytes(second + ".pidx", index); + + CollectionAssert.AreEqual(fallback.Select(s => s.ToString()).ToArray(), ReadDiskArchives(new[] { first, second }, out fallbackVisits).Select(s => s.ToString()).ToArray()); + + index[0] ^= 1; // Digest corruption must fall back, never return false negatives. + File.WriteAllBytes(first + ".pidx", index); + + CollectionAssert.AreEqual(fallback.Select(s => s.ToString()).ToArray(), ReadDiskArchives(new[] { first, second }, out fallbackVisits).Select(s => s.ToString()).ToArray()); + + File.WriteAllBytes(first + ".pidx", Enumerable.Repeat((byte)255, 128).ToArray()); + + Assert.AreEqual(20, ReadDiskArchives(new[] { first, second }, out fallbackVisits).Count); + + File.WriteAllBytes(first + ".pidx", new byte[] { 1, 2, 3 }); + + Assert.AreEqual(20, ReadDiskArchives(new[] { first, second }, out fallbackVisits).Count); + } + finally + { + foreach (string path in Directory.GetFiles(folder)) File.Delete(path); + Directory.Delete(folder); + } + } + + [Test] + public void FileIDIndexValidatesSnapshotAndFollowsArchiveLifecycle() + { + string folder = Path.Combine(Path.GetTempPath(), "nearest-index-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(folder); + string path = Path.Combine(folder, "one.d2i"); + + try + { + WriteIndexedArchive(path, Enumerable.Range(0, 10000).Select(id => new Sample { Time = 1, ID = (ulong)id })); + + using (SortedTreeFile file = SortedTreeFile.OpenFile(path, true)) + { + Guid archive = file.Snapshot.Header.ArchiveId; + uint sequence = file.Snapshot.Header.SnapshotSequenceNumber; + Guid key = new HistorianKey().GenericTypeGuid; + Guid value = new HistorianValue().GenericTypeGuid; + + Assert.AreEqual(10000, ArchivePointIDIndex.Load(path, archive, sequence, key, value).Length); + Assert.IsNull(ArchivePointIDIndex.Load(path, archive, sequence + 1, key, value)); + Assert.IsNull(ArchivePointIDIndex.Load(path, archive, sequence, Guid.NewGuid(), value)); + + long bytes = new FileInfo(path + ".pidx").Length; + + Assert.Less(bytes, 80000); + + System.Console.WriteLine("10,000-ID compressed sidecar: {0} bytes", bytes); + + file.ChangeExtension(".d2", true, true); + + Assert.IsFalse(File.Exists(path + ".pidx")); + Assert.IsTrue(File.Exists(file.FilePath + ".pidx")); + Assert.AreEqual(10000, ArchivePointIDIndex.Load(file.FilePath, archive, sequence, key, value).Length); + + path = file.FilePath; + file.Delete(); + + Assert.IsFalse(File.Exists(path)); + Assert.IsFalse(File.Exists(path + ".pidx")); + } + + // A sidecar publication conflict must not invalidate a successfully written archive. + Directory.CreateDirectory(path + ".pidx"); + + WriteIndexedArchive(path, new[] { new Sample { Time = 1, ID = 1 } }); + + using (SortedTreeFile file = SortedTreeFile.OpenFile(path, true)) + Assert.AreEqual(1UL, file.OpenTable().FirstKey.PointID); + + Directory.Delete(path + ".pidx"); + } + finally + { + foreach (string entry in Directory.GetFiles(folder)) File.Delete(entry); + foreach (string entry in Directory.GetDirectories(folder)) Directory.Delete(entry); + Directory.Delete(folder); + } + } + + + [Test] + public void BloomMembershipHasNoFalseNegativesAndBoundsFalsePositives() + { + Assert.Throws(() => new TimeBucketBloomFilterSettings(true, 0)); + Assert.Throws(() => new TimeBucketBloomFilterSettings(true, 900, double.NaN)); + + ArchiveTimeBucketBloomIndex.Builder builder = new ArchiveTimeBucketBloomIndex.Builder(new TimeBucketBloomFilterSettings(true, 1)); + + const ulong Second = TimeSpan.TicksPerSecond; + + for (ulong id = 0; id < 4000; id++) + builder.Add(Second, id * 7919); + + builder.Add(ulong.MaxValue, ulong.MaxValue); + + ArchiveTimeBucketBloomIndex index = builder.Complete(); + + for (ulong id = 0; id < 4000; id++) + Assert.IsTrue(index.MayContain(id * 7919, Second, Second)); + + Assert.IsTrue(index.MayContain(ulong.MaxValue, ulong.MaxValue, ulong.MaxValue)); + Assert.IsFalse(index.MayContain(0, 0, Second - 1)); + Assert.IsFalse(index.MayContain(0, 2 * Second, 3 * Second)); + + int positives = 0; + + for (ulong id = 1; id <= 10000; id++) + { + if (index.MayContain(id * 7919 + 1, Second, Second)) + positives++; + } + + Assert.Less(positives, 300, "False-positive performance regression (not a correctness guarantee)."); + + System.Console.WriteLine("Bloom absent-ID probes: {0}/10000 false positives", positives); + + // A construction limit must drop the whole synopsis, not omit populated buckets. + builder = new ArchiveTimeBucketBloomIndex.Builder(new TimeBucketBloomFilterSettings(true, 1)); + + for (ulong bucket = 0; bucket <= ArchiveTimeBucketBloomIndex.MaximumBuckets; bucket++) + builder.Add(bucket * Second, 1); + + Assert.IsNull(builder.Complete()); + } + + [TestCase(false)] + [TestCase(true)] + public void BloomSidecarPrunesInternalGapsAndFailsOpen(bool forwardOnly) + { + string folder = Path.Combine(Path.GetTempPath(), "nearest-bloom-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(folder); + string path = Path.Combine(folder, "gap.d2"); + const ulong Second = TimeSpan.TicksPerSecond; + + try + { + List samples = Enumerable.Range(0, 10000).Select(t => new Sample + { + Time = (ulong)t * Second, + ID = 1 + }).ToList(); + + samples.Add(new Sample + { + Time = 0, + ID = 2 + }); + samples.Add(new Sample + { + Time = 9999 * Second, + ID = 2 + }); + samples = samples.OrderBy(s => s.Time).ThenBy(s => s.ID).ToList(); + + using (DiskSamples stream = new DiskSamples(samples)) + SortedTreeFileSimpleWriter.CreateWithBloomFilter(path + ".pending", path, 4096, null, HistorianFileEncodingDefinition.TypeGuid, stream, new TimeBucketBloomFilterSettings(true, 10)); + + Assert.IsTrue(File.Exists(path + ".bidx")); + + byte[] bytes = File.ReadAllBytes(path + ".bidx"); + long[] visits = new long[6]; + + for (int pass = 0; pass < 6; pass++) + { + switch (pass) + { + case 1: + File.Delete(path + ".bidx"); + break; + case 2: + File.WriteAllBytes(path + ".bidx", Enumerable.Repeat((byte)255, 128).ToArray()); + break; + case 3: + case 4: + bytes[0] ^= 1; + File.WriteAllBytes(path + ".bidx", bytes); + break; + } + + using SortedTreeFile file = SortedTreeFile.OpenFile(path, true); + using ArchiveList archives = new ArchiveList(); + using (ArchiveListEditor editor = archives.AcquireEditLock()) + editor.Add(file.OpenTable()); + + long before = Stats.PointsScanned; + + List result = Read(archives, new NearestTimestampSeekFilter(5000 * Second, 5100 * Second, 100 * Second, new ulong[] { 1, 2 }, null, forwardOnly ? TimestampSearchMode.AtOrAfter : TimestampSearchMode.Nearest), null, new SortedTreeEngineReaderOptions(pass != 5, pass != 4)); + + visits[pass] = Stats.PointsScanned - before; + + Assert.AreEqual(2, result.Count); + Assert.IsTrue(result.All(sample => sample.ID == 1)); + } + + Assert.Less(visits[0] * 10, visits[1]); + Assert.AreEqual(visits[1], visits[2]); + Assert.AreEqual(visits[1], visits[3]); + Assert.AreEqual(visits[1], visits[4], "Bloom bypass uses ordinary scanning despite a valid sidecar."); + Assert.AreEqual(visits[0], visits[5], "Bloom remains usable when exact-ID indexing is bypassed."); + + System.Console.WriteLine("Bloom internal gap ({0}): indexed visits={1}, fallback={2}, sidecar={3} bytes", forwardOnly, visits[0], visits[1], bytes.Length); + } + finally + { + foreach (string entry in Directory.GetFiles(folder)) + File.Delete(entry); + + Directory.Delete(folder); + } + } + + [Test] + public void BloomDiskOracleIdentityAndLifecycle() + { + string folder = Path.Combine(Path.GetTempPath(), "nearest-bloom-oracle-" + Guid.NewGuid().ToString("N")); + + Directory.CreateDirectory(folder); + + string path = Path.Combine(folder, "oracle.d2i"); + const ulong Second = TimeSpan.TicksPerSecond; + + List data = new List(); + Random random = new Random(8182); + + for (ulong time = 0; time < 200 * Second; time += 3 * Second + 1) + { + for (ulong id = 0; id < 10; id++) + { + if (random.Next(3) == 0) + data.Add(new Sample { Time = time, ID = id }); + } + } + + data.Add(new Sample { Time = 200 * Second, ID = 9 }); + + try + { + using (DiskSamples stream = new DiskSamples(data)) + SortedTreeFileSimpleWriter.CreateWithBloomFilter(path + ".pending", path, 4096, null, HistorianFileEncodingDefinition.TypeGuid, stream, new TimeBucketBloomFilterSettings(true, 10)); + + using (SortedTreeFile file = SortedTreeFile.OpenFile(path, true)) + using (ArchiveList archives = new ArchiveList()) + { + using (ArchiveListEditor editor = archives.AcquireEditLock()) editor.Add(file.OpenTable()); + ulong[] ids = Enumerable.Range(0, 11).Select(id => (ulong)id).ToArray(); + + foreach (bool forward in new[] { false, true }) + { + List actual = Read(archives, new NearestTimestampSeekFilter(0, 200 * Second, 20 * Second, ids, null, forward ? TimestampSearchMode.AtOrAfter : TimestampSearchMode.Nearest)); + CollectionAssert.AreEqual(Expected(data, 0, 200 * Second, 20 * Second, 10 * Second, ids, forward).Select(s => s.ToString()).ToArray(), actual.Select(s => s.ToString()).ToArray()); + } + + Guid archive = file.Snapshot.Header.ArchiveId; + uint seq = file.Snapshot.Header.SnapshotSequenceNumber; + Guid key = new HistorianKey().GenericTypeGuid, value = new HistorianValue().GenericTypeGuid; + + Assert.IsNull(ArchiveTimeBucketBloomIndex.Load(path, Guid.NewGuid(), seq, key, value)); + Assert.IsNull(ArchiveTimeBucketBloomIndex.Load(path, archive, seq + 1, key, value)); + Assert.IsNull(ArchiveTimeBucketBloomIndex.Load(path, archive, seq, Guid.NewGuid(), value)); + + ArchiveTimeBucketBloomIndex index = ArchiveTimeBucketBloomIndex.Load(path, archive, seq, key, value); + + foreach (Sample sample in data) + Assert.IsTrue(index.MayContain(sample.ID, sample.Time, sample.Time)); + } + + using (SortedTreeFile file = SortedTreeFile.OpenFile(path, true)) + { + file.ChangeExtension(".d2", true, true); + Assert.IsFalse(File.Exists(path + ".bidx")); + + path = file.FilePath; + Assert.IsTrue(File.Exists(path + ".bidx")); + + file.Delete(); + Assert.IsFalse(File.Exists(path + ".bidx")); + } + } + finally { foreach (string entry in Directory.GetFiles(folder)) File.Delete(entry); Directory.Delete(folder); } + } + + [Test] + public void BloomSettingsRoundTripAndReachEveryRollover() + { + AdvancedServerDatabaseConfig config = new AdvancedServerDatabaseConfig("test", Path.GetTempPath(), true) + { + BloomFilter = new TimeBucketBloomFilterSettings(true, 120, 0.001) + }; + + ServerDatabaseSettings server = config.ToServerDatabaseSettings(); + SimplifiedArchiveInitializerSettings settings = server.WriteProcessor.FirstStageWriter.FinalSettings; + + Assert.IsTrue(settings.BloomFilter.Enabled); + + foreach (CombineFilesSettings rollover in server.WriteProcessor.StagingRollovers) + Assert.AreEqual(120, rollover.ArchiveSettings.BloomFilter.BucketDurationSeconds); + + using MemoryStream stream = new MemoryStream(); + + settings.Save(stream); + stream.Position = 0; + + SimplifiedArchiveInitializerSettings restored = new SimplifiedArchiveInitializerSettings(); + restored.Load(stream); + + Assert.IsTrue(restored.BloomFilter.Enabled); + Assert.AreEqual(120, restored.BloomFilter.BucketDurationSeconds); + Assert.AreEqual(0.001, restored.BloomFilter.FalsePositiveProbability); + Assert.AreEqual(stream.Length, stream.Position); + + stream.SetLength(0); + settings.BloomFilter = new TimeBucketBloomFilterSettings(); + settings.Save(stream); + stream.Position = 0; + restored.Load(stream); + + Assert.IsFalse(restored.BloomFilter.Enabled); + Assert.AreEqual(stream.Length, stream.Position); + } + + [Test] + public void BloomSettingsPreserveLegacyVersionsAndDirectoryFillMode() + { + SimplifiedArchiveInitializerSettings settings = new SimplifiedArchiveInitializerSettings { FillMethod = ArchiveDirectoryFillMethod.RoundRobin }; + + using MemoryStream stream = new MemoryStream(); + + settings.Save(stream); + + byte[] versionThree = stream.ToArray(); + Assert.AreEqual(3, versionThree[0]); + + SimplifiedArchiveInitializerSettings loaded = new SimplifiedArchiveInitializerSettings(); + + stream.Position = 0; + loaded.Load(stream); + + Assert.AreEqual(ArchiveDirectoryFillMethod.RoundRobin, loaded.FillMethod); + Assert.IsFalse(loaded.BloomFilter.Enabled); + Assert.AreEqual(stream.Length, stream.Position); + + // Version 2 lacks the trailing index settings: bool + int + double + bool + int + byte[] versionTwo = versionThree.Take(versionThree.Length - 18).ToArray(); + + versionTwo[0] = 2; + + using (MemoryStream old = new MemoryStream(versionTwo)) loaded.Load(old); + + Assert.AreEqual(ArchiveDirectoryFillMethod.RoundRobin, loaded.FillMethod); + Assert.IsTrue(loaded.EnablePointIDIndex); + Assert.IsFalse(loaded.BloomFilter.Enabled); + + byte[] versionOne = versionTwo.Take(5).Concat(versionTwo.Skip(9)).ToArray(); + + versionOne[0] = 1; + + using (MemoryStream old = new MemoryStream(versionOne)) loaded.Load(old); + + Assert.AreEqual(ArchiveDirectoryFillMethod.Sequential, loaded.FillMethod); + Assert.IsFalse(loaded.BloomFilter.Enabled); + + stream.SetLength(0); + + settings.BloomFilter = new TimeBucketBloomFilterSettings(true); + settings.Save(stream); + + Assert.AreEqual(3, stream.ToArray()[0]); + stream.Position = 0; + loaded.Load(stream); + + Assert.AreEqual(ArchiveDirectoryFillMethod.RoundRobin, loaded.FillMethod); + Assert.IsTrue(loaded.BloomFilter.Enabled); + } + + [Test] + public void IndexControlsRoundTripAndBudgetIsEnforced() + { + Assert.Throws(() => new TimeBucketBloomFilterSettings(maximumIndexSizeMiB: 0)); + Assert.Throws(() => new TimeBucketBloomFilterSettings(maximumIndexSizeMiB: 65)); + + foreach (bool exact in new[] { false, true }) + { + foreach (bool bloom in new[] { false, true }) + { + using MemoryStream memory = new MemoryStream(); + using BinaryStreamWrapper stream = new BinaryStreamWrapper(memory, false); + SortedTreeEngineReaderOptions original = new SortedTreeEngineReaderOptions(exact, bloom, TimeSpan.FromSeconds(2), 3, 4, 5); + + original.Save(stream); + + Assert.AreEqual(exact && bloom ? 0 : 1, memory.ToArray()[0]); + memory.Position = 0; + + SortedTreeEngineReaderOptions restored = new SortedTreeEngineReaderOptions(stream); + + Assert.AreEqual(exact, restored.UsePointIDIndex); + Assert.AreEqual(bloom, restored.UseTimeBucketBloomFilter); + Assert.AreEqual(original.Timeout, restored.Timeout); + Assert.AreEqual(3, restored.MaxReturnedCount); + Assert.AreEqual(4, restored.MaxScanCount); + Assert.AreEqual(5, restored.MaxSeekCount); + Assert.AreEqual(memory.Length, memory.Position); + } + } + + AdvancedServerDatabaseConfig config = new AdvancedServerDatabaseConfig("test", Path.GetTempPath(), true) + { + EnablePointIDIndex = false, + BloomFilter = new TimeBucketBloomFilterSettings(true, 1, 0.01, 1) + }; + + ServerDatabaseSettings server = config.ToServerDatabaseSettings(); + SimplifiedArchiveInitializerSettings settings = server.WriteProcessor.FirstStageWriter.FinalSettings; + + Assert.IsFalse(settings.EnablePointIDIndex); + + foreach (CombineFilesSettings rollover in server.WriteProcessor.StagingRollovers) + { + Assert.IsFalse(rollover.ArchiveSettings.EnablePointIDIndex); + Assert.AreEqual(1, rollover.ArchiveSettings.BloomFilter.MaximumIndexSizeMiB); + } + + using (MemoryStream memory = new MemoryStream()) + { + settings.Save(memory); + memory.Position = 0; + + SimplifiedArchiveInitializerSettings restored = new SimplifiedArchiveInitializerSettings(); + + restored.Load(memory); + + Assert.IsFalse(restored.EnablePointIDIndex); + Assert.AreEqual(1, restored.BloomFilter.MaximumIndexSizeMiB); + Assert.IsTrue(restored.BloomFilter.Enabled); + Assert.AreEqual(memory.Length, memory.Position); + } + + ArchiveTimeBucketBloomIndex.Builder small = new ArchiveTimeBucketBloomIndex.Builder(new TimeBucketBloomFilterSettings(true, 1, 0.01, 1)); + ArchiveTimeBucketBloomIndex.Builder larger = new ArchiveTimeBucketBloomIndex.Builder(new TimeBucketBloomFilterSettings(true, 1, 0.01, 2)); + + for (ulong bucket = 0; bucket < 600; bucket++) + { + for (ulong id = 0; id < 1000; id++) + { + small.Add(bucket * TimeSpan.TicksPerSecond, id); + larger.Add(bucket * TimeSpan.TicksPerSecond, id); + } + } + + Assert.IsNull(small.Complete(), "An exhausted budget must drop the complete index."); + Assert.IsNotNull(larger.Complete()); + } + + [TestCase(false, false)] + [TestCase(false, true)] + [TestCase(true, false)] + [TestCase(true, true)] + public void IndexGenerationSwitchesAreIndependent(bool exact, bool bloom) + { + string folder = Path.Combine(Path.GetTempPath(), "index-controls-" + Guid.NewGuid().ToString("N")); + + Directory.CreateDirectory(folder); + + try + { + SimplifiedArchiveInitializerSettings settings = new SimplifiedArchiveInitializerSettings(); + + settings.ConfigureOnDisk(new[] { folder }, 100 * 1024 * 1024, ArchiveDirectoryMethod.TopDirectoryOnly, HistorianFileEncodingDefinition.TypeGuid, "test", ".pending", ".d2"); + settings.EnablePointIDIndex = exact; + settings.BloomFilter = new TimeBucketBloomFilterSettings(bloom, maximumIndexSizeMiB: 1); + + SimplifiedArchiveInitializer initializer = new SimplifiedArchiveInitializer(settings); + using DiskSamples stream = new DiskSamples(new[] + { + new Sample + { + Time = 20, + ID = 42 + } + }); + SortedTreeTable table = initializer.CreateArchiveFile(new HistorianKey { Timestamp = 20 }, new HistorianKey { Timestamp = 20 }, 0, stream, null); + using SortedTreeFile file = table.BaseFile; + + Assert.AreEqual(exact, File.Exists(file.FilePath + ".pidx")); + Assert.AreEqual(bloom, File.Exists(file.FilePath + ".bidx")); + Assert.AreEqual(42UL, table.FirstKey.PointID); + } + finally + { + foreach (string entry in Directory.GetFiles(folder)) + File.Delete(entry); Directory.Delete(folder); + } + } + + private sealed class SelectedIDs : MatchFilterBase + { + private readonly HashSet m_ids; + public SelectedIDs(IEnumerable ids) { m_ids = new HashSet(ids); } + public override Guid FilterType => Guid.Empty; + public override bool Contains(HistorianKey key, HistorianValue value) => m_ids.Contains(key.PointID); + public override void Save(BinaryStreamBase stream) { throw new NotSupportedException(); } + } + + [Test] + public void GeneratedYearSkipsInterveningSamples() + { + const ulong Minute = TimeSpan.TicksPerMinute; + const ulong Interval = 240 * Minute; + const ulong End = 365 * 24 * 60 * Minute; + + List data = new List(); + + for (ulong t = 0; t < End; t += 5 * Minute) + { + for (ulong id = 1; id <= 3; id++) + data.Add(new Sample { Time = t + id * 7 * TimeSpan.TicksPerSecond, ID = id, Value = t + id }); + } + + using ArchiveList archives = new ArchiveList(); + AddArchive(archives, data, true); + + // Reproduce the original Grafana failure with a 1 ms interval tolerance. + Assert.AreEqual(0, Read(archives, TimestampSeekFilter.CreateFromIntervalData(Interval, End - 1, Interval, TimeSpan.TicksPerMillisecond)).Count); + + long before = Stats.PointsScanned; + Stopwatch timer = Stopwatch.StartNew(); + List result = Read(archives, new NearestTimestampSeekFilter(0, End - 1, Interval, new ulong[] { 1, 2, 3 })); + + timer.Stop(); + + long visited = Stats.PointsScanned - before; + + Assert.AreEqual(2190 * 3, result.Count); + Assert.Less(visited, data.Count / 5, "Must skip the data between targets, not perform a full scan."); + + foreach (Sample sample in result) + Assert.AreEqual(sample.ID * 7 * TimeSpan.TicksPerSecond, sample.Time % Interval); + + System.Console.WriteLine("Nearest year: source={0}, returned={1}, visited={2}, elapsed={3} ms", data.Count, result.Count, visited, timer.ElapsedMilliseconds); + + if (Environment.GetEnvironmentVariable("SNAPDB_NEAREST_BENCHMARK") != "1") + return; + + // Keep test-framework assertions and result materialization out of the timed + // section; NUnit 2 and NUnit 3 have different per-assertion overheads. + Diagnostics.Logger.Console.Verbose = Diagnostics.VerboseLevel.None; + double[] milliseconds = new double[100]; + + for (int iteration = 0; iteration < milliseconds.Length; iteration++) + { + int count = 0; + HistorianKey key = new HistorianKey(); + HistorianValue value = new HistorianValue(); + timer.Restart(); + + using (SequentialReaderStream reader = new SequentialReaderStream(archives, null, new NearestTimestampSeekFilter(0, End - 1, Interval, new ulong[] { 1, 2, 3 }))) + while (reader.Read(key, value)) count++; + + timer.Stop(); + milliseconds[iteration] = timer.Elapsed.TotalMilliseconds; + + Assert.AreEqual(6570, count); + } + + System.Console.WriteLine("Engine-only query milliseconds (100 runs): " + string.Join(", ", milliseconds.Select(ms => ms.ToString("F2")))); + System.Console.WriteLine("Engine-only final-10 median: {0:F2} ms; 64-bit={1}; CLR={2}", milliseconds.Skip(milliseconds.Length - 10).OrderBy(ms => ms).ElementAt(5), Environment.Is64BitProcess, Environment.Version); + } + } +} diff --git a/Source/Libraries/openHistorian.Core/Net/HistorianServerDatabaseConfig.cs b/Source/Libraries/openHistorian.Core/Net/HistorianServerDatabaseConfig.cs index 21481400d2..d0d2a190e7 100755 --- a/Source/Libraries/openHistorian.Core/Net/HistorianServerDatabaseConfig.cs +++ b/Source/Libraries/openHistorian.Core/Net/HistorianServerDatabaseConfig.cs @@ -21,6 +21,9 @@ // 10/15/2019 - J. Ritchie Carroll // Added DesiredRemainingSpace property for configurable target disk remaining space for // final staging files / general code cleanup. +// 09/22/2026 - J. Ritchie Carroll +// Added EnablePointIDIndex and BloomFilter properties for optional per-file PointID index +// and time-bucket Bloom sidecar support. // //****************************************************************************************************** @@ -29,182 +32,193 @@ using GSF.Snap; using GSF.Snap.Services; using GSF.Snap.Services.Configuration; +using GSF.Snap.Storage; using GSF.Units; using openHistorian.Snap; using openHistorian.Snap.Definitions; -namespace openHistorian.Net +namespace openHistorian.Net; + +/// +/// Creates a configuration for the database to utilize. +/// +public class HistorianServerDatabaseConfig : IToServerDatabaseSettings { + private readonly AdvancedServerDatabaseConfig m_config; + /// - /// Creates a configuration for the database to utilize. + /// Controls exact per-file ID index generation on finalized archives. /// - public class HistorianServerDatabaseConfig - : IToServerDatabaseSettings + public bool EnablePointIDIndex { - private readonly AdvancedServerDatabaseConfig m_config; + get => m_config.EnablePointIDIndex; + set => m_config.EnablePointIDIndex = value; + } - /// - /// Gets a database config. - /// - public HistorianServerDatabaseConfig(string databaseName, string mainPath, bool supportsWriting) - { - m_config = new AdvancedServerDatabaseConfig(databaseName, mainPath, supportsWriting) - { - ArchiveEncodingMethod = HistorianFileEncodingDefinition.TypeGuid - }; - - m_config.StreamingEncodingMethods.Add(HistorianStreamEncodingDefinition.TypeGuid); - m_config.StreamingEncodingMethods.Add(EncodingDefinition.FixedSizeCombinedEncoding); - m_config.IntermediateFileExtension = ".d2i"; - m_config.FinalFileExtension = ".d2"; - m_config.DirectoryMethod = ArchiveDirectoryMethod.YearThenMonth; - } + /// + /// Controls optional time-bucket Bloom sidecars for finalized archives. + /// + public TimeBucketBloomFilterSettings BloomFilter + { + get => m_config.BloomFilter; + set => m_config.BloomFilter = value; + } - /// - /// Specify how archive files will be written into the final directory. - /// - public ArchiveDirectoryMethod DirectoryMethod + /// + /// Gets a database config. + /// + public HistorianServerDatabaseConfig(string databaseName, string mainPath, bool supportsWriting) + { + m_config = new AdvancedServerDatabaseConfig(databaseName, mainPath, supportsWriting) { - get => m_config.DirectoryMethod; - set => m_config.DirectoryMethod = value; - } + ArchiveEncodingMethod = HistorianFileEncodingDefinition.TypeGuid + }; + + m_config.StreamingEncodingMethods.Add(HistorianStreamEncodingDefinition.TypeGuid); + m_config.StreamingEncodingMethods.Add(EncodingDefinition.FixedSizeCombinedEncoding); + m_config.IntermediateFileExtension = ".d2i"; + m_config.FinalFileExtension = ".d2"; + m_config.DirectoryMethod = ArchiveDirectoryMethod.YearThenMonth; + } - /// - /// Specify how a write path is selected from the final archive directories when more than one is configured. - /// - public ArchiveDirectoryFillMethod FillMethod - { - get => m_config.FillMethod; - set => m_config.FillMethod = value; - } + /// + /// Specify how archive files will be written into the final directory. + /// + public ArchiveDirectoryMethod DirectoryMethod + { + get => m_config.DirectoryMethod; + set => m_config.DirectoryMethod = value; + } - /// - /// Gets or sets the desired size of the final stage archive files. - /// - /// Must be between 100MB and 1TB - public long TargetFileSize - { - get => m_config.TargetFileSize; - set - { - if (value < 100 * SI2.Mega) - throw new ArgumentOutOfRangeException(nameof(value), "Target file size must be between 100MB and 1TB"); + /// + /// Specify how a write path is selected from the final archive directories when more than one is configured. + /// + public ArchiveDirectoryFillMethod FillMethod + { + get => m_config.FillMethod; + set => m_config.FillMethod = value; + } - if (value > SI2.Tera) - throw new ArgumentOutOfRangeException(nameof(value), "Target file size must be between 100MB and 1TB"); + /// + /// Gets or sets the desired size of the final stage archive files. + /// + /// Must be between 100MB and 1TB + public long TargetFileSize + { + get => m_config.TargetFileSize; + set + { + if (value is < 100 * SI2.Mega or > SI2.Tera) + throw new ArgumentOutOfRangeException(nameof(value), "Target file size must be between 100MB and 1TB"); - m_config.TargetFileSize = value; - } + m_config.TargetFileSize = value; } + } - /// - /// Gets or sets the desired remaining drive space, in bytes, for final stage files. - /// - /// Must be between 100MB and 1TB - public long DesiredRemainingSpace + /// + /// Gets or sets the desired remaining drive space, in bytes, for final stage files. + /// + /// Must be between 100MB and 1TB + public long DesiredRemainingSpace + { + get => m_config.DesiredRemainingSpace; + set { - get => m_config.DesiredRemainingSpace; - set - { - if (value < 100 * SI2.Mega) - throw new ArgumentOutOfRangeException(nameof(value), "Desired remaining space must be between 100MB and 1TB"); - - if (value > SI2.Tera) - throw new ArgumentOutOfRangeException(nameof(value), "Desired remaining space must be between 100MB and 1TB"); + if (value is < 100 * SI2.Mega or > SI2.Tera) + throw new ArgumentOutOfRangeException(nameof(value), "Desired remaining space must be between 100MB and 1TB"); - m_config.DesiredRemainingSpace = value; - } + m_config.DesiredRemainingSpace = value; } + } - /// - /// The number of milliseconds before data is automatically flushed to the disk. - /// - /// - /// Must be between 1,000 ms and 60,000 ms. - /// - public int DiskFlushInterval + /// + /// The number of milliseconds before data is automatically flushed to the disk. + /// + /// + /// Must be between 1,000 ms and 60,000 ms. + /// + public int DiskFlushInterval + { + get => m_config.DiskFlushInterval; + set { - get => m_config.DiskFlushInterval; - set - { - if (value < 1000 || value > 60000) - throw new ArgumentOutOfRangeException(nameof(value), "Must be between 1,000 ms and 60,000 ms."); - - m_config.DiskFlushInterval = value; - } - } + if (value is < 1000 or > 60000) + throw new ArgumentOutOfRangeException(nameof(value), "Must be between 1,000 ms and 60,000 ms."); - /// - /// The number of milliseconds before data is taken from its cache and put in the - /// memory file. - /// - /// - /// Must be between 1 and 1,000 - /// - public int CacheFlushInterval - { - get => m_config.CacheFlushInterval; - set - { - if (value < 1 || value > 1000) - throw new ArgumentOutOfRangeException(nameof(value), "Must be between 1 and 1,000"); - - m_config.CacheFlushInterval = value; - } + m_config.DiskFlushInterval = value; } + } - /// - /// The number of stages to progress through before writing the final file. - /// - /// - /// This defaults to 3 stages which allows files up to 10 hours of data to be combined - /// into a single archive file. If is large and files of this - /// size are not being created, increase this to 4. - /// - /// Valid settings are 3 or 4. - /// - public int StagingCount + /// + /// The number of milliseconds before data is taken from its cache and put in the + /// memory file. + /// + /// + /// Must be between 1 and 1,000 + /// + public int CacheFlushInterval + { + get => m_config.CacheFlushInterval; + set { - get => m_config.StagingCount; - set - { - if (value < 3 || value > 4) - throw new ArgumentOutOfRangeException(nameof(value), "StagingCount must be 3 or 4"); - - m_config.StagingCount = value; - } - } + if (value is < 1 or > 1000) + throw new ArgumentOutOfRangeException(nameof(value), "Must be between 1 and 1,000"); - /// - /// The name associated with the database. - /// - public string DatabaseName => m_config.DatabaseName; + m_config.CacheFlushInterval = value; + } + } - /// - /// Determines whether the historian should import attached paths at startup. - /// - public bool ImportAttachedPathsAtStartup + /// + /// The number of stages to progress through before writing the final file. + /// + /// + /// This defaults to 3 stages which allows files up to 10 hours of data to be combined + /// into a single archive file. If is large and files of this + /// size are not being created, increase this to 4. + /// + /// Valid settings are 3 or 4. + /// + public int StagingCount + { + get => m_config.StagingCount; + set { - get => m_config.ImportAttachedPathsAtStartup; - set => m_config.ImportAttachedPathsAtStartup = value; + if (value is < 3 or > 4) + throw new ArgumentOutOfRangeException(nameof(value), "StagingCount must be 3 or 4"); + + m_config.StagingCount = value; } + } + + /// + /// The name associated with the database. + /// + public string DatabaseName => m_config.DatabaseName; - /// - /// Gets all the paths that are known by this historian. - /// A path can be a file name or a folder. - /// - public List ImportPaths => m_config.ImportPaths; - - /// - /// The list of directories where final files can be placed written. - /// If nothing is specified, the main directory is used. - /// - public List FinalWritePaths => m_config.FinalWritePaths; - - /// - /// Creates a configuration that can be used for - /// - /// - public ServerDatabaseSettings ToServerDatabaseSettings() => m_config.ToServerDatabaseSettings(); + /// + /// Determines whether the historian should import attached paths at startup. + /// + public bool ImportAttachedPathsAtStartup + { + get => m_config.ImportAttachedPathsAtStartup; + set => m_config.ImportAttachedPathsAtStartup = value; } + + /// + /// Gets all the paths that are known by this historian. + /// A path can be a file name or a folder. + /// + public List ImportPaths => m_config.ImportPaths; + + /// + /// The list of directories where final files can be placed written. + /// If nothing is specified, the main directory is used. + /// + public List FinalWritePaths => m_config.FinalWritePaths; + + /// + /// Creates a configuration that can be used for + /// + /// + public ServerDatabaseSettings ToServerDatabaseSettings() => m_config.ToServerDatabaseSettings(); } \ No newline at end of file diff --git a/Source/Libraries/openHistorian.Core/openHistorian.Core.SqlClr.csproj b/Source/Libraries/openHistorian.Core/openHistorian.Core.SqlClr.csproj index 5d4c55b50a..2b51e0a77d 100755 --- a/Source/Libraries/openHistorian.Core/openHistorian.Core.SqlClr.csproj +++ b/Source/Libraries/openHistorian.Core/openHistorian.Core.SqlClr.csproj @@ -12,7 +12,7 @@ v4.8 512 - 8 + latest true diff --git a/Source/Libraries/openHistorian.Core/openHistorian.Core.csproj b/Source/Libraries/openHistorian.Core/openHistorian.Core.csproj index 8ca7841555..fe45e6967b 100755 --- a/Source/Libraries/openHistorian.Core/openHistorian.Core.csproj +++ b/Source/Libraries/openHistorian.Core/openHistorian.Core.csproj @@ -12,7 +12,7 @@ v4.8 512 - 8 + latest true diff --git a/Source/openHistorian.sln.DotSettings b/Source/openHistorian.sln.DotSettings index b99ef53ee7..62339678c4 100644 --- a/Source/openHistorian.sln.DotSettings +++ b/Source/openHistorian.sln.DotSettings @@ -2,6 +2,7 @@ CFF DB ID + MB MD URL True