diff --git a/.gitignore b/.gitignore index 308bc974..9d720f7f 100644 --- a/.gitignore +++ b/.gitignore @@ -33,8 +33,6 @@ x86/ bld/ [Bb]in/ [Oo]bj/ -[Ll]og/ -[Ll]ogs/ # Visual Studio 2015/2017 cache/options directory .vs/ diff --git a/src/LogExpert/Classes/Log/LogfileReader.cs b/src/LogExpert/Classes/Log/LogfileReader.cs index 521edf06..200a59f9 100644 --- a/src/LogExpert/Classes/Log/LogfileReader.cs +++ b/src/LogExpert/Classes/Log/LogfileReader.cs @@ -31,7 +31,6 @@ public class LogfileReader : IAutoLogLineColumnizerCallback private IList _bufferList; private ReaderWriterLock _bufferListLock; - private IList _bufferLru; private bool _contentDeleted; private int _currLineCount; private ReaderWriterLock _disposeLock; @@ -47,9 +46,8 @@ public class LogfileReader : IAutoLogLineColumnizerCallback private bool _isFastFailOnGetLogLine; private bool _isLineCountDirty = true; private IList _logFileInfoList = []; - private Dictionary _lruCacheDict; - private ReaderWriterLock _lruCacheDictLock; + private LruCacheManager _lruCacheManager; private bool _shouldStop; @@ -58,73 +56,78 @@ public class LogfileReader : IAutoLogLineColumnizerCallback #endregion #region cTor - public LogfileReader(string fileName, EncodingOptions encodingOptions, bool multiFile, int bufferCount, int linesPerBuffer, MultiFileOptions multiFileOptions) { if (fileName == null) { - return; + throw new ArgumentNullException(nameof(fileName)); } _fileName = fileName; - EncodingOptions = encodingOptions; - IsMultiFile = multiFile; _MAX_BUFFERS = bufferCount; _MAX_LINES_PER_BUFFER = linesPerBuffer; _multiFileOptions = multiFileOptions; _logLineFx = GetLogLineInternal; - InitLruBuffers(); - - if (multiFile) - { - ILogFileInfo info = GetLogFileInfo(fileName); - RolloverFilenameHandler rolloverHandler = new(info, _multiFileOptions); - LinkedList nameList = rolloverHandler.GetNameList(); - - ILogFileInfo fileInfo = null; - foreach (string name in nameList) - { - fileInfo = AddFile(name); - } - - _watchedILogFileInfo = fileInfo; // last added file in the list is the watched file - } - else - { - _watchedILogFileInfo = AddFile(fileName); - } + IsMultiFile = multiFile; - StartGCThread(); + InitializeReader(encodingOptions, null); } public LogfileReader(string[] fileNames, EncodingOptions encodingOptions, int bufferCount, int linesPerBuffer, MultiFileOptions multiFileOptions) { if (fileNames == null || fileNames.Length < 1) { - return; + throw new ArgumentNullException(nameof(fileNames)); } - EncodingOptions = encodingOptions; - IsMultiFile = true; + _fileName = fileNames[0]; _MAX_BUFFERS = bufferCount; _MAX_LINES_PER_BUFFER = linesPerBuffer; _multiFileOptions = multiFileOptions; _logLineFx = GetLogLineInternal; + IsMultiFile = true; + + InitializeReader(encodingOptions, fileNames); + } + private void InitializeReader(EncodingOptions encodingOptions, string[]? additionalFiles) + { + EncodingOptions = encodingOptions; + _lruCacheManager = new LruCacheManager(_MAX_BUFFERS); InitLruBuffers(); - ILogFileInfo fileInfo = null; - foreach (string name in fileNames) + if (additionalFiles != null) { - fileInfo = AddFile(name); + // Handle array constructor case + ILogFileInfo fileInfo = null; + foreach (string name in additionalFiles) + { + fileInfo = AddFile(name); + } + _watchedILogFileInfo = fileInfo; } + else if (IsMultiFile) + { + // Handle single file multiFile case + ILogFileInfo info = GetLogFileInfo(_fileName); + RolloverFilenameHandler rolloverHandler = new(info, _multiFileOptions); + LinkedList nameList = rolloverHandler.GetNameList(); - _watchedILogFileInfo = fileInfo; - _fileName = fileInfo.FullName; + ILogFileInfo fileInfo = null; + foreach (string name in nameList) + { + fileInfo = AddFile(name); + } + _watchedILogFileInfo = fileInfo; // last added file in the list is the watched file + } + else + { + // Handle single file case + _watchedILogFileInfo = AddFile(_fileName); + } StartGCThread(); } - #endregion #region Delegates @@ -224,7 +227,8 @@ public void ReadFiles() //this.lastReturnedLineNum = -1; //this.lastReturnedLineNumForBuffer = -1; _isDeleted = false; - ClearLru(); + _lruCacheManager.ClearLru(); + AcquireBufferListWriterLock(); _bufferList.Clear(); ReleaseBufferListWriterLock(); @@ -357,14 +361,13 @@ public int ShiftBuffers() } } - _lruCacheDictLock.AcquireWriterLock(Timeout.Infinite); + _lruCacheManager.Lock.EnterWriteLock(); _logger.Info("Adjusting StartLine values in {0} buffers by offset {1}", _bufferList.Count, offset); foreach (LogBuffer buffer in _bufferList) { - SetNewStartLineForBuffer(buffer, buffer.StartLine - offset); + _lruCacheManager.SetNewStartLineForBuffer(buffer, buffer.StartLine - offset); } - - _lruCacheDictLock.ReleaseWriterLock(); + _lruCacheManager.Lock.ExitWriteLock(); #if DEBUG if (_bufferList.Count > 0) { @@ -636,7 +639,6 @@ public void DeleteAllContent() _logger.Info("Deleting all log buffers for {0}. Used mem: {1:N0}", Util.GetNameFromPath(_fileName), GC.GetTotalMemory(true)); //TODO [Z] uh GC collect calls creepy AcquireBufferListWriterLock(); - _lruCacheDictLock.AcquireWriterLock(Timeout.Infinite); _disposeLock.AcquireWriterLock(Timeout.Infinite); foreach (LogBuffer logBuffer in _bufferList) @@ -647,11 +649,9 @@ public void DeleteAllContent() } } - _lruCacheDict.Clear(); _bufferList.Clear(); - _disposeLock.ReleaseWriterLock(); - _lruCacheDictLock.ReleaseWriterLock(); + _lruCacheManager.ClearLru(); ReleaseBufferListWriterLock(); GC.Collect(); _contentDeleted = true; @@ -667,7 +667,7 @@ public void ChangeEncoding(Encoding encoding) CurrentEncoding = encoding; EncodingOptions.Encoding = encoding; ResetBufferCache(); - ClearLru(); + _lruCacheManager.ClearLru(); } /// @@ -720,10 +720,8 @@ internal void LogBufferInfoForLine(int lineNum) internal void LogBufferDiagnostic() { _logger.Info("-------- Buffer diagnostics -------"); - _lruCacheDictLock.AcquireReaderLock(Timeout.Infinite); - int cacheCount = _lruCacheDict.Count; + int cacheCount = _lruCacheManager.Count; _logger.Info("LRU entries: {0}", cacheCount); - _lruCacheDictLock.ReleaseReaderLock(); AcquireBufferListReaderLock(); _logger.Info("File: {0}\r\nBuffer count: {1}\r\nDisposed buffers: {2}", _fileName, _bufferList.Count, _bufferList.Count - cacheCount); @@ -809,10 +807,7 @@ private Task GetLogLineInternal(int lineNum) private void InitLruBuffers() { _bufferList = []; - _bufferLru = new List(_MAX_BUFFERS + 1); //this.lruDict = new Dictionary(this.MAX_BUFFERS + 1); // key=startline, value = index in bufferLru - _lruCacheDict = new Dictionary(_MAX_BUFFERS + 1); - _lruCacheDictLock = new ReaderWriterLock(); _bufferListLock = new ReaderWriterLock(); _disposeLock = new ReaderWriterLock(); } @@ -872,41 +867,31 @@ private void ReplaceBufferInfos(ILogFileInfo oldLogFileInfo, ILogFileInfo newLog private LogBuffer DeleteBuffersForInfo(ILogFileInfo ILogFileInfo, bool matchNamesOnly) { - _logger.Info("Deleting buffers for file {0}", ILogFileInfo.FullName); LogBuffer lastRemovedBuffer = null; IList deleteList = []; + _logger.Info("Deleting buffers for file {0}", ILogFileInfo.FullName); + Func matchPredicate = matchNamesOnly + ? buffer => buffer.FileInfo.FullName.Equals(ILogFileInfo.FullName, StringComparison.OrdinalIgnoreCase) + : buffer => buffer.FileInfo == ILogFileInfo; + AcquireBufferListWriterLock(); - _lruCacheDictLock.AcquireWriterLock(Timeout.Infinite); - if (matchNamesOnly) + for (int i = _bufferList.Count - 1; i >= 0; i--) { - foreach (LogBuffer buffer in _bufferList) - { - if (buffer.FileInfo.FullName.ToLower().Equals(ILogFileInfo.FullName.ToLower())) - { - lastRemovedBuffer = buffer; - deleteList.Add(buffer); - } - } - } - else - { - foreach (LogBuffer buffer in _bufferList) + LogBuffer buffer = _bufferList[i]; + if (matchPredicate(buffer)) { - if (buffer.FileInfo == ILogFileInfo) - { - lastRemovedBuffer = buffer; - deleteList.Add(buffer); - } + lastRemovedBuffer = buffer; + Util.AssertTrue(_bufferListLock.IsWriterLockHeld, "No writer lock for buffer list"); + deleteList.Add(buffer); + _bufferList.RemoveAt(i); } } + ReleaseBufferListWriterLock(); - foreach (LogBuffer buffer in deleteList) - { - RemoveFromBufferList(buffer); - } - _lruCacheDictLock.ReleaseWriterLock(); - ReleaseBufferListWriterLock(); + + _lruCacheManager.RemoveBuffers(deleteList); + if (lastRemovedBuffer == null) { _logger.Info("lastRemovedBuffer is null"); @@ -919,18 +904,6 @@ private LogBuffer DeleteBuffersForInfo(ILogFileInfo ILogFileInfo, bool matchName return lastRemovedBuffer; } - /// - /// The caller must have writer locks for lruCache and buffer list! - /// - /// - private void RemoveFromBufferList(LogBuffer buffer) - { - Util.AssertTrue(_lruCacheDictLock.IsWriterLockHeld, "No writer lock for lru cache"); - Util.AssertTrue(_bufferListLock.IsWriterLockHeld, "No writer lock for buffer list"); - _lruCacheDict.Remove(buffer.StartLine); - _bufferList.Remove(buffer); - } - private void ReadToBufferList(ILogFileInfo logFileInfo, long filePos, int startLine) { try @@ -1059,133 +1032,7 @@ private void AddBufferToList(LogBuffer logBuffer) #endif _bufferList.Add(logBuffer); //UpdateLru(logBuffer); - UpdateLruCache(logBuffer); - } - - private void UpdateLruCache(LogBuffer logBuffer) - { - _lruCacheDictLock.AcquireReaderLock(Timeout.Infinite); - if (_lruCacheDict.TryGetValue(logBuffer.StartLine, out LogBufferCacheEntry cacheEntry)) - { - cacheEntry.Touch(); - } - else - { - LockCookie cookie = _lruCacheDictLock.UpgradeToWriterLock(Timeout.Infinite); - if (!_lruCacheDict.TryGetValue(logBuffer.StartLine, out cacheEntry) - ) // #536: re-test, because multiple threads may have been waiting for writer lock - { - cacheEntry = new LogBufferCacheEntry(); - cacheEntry.LogBuffer = logBuffer; - try - { - _lruCacheDict.Add(logBuffer.StartLine, cacheEntry); - } - catch (ArgumentException e) - { - _logger.Error(e, "Error in LRU cache: " + e.Message); -#if DEBUG // there seems to be a bug with double added key - - _logger.Info("Added buffer:"); - DumpBufferInfos(logBuffer); - if (_lruCacheDict.TryGetValue(logBuffer.StartLine, out LogBufferCacheEntry existingEntry)) - { - _logger.Info("Existing buffer: "); - DumpBufferInfos(existingEntry.LogBuffer); - } - else - { - _logger.Warn("Ooops? Cannot find the already existing entry in LRU."); - } -#endif - _lruCacheDictLock.ReleaseLock(); - throw; - } - } - - _lruCacheDictLock.DowngradeFromWriterLock(ref cookie); - } - - _lruCacheDictLock.ReleaseReaderLock(); - } - - /// - /// Sets a new start line in the given buffer and updates the LRU cache, if the buffer - /// is present in the cache. The caller must have write lock for 'lruCacheDictLock'; - /// - /// - /// - private void SetNewStartLineForBuffer(LogBuffer logBuffer, int newLineNum) - { - Util.AssertTrue(_lruCacheDictLock.IsWriterLockHeld, "No writer lock for lru cache"); - if (_lruCacheDict.ContainsKey(logBuffer.StartLine)) - { - _lruCacheDict.Remove(logBuffer.StartLine); - logBuffer.StartLine = newLineNum; - LogBufferCacheEntry cacheEntry = new(); - cacheEntry.LogBuffer = logBuffer; - _lruCacheDict.Add(logBuffer.StartLine, cacheEntry); - } - else - { - logBuffer.StartLine = newLineNum; - } - } - - private void GarbageCollectLruCache() - { -#if DEBUG - long startTime = Environment.TickCount; -#endif - _logger.Debug("Starting garbage collection"); - int threshold = 10; - _lruCacheDictLock.AcquireWriterLock(Timeout.Infinite); - int diff = 0; - if (_lruCacheDict.Count - (_MAX_BUFFERS + threshold) > 0) - { - diff = _lruCacheDict.Count - _MAX_BUFFERS; -#if DEBUG - if (diff > 0) - { - _logger.Info("Removing {0} entries from LRU cache for {1}", diff, Util.GetNameFromPath(_fileName)); - } -#endif - SortedList useSorterList = []; - // sort by usage counter - foreach (LogBufferCacheEntry entry in _lruCacheDict.Values) - { - if (!useSorterList.ContainsKey(entry.LastUseTimeStamp)) - { - useSorterList.Add(entry.LastUseTimeStamp, entry.LogBuffer.StartLine); - } - } - - // remove first entries (least usage) - _disposeLock.AcquireWriterLock(Timeout.Infinite); - for (int i = 0; i < diff; ++i) - { - if (i >= useSorterList.Count) - { - break; - } - - int startLine = useSorterList.Values[i]; - LogBufferCacheEntry entry = _lruCacheDict[startLine]; - _lruCacheDict.Remove(startLine); - entry.LogBuffer.DisposeContent(); - } - - _disposeLock.ReleaseWriterLock(); - } - - _lruCacheDictLock.ReleaseWriterLock(); -#if DEBUG - if (diff > 0) - { - long endTime = Environment.TickCount; - _logger.Info("Garbage collector time: " + (endTime - startTime) + " ms."); - } -#endif + _lruCacheManager.UpdateLruCache(logBuffer); } private void GarbageCollectorThreadProc() @@ -1199,39 +1046,11 @@ private void GarbageCollectorThreadProc() catch (Exception) { } - - GarbageCollectLruCache(); + _lruCacheManager.GarbageCollectLruCache(); } } - // private void UpdateLru(LogBuffer logBuffer) - // { - // lock (this.monitor) - // { - // int index; - // if (this.lruDict.TryGetValue(logBuffer.StartLine, out index)) - // { - // RemoveBufferFromLru(logBuffer, index); - // AddBufferToLru(logBuffer); - // } - // else - // { - // if (this.bufferLru.Count > MAX_BUFFERS - 1) - // { - // LogBuffer looser = this.bufferLru[0]; - // if (looser != null) - // { - //#if DEBUG - // _logger.logDebug("Disposing buffer: " + looser.StartLine + "/" + looser.LineCount + "/" + looser.FileInfo.FileName); - //#endif - // looser.DisposeContent(); - // RemoveBufferFromLru(looser); - // } - // } - // AddBufferToLru(logBuffer); - // } - // } - // } + ///// ///// Removes a LogBuffer from the LRU. Note that the LogBuffer is searched in the lruDict @@ -1280,30 +1099,6 @@ private void GarbageCollectorThreadProc() // } //} - private void ClearLru() - { - //lock (this.monitor) - //{ - // foreach (LogBuffer buffer in this.bufferLru) - // { - // buffer.DisposeContent(); - // } - // this.bufferLru.Clear(); - // this.lruDict.Clear(); - //} - _logger.Info("Clearing LRU cache."); - _lruCacheDictLock.AcquireWriterLock(Timeout.Infinite); - _disposeLock.AcquireWriterLock(Timeout.Infinite); - foreach (LogBufferCacheEntry entry in _lruCacheDict.Values) - { - entry.LogBuffer.DisposeContent(); - } - - _lruCacheDict.Clear(); - _disposeLock.ReleaseWriterLock(); - _lruCacheDictLock.ReleaseWriterLock(); - _logger.Info("Clearing done."); - } private void ReReadBuffer(LogBuffer logBuffer) { @@ -1408,7 +1203,7 @@ private LogBuffer GetBufferForLine(int lineNum) if (lineNum >= logBuffer.StartLine && lineNum < logBuffer.StartLine + logBuffer.LineCount) { //UpdateLru(logBuffer); - UpdateLruCache(logBuffer); + _lruCacheManager.UpdateLruCache(logBuffer); //this.lastReturnedLineNumForBuffer = lineNum; //this.lastReturnedBuffer = logBuffer; break; diff --git a/src/LogExpert/Classes/Log/LruCacheManager.cs b/src/LogExpert/Classes/Log/LruCacheManager.cs new file mode 100644 index 00000000..aa822add --- /dev/null +++ b/src/LogExpert/Classes/Log/LruCacheManager.cs @@ -0,0 +1,188 @@ +using LogExpert.Entities; +using NLog; +using System; +using System.Collections.Generic; +using System.Threading; + +namespace LogExpert.Classes.Log +{ + public class LruCacheManager + { + private static readonly ILogger _logger = LogManager.GetCurrentClassLogger(); + + private readonly int _maxBuffers; + private readonly ReaderWriterLockSlim _lruCacheDictLock; + private readonly ReaderWriterLockSlim _disposeLock; + private Dictionary _lruCacheDict; + + public ReaderWriterLockSlim Lock { get { return _lruCacheDictLock; } +} + +public int Count + { + get + { + _lruCacheDictLock.EnterReadLock(); + try + { + return _lruCacheDict.Count; + } + finally + { + _lruCacheDictLock.ExitReadLock(); + } + } + } + + public LruCacheManager(int maxBuffers) + { + _maxBuffers = maxBuffers; + _lruCacheDictLock = new ReaderWriterLockSlim(); + _disposeLock = new ReaderWriterLockSlim(); + _lruCacheDict = new Dictionary(_maxBuffers + 1); + } + + public void UpdateLruCache(LogBuffer logBuffer) + { + _lruCacheDictLock.EnterUpgradeableReadLock(); + try + { + if (_lruCacheDict.TryGetValue(logBuffer.StartLine, out LogBufferCacheEntry cacheEntry)) + { + cacheEntry.Touch(); + } + else + { + _lruCacheDictLock.EnterWriteLock(); + try + { + if (!_lruCacheDict.TryGetValue(logBuffer.StartLine, out cacheEntry)) + { + cacheEntry = new LogBufferCacheEntry { LogBuffer = logBuffer }; + _lruCacheDict.Add(logBuffer.StartLine, cacheEntry); + } + } + finally + { + _lruCacheDictLock.ExitWriteLock(); + } + } + } + finally + { + _lruCacheDictLock.ExitUpgradeableReadLock(); + } + } + + public void ClearLru() + { + _logger.Info("Clearing LRU cache."); + _lruCacheDictLock.EnterWriteLock(); + _disposeLock.EnterWriteLock(); + try + { + foreach (LogBufferCacheEntry entry in _lruCacheDict.Values) + { + entry.LogBuffer.DisposeContent(); + } + + _lruCacheDict.Clear(); + } + finally + { + _disposeLock.ExitWriteLock(); + _lruCacheDictLock.ExitWriteLock(); + } + _logger.Info("Clearing done."); + } + + public void GarbageCollectLruCache() + { +#if DEBUG + long startTime = Environment.TickCount; +#endif + _logger.Debug("Starting garbage collection"); + int threshold = 10; + _lruCacheDictLock.EnterWriteLock(); + try + { + int diff = _lruCacheDict.Count - _maxBuffers; + if (diff > threshold) + { +#if DEBUG + _logger.Info("Removing {0} entries from LRU cache", diff); +#endif + SortedList useSorterList = new(); + foreach (LogBufferCacheEntry entry in _lruCacheDict.Values) + { + if (!useSorterList.ContainsKey(entry.LastUseTimeStamp)) + { + useSorterList.Add(entry.LastUseTimeStamp, entry.LogBuffer.StartLine); + } + } + + _disposeLock.EnterWriteLock(); + try + { + for (int i = 0; i < diff; ++i) + { + if (i >= useSorterList.Count) + { + break; + } + + int startLine = useSorterList.Values[i]; + LogBufferCacheEntry entry = _lruCacheDict[startLine]; + _lruCacheDict.Remove(startLine); + entry.LogBuffer.DisposeContent(); + } + } + finally + { + _disposeLock.ExitWriteLock(); + } + } + } + finally + { + _lruCacheDictLock.ExitWriteLock(); + } +#if DEBUG + long endTime = Environment.TickCount; + _logger.Info("Garbage collector time: " + (endTime - startTime) + " ms."); +#endif + } + + public void SetNewStartLineForBuffer(LogBuffer logBuffer, int newLineNum) + { + Util.AssertTrue(_lruCacheDictLock.IsWriteLockHeld, "No writer lock for lru cache"); + if (_lruCacheDict.ContainsKey(logBuffer.StartLine)) + { + _lruCacheDict.Remove(logBuffer.StartLine); + logBuffer.StartLine = newLineNum; + LogBufferCacheEntry cacheEntry = new() { LogBuffer = logBuffer }; + _lruCacheDict.Add(logBuffer.StartLine, cacheEntry); + } + else + { + logBuffer.StartLine = newLineNum; + } + } + + internal void RemoveBuffers(IList deleteList) + { + _lruCacheDictLock.EnterWriteLock(); + try + { + foreach (LogBuffer buffer in deleteList) + { + _lruCacheDict.Remove(buffer.StartLine); + } + } + finally + { + _lruCacheDictLock.ExitWriteLock(); + } + } + } +}