Fixed issue with Quoting Control not modifying setting.

FIxed issue with file modification showing many dialog boxes
Fixed issue with leading spaces if no trimming is selected
Ignoring rows with single column if we expect more than 3 columns
Changed GetIdFromFileName
Supporting LongPath in improved stream

Author: RNoeldner

Application/FormMain.Designer.cs

@@ -227,8 +227,9 @@ private void DisableIgnoreRead()
     private void Display_Activated(object sender, EventArgs e)
     {
       if (!m_FileChanged) return;
-
-      if (MessageBox.Show(this, "The displayed file has changed do you want to reload the data?", "File changed", MessageBoxButtons.YesNo) == DialogResult.Yes) OpenDataReader(true);
+      m_FileChanged = false;
+      if (_MessageBox.Show(this, "The displayed file has changed do you want to reload the data?", "File changed", MessageBoxButtons.YesNo, MessageBoxIcon.Question, MessageBoxDefaultButton.Button2) == DialogResult.Yes)
+        OpenDataReader(true);
     }
 
     private void Display_FormClosing(object sender, FormClosingEventArgs e)
@@ -294,7 +295,6 @@ private void Display_Shown(object sender, EventArgs e)
       }
 
       if (doClose) return;
-      fileSystemWatcher.EnableRaisingEvents = Settings.Default.DetectFileChanges;
       OpenDataReader(false);
     }
 
@@ -528,9 +528,11 @@ private bool InitFileSettings()
     /// </summary>
     private void OpenDataReader(bool clear)
     {
-      m_SettingsChangedTimerChange.Stop();
       if (m_FileSetting == null) return;
 
+      m_SettingsChangedTimerChange.Stop();
+      fileSystemWatcher.EnableRaisingEvents = false;
+
       var oldCursor = Cursor.Current == Cursors.WaitCursor ? Cursors.WaitCursor : Cursors.Default;
       Cursor.Current = Cursors.WaitCursor;
 
@@ -610,6 +612,7 @@ private void OpenDataReader(bool clear)
           col.PropertyChanged += FileSetting_PropertyChanged;
         }
         m_ConfigChanged = false;
+        fileSystemWatcher.EnableRaisingEvents = Settings.Default.DetectFileChanges;
         m_FileChanged = false;
         // Re enable event watching
         m_FileSetting.PropertyChanged += FileSetting_PropertyChanged;
@@ -621,7 +624,7 @@ private void SaveSetting()
       try
       {
         var pathSetting = m_FileSetting.FileName + CsvFile.cCsvSettingExtension;
-        m_FileSetting.FileName = m_FileSetting.FileName.Substring(m_FileSetting.FileName.LastIndexOf('\\'));
+        m_FileSetting.FileName = FileSystemUtils.SplitPath(m_FileSetting.FileName).FileName;
         var answer = DialogResult.No;
 
         if (FileSystemUtils.FileExists(pathSetting))

Application/FormMain.cs

@@ -215,6 +215,23 @@ public virtual IDataReader GetOpenDataReader(uint recordLimit)
       return null;
     }
 
+    public virtual IDataReader GetOpenSourceReader(uint recordLimit)
+    {
+      var sourceSetting = GetSourceSetting();
+      if (sourceSetting == null)
+      {
+        // Using the connection string
+        if (string.IsNullOrEmpty(m_FileSetting.SqlStatement)) return null;
+        return ApplicationSetting.SQLDataReader(m_FileSetting.SqlStatement);
+      }
+      else
+      {
+        var reader = sourceSetting.GetFileReader();
+        reader.Open(m_CancellationToken, true);
+        return reader;
+      }
+    }
+
     /// <summary>
     ///   Gets the source data table.
     /// </summary>

Library/ClassLibraryCSV/BaseFileWriter.cs

@@ -706,11 +706,10 @@ private bool GetNextRecord()
       bool hasWarningCombinedWrning = false;
       Restart2:
       var rowLength = CurrentRowColumnText.Length;
-
       if (rowLength == FieldCount)
       {
         // Check if we have row that matches the header row
-        if (m_HeaderRow != null && m_CsvFile.HasFieldHeader)
+        if (m_HeaderRow != null && m_CsvFile.HasFieldHeader && !hasWarningCombinedWrning)
         {
           bool isRepeatedHeader = true;
           for (int col = 0; col < FieldCount; col++)
@@ -740,11 +739,11 @@ private bool GetNextRecord()
           var oldPos = m_BufferPos;
           var startLine = StartLineNumber;
           // get the next row
-          var nextRow = ReadNextRow(true, true);
+          var nextLine = ReadNextRow(true, true);
           StartLineNumber = startLine;
 
           // allow up to two extra columns they can be combined later
-          if (nextRow.Length > 0 && nextRow.Length + rowLength < FieldCount + 4)
+          if (nextLine.Length > 0 && nextLine.Length + rowLength < FieldCount + 4)
           {
             var combined = new List<string>(CurrentRowColumnText);
 
@@ -753,14 +752,14 @@ private bool GetNextRecord()
             m_NumWarningsLinefeed++;
             HandleWarning(rowLength - 1,
               $"Added first column from line {EndLineNumber}, assuming a linefeed has split the rows into an additional line.");
-            combined[rowLength - 1] += ' ' + nextRow[0];
+            combined[rowLength - 1] += ' ' + nextLine[0];
 
-            for (int col = 1; col < nextRow.Length; col++)
-              combined.Add(nextRow[col]);
+            for (int col = 1; col < nextLine.Length; col++)
+              combined.Add(nextLine[col]);
 
             if (!hasWarningCombinedWrning)
             {
-              HandleWarning(-1, $"Line {StartLineNumber}{cLessColumns}. Rows have been combined.");
+              HandleWarning(-1, $"Line {StartLineNumber}-{EndLineNumber - 1}{cLessColumns}. Lines have been combined.");
               hasWarningCombinedWrning = true;
             }
 
@@ -775,14 +774,23 @@ private bool GetNextRecord()
             else
             {
               // return to the old position so reading the next row did not matter
-              HandleWarning(-1, $"Line {StartLineNumber}{cLessColumns}({rowLength}/{FieldCount}).");
+              if (!hasWarningCombinedWrning)
+                HandleWarning(-1, $"Line {StartLineNumber}{cLessColumns}({rowLength}/{FieldCount}).");
               m_BufferPos = oldPos;
             }
           }
         }
       }
-      else if (rowLength > FieldCount && m_CsvFile.WarnEmptyTailingColumns)
+
+      // if we still have only one column and we should have a number of columns assume this was nonsense like a report footer
+      if (rowLength == 1 && FieldCount > 3)
+      {
+        RecordNumber--;
+        goto Restart;
+      }
+
       // If more columns are present...
+      if (rowLength > FieldCount && (m_CsvFile.WarnEmptyTailingColumns || m_RealignColumns != null))
       {
         // check if the additional columns have contents
         var hasContens = false;
@@ -1014,7 +1022,7 @@ private string ReadNextColumn(int columnNo, bool storeWarnings)
           if (IsWhiteSpace(character))
           {
             // Store the white spaces if we do any kind of trimming
-            if (m_CsvFile.TrimmingOption != TrimmingOption.None)
+            if (m_CsvFile.TrimmingOption == TrimmingOption.None)
               // Values will be trimmed later but we need to find out, if the filed is quoted first
               stringBuilder.Append(character);
             continue;
@@ -1285,7 +1293,7 @@ private void ReadToEOL()
     private void ResetPositionToStart()
     {
       if (m_ImprovedStream == null)
-        m_ImprovedStream = ImprovedStream.OpenRead(m_CsvFile, ProcessDisplay);
+        m_ImprovedStream = ImprovedStream.OpenRead(m_CsvFile);
       m_ImprovedStream.ResetToStart(delegate (Stream str)
       {
         // in case we can not seek need to reopen the stream reader

Library/ClassLibraryCSV/CsvFileReader.cs

@@ -195,7 +195,7 @@ public static void GuessCodePage(ICsvFile setting)
       // Read 256 kBytes
       var buff = new byte[262144];
       int length;
-      using (var fileStream = ImprovedStream.OpenRead(setting, null))
+      using (var fileStream = ImprovedStream.OpenRead(setting))
       {
         length = fileStream.Stream.Read(buff, 0, buff.Length);
       }
@@ -237,7 +237,7 @@ public static string GuessDelimiter(ICsvFile setting)
     {
       Contract.Requires(setting != null);
       Contract.Ensures(Contract.Result<string>() != null);
-      using (var improvedStream = ImprovedStream.OpenRead(setting, null))
+      using (var improvedStream = ImprovedStream.OpenRead(setting))
       using (var streamReader = new StreamReader(improvedStream.Stream, setting.GetEncoding(), setting.ByteOrderMark))
       {
         for (int i = 0; i < setting.SkipRows; i++)
@@ -300,7 +300,7 @@ public static bool GuessHasHeader(ICsvFile setting, IProcessDisplay processDispl
     public static string GuessNewline(ICsvFile setting)
     {
       Contract.Requires(setting != null);
-      using (var improvedStream = ImprovedStream.OpenRead(setting, null))
+      using (var improvedStream = ImprovedStream.OpenRead(setting))
       using (var streamReader = new StreamReader(improvedStream.Stream, setting.GetEncoding(), setting.ByteOrderMark))
       {
         for (int i = 0; i < setting.SkipRows; i++)
@@ -317,7 +317,7 @@ public static string GuessNewline(ICsvFile setting)
     public static bool GuessNotADelimitedFile(ICsvFile setting)
     {
       Contract.Requires(setting != null);
-      using (var improvedStream = ImprovedStream.OpenRead(setting, null))
+      using (var improvedStream = ImprovedStream.OpenRead(setting))
       using (var streamReader = new StreamReader(improvedStream.Stream, setting.GetEncoding(), setting.ByteOrderMark))
       {
         for (int i = 0; i < setting.SkipRows; i++)
@@ -346,7 +346,7 @@ public static bool GuessNotADelimitedFile(ICsvFile setting)
     public static int GuessStartRow(ICsvFile setting)
     {
       Contract.Requires(setting != null);
-      using (var improvedStream = ImprovedStream.OpenRead(setting, null))
+      using (var improvedStream = ImprovedStream.OpenRead(setting))
       using (var streamReader = new StreamReader(improvedStream.Stream, setting.GetEncoding(), setting.ByteOrderMark))
       {
         return GuessStartRow(streamReader, setting.FileFormat.FieldDelimiterChar,
@@ -369,7 +369,7 @@ public static bool HasUsedQualifier(ICsvFile setting, CancellationToken token)
       if (string.IsNullOrEmpty(setting.FileFormat.FieldQualifier) || token.IsCancellationRequested)
         return false;
 
-      using (var improvedStream = ImprovedStream.OpenRead(setting, null))
+      using (var improvedStream = ImprovedStream.OpenRead(setting))
       using (var streamReader = new StreamReader(improvedStream.Stream, setting.GetEncoding(), setting.ByteOrderMark))
       {
         for (int i = 0; i < setting.SkipRows; i++)

Library/ClassLibraryCSV/CsvHelper.cs

@@ -19,9 +19,9 @@
 using System.Diagnostics.CodeAnalysis;
 using System.Diagnostics.Contracts;
 using System.Globalization;
-using System.IO;
 using System.Linq;
 using System.Text;
+using System.Text.RegularExpressions;
 using System.Threading;
 
 namespace CsvTools
@@ -305,7 +305,7 @@ public static string DataTypeDisplay(this DataType dt)
           return "Text Part";
 
         case DataType.TextToHtml:
-          return "Encode HTML (Linefeeds and CData Tags)";
+          return "Encode HTML (Linefeed and CData Tags)";
 
         case DataType.TextToHtmlFull:
           return "Encode HTML ('<' -> '&lt;')";
@@ -418,20 +418,33 @@ public static string GetIdFromFileName(this string path)
       Contract.Requires(path != null);
       Contract.Ensures(Contract.Result<string>() != null);
 
-      var fileName = Path.GetFileNameWithoutExtension(path);
-      if (path.AssumePgp() || path.AssumeGZip())
-        fileName = Path.GetFileNameWithoutExtension(fileName);
-
-      // get rid of _PM and _AM
-      if (fileName != null && (fileName.EndsWith("_PM", StringComparison.OrdinalIgnoreCase) ||
-                               fileName.EndsWith("_AM", StringComparison.OrdinalIgnoreCase)))
-        fileName = fileName.Substring(0, fileName.Length - 3);
-
-      return StringUtils.ProcessByCategory(fileName, x =>
-          x == UnicodeCategory.UppercaseLetter || x == UnicodeCategory.LowercaseLetter
-                                               || x == UnicodeCategory.ConnectorPunctuation ||
-                                               x == UnicodeCategory.DashPunctuation)
-        .Trim('_', '-').Replace("__", "_").Replace("__", "_").Replace("--", "-").Replace("--", "-");
+      var fileName = StringUtils.ProcessByCategory(FileSystemUtils.SplitPath(path).FileName, x =>
+           x == UnicodeCategory.UppercaseLetter || x == UnicodeCategory.LowercaseLetter || x == UnicodeCategory.OtherLetter ||
+           x == UnicodeCategory.ConnectorPunctuation || x == UnicodeCategory.DashPunctuation || x == UnicodeCategory.OtherPunctuation ||
+           x == UnicodeCategory.DecimalDigitNumber);
+
+      const string T = @"(:|-|_)";
+      const string HH = @"(2[0-3]|((0|1)?\d))";    // 0-9 10-19 20-23
+      const string MS = @"([0-5][0-9])";           // 00-59
+      const string TT = @"((_| )?(AM|PM))?";
+      const string S = @"(\/|.|-|_)?";
+      const string YYYY = @"((19\d{2})|(2\d{3}))"; // 1900 - 2999
+      const string MM = @"([0,1]?\d{1})";
+      const string DD = @"(([0-2]?\d{1})|([3][0,1]{1}))";
+      // Replace Times 3_53_34_AM
+      fileName = Regex.Replace(fileName, T + "?" + HH + T + MS + T + "?" + MS + "?" + TT, string.Empty, RegexOptions.IgnoreCase | RegexOptions.Singleline | RegexOptions.CultureInvariant);
+
+      // Replace Dates YYYYMMDDHHMMSS
+      fileName = Regex.Replace(fileName, S + YYYY + S + MM + S + DD + T + "?" + HH + T + MS + T + "?" + MS + "?" + TT, string.Empty, RegexOptions.IgnoreCase | RegexOptions.Singleline | RegexOptions.CultureInvariant);
+
+      // Replace Dates DDMMYYYY
+      fileName = Regex.Replace(fileName, S + YYYY + S + MM + S + DD, string.Empty, RegexOptions.IgnoreCase | RegexOptions.Singleline | RegexOptions.CultureInvariant);
+      // Replace Dates MMDDYYYY
+      fileName = Regex.Replace(fileName, S + MM + S + DD + S + YYYY, string.Empty, RegexOptions.IgnoreCase | RegexOptions.Singleline | RegexOptions.CultureInvariant);
+      // Replace Dates DDMMYYYY
+      fileName = Regex.Replace(fileName, S + DD + S + MM + S + YYYY, string.Empty, RegexOptions.IgnoreCase | RegexOptions.Singleline | RegexOptions.CultureInvariant);
+
+      return fileName.Trim('_', '-', ' ', '\t').Replace("__", "_").Replace("__", "_").Replace("--", "-").Replace("--", "-");
     }
 
     /// <summary>

Library/ClassLibraryCSV/Extensions.cs

@@ -14,6 +14,8 @@
 
 using System;
 using System.IO;
+using File = Pri.LongPath.File;
+using FileInfo = Pri.LongPath.FileInfo;
 
 namespace CsvTools
 {
@@ -114,16 +116,15 @@ public void ResetToStart(Action<Stream> AfterInit)
     /// </summary>
     /// <param name="path">The path.</param>
     /// <param name="decryptedPassphrase">The decrypted passphrase.</param>
-    /// <param name="processDisplay">The process display.</param>
     /// <returns></returns>
     /// <exception cref="ArgumentException">Path must be set - path</exception>
-    public static ImprovedStream OpenRead(string path, Func<string> encryptedPassphrase = null, IProcessDisplay processDisplay = null)
+    public static ImprovedStream OpenRead(string path, Func<string> encryptedPassphrase = null)
     {
       if (string.IsNullOrEmpty(path))
       {
         throw new ArgumentException("Path must be set", nameof(path));
       }
-      var retVal = OpenBaseStream(path, encryptedPassphrase, processDisplay);
+      var retVal = OpenBaseStream(path, encryptedPassphrase);
       retVal.ResetToStart(null);
       return retVal;
     }
@@ -132,16 +133,15 @@ public static ImprovedStream OpenRead(string path, Func<string> encryptedPassphr
     /// Opens the a setting for reading
     /// </summary>
     /// <param name="setting">The setting.</param>
-    /// <param name="processDisplay">The process display.</param>
     /// <returns></returns>
     /// <exception cref="ApplicationException">
     /// Please provide a passphrase.
     /// or
     /// Please reenter the passphrase, the passphrase could not be decrypted.
     /// </exception>
-    public static ImprovedStream OpenRead(IFileSetting setting, IProcessDisplay processDisplay)
+    public static ImprovedStream OpenRead(IFileSetting setting)
     {
-      var retVal = OpenBaseStream(setting.FullPath, setting.GetEncryptedPassphraseFunction, processDisplay);
+      var retVal = OpenBaseStream(setting.FullPath, setting.GetEncryptedPassphraseFunction);
 
       retVal.ResetToStart(delegate (Stream sr)
       {
@@ -193,7 +193,6 @@ public static ImprovedStream OpenWrite(string path, IProcessDisplay processDispl
     /// <summary>
     /// Closes the stream in case of a file opened for writing it would be uploaded to the sFTP
     /// </summary>
-    /// <param name="processDisplay">The display.</param>
     public void Close()
     {
       if (Stream != null)
@@ -216,9 +215,11 @@ public void Close()
     /// Opens the base stream, handling sFTP access
     /// </summary>
     /// <param name="path">The path.</param>
-    /// <param name="processDisplay">The process display.</param>
-    /// <returns>An improved stream where the base stream is set</returns>
-    private static ImprovedStream OpenBaseStream(string path, Func<string> encryptedPassphraseFunc, IProcessDisplay processDisplay = null)
+    /// <param name="encryptedPassphraseFunc">The encrypted passphrase function.</param>
+    /// <returns>
+    /// An improved stream where the base stream is set
+    /// </returns>
+    private static ImprovedStream OpenBaseStream(string path, Func<string> encryptedPassphraseFunc)
     {
       var retVal = new ImprovedStream
       {
@@ -231,8 +232,8 @@ private static ImprovedStream OpenBaseStream(string path, Func<string> encrypted
       }
       try
       {
-        retVal.BasePath = path.LongPathPrefix();
-        retVal.BaseStream = File.OpenRead(path.LongPathPrefix());
+        retVal.BasePath = path;
+        retVal.BaseStream = File.OpenRead(path);
         return retVal;
       }
       catch (Exception)
@@ -291,7 +292,7 @@ private void CloseWrite()
           else if (WritePath.AssumePgp())
           {
             using (FileStream inputStream = new FileInfo(TempFile).OpenRead(),
-                                   output = new FileStream(WritePath, FileMode.Create))
+                                   output = new FileStream(WritePath.LongPathPrefix(), FileMode.Create))
             {
               ApplicationSetting.ToolSetting.PGPInformation.PgpEncrypt(inputStream, output, Recipient, ProcessDisplay);
             }