Skip to content

Commit 8f1f857

Browse files
author
Doug Schmidt
authored
Merge pull request #276 from DougSchmidt-AI/feature/PF-1205-PointZillaNotes
PF-1205 - Fixed the writing of notes to a separate CSV file
2 parents 15ae92d + 988d550 commit 8f1f857

File tree

14 files changed

+466
-285
lines changed

14 files changed

+466
-285
lines changed

‎TimeSeries/PublicApis/SdkExamples/PointZilla/Context.cs‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,7 @@ public class Context
5555
public Instant? SourceQueryTo { get; set; }
5656
public string SaveCsvPath { get; set; }
5757
public bool StopAfterSavingCsv { get; set; }
58+
public SaveNotesMode SaveNotesMode { get; set; }
5859

5960
public List<TimeSeriesPoint> ManualPoints { get; set; } = new List<TimeSeriesPoint>();
6061
public List<TimeSeriesNote> ManualNotes { get; set; } = new List<TimeSeriesNote>();
@@ -103,8 +104,8 @@ public class Context
103104
public string DbNotesQuery { get; set; }
104105

105106
public bool IgnoreNotes { get; set; }
106-
public Field NoteStartField { get; set; } = Field.Parse("Start", "Start");
107-
public Field NoteEndField { get; set; } = Field.Parse("End", "End");
108-
public Field NoteTextField { get; set; } = Field.Parse("NoteText", "NoteText");
107+
public Field NoteStartField { get; set; } = Field.Parse("StartTime", nameof(Context.NoteStartField));
108+
public Field NoteEndField { get; set; } = Field.Parse("EndTime", nameof(Context.NoteEndField));
109+
public Field NoteTextField { get; set; } = Field.Parse("NoteText", nameof(Context.NoteTextField));
109110
}
110111
}

‎TimeSeries/PublicApis/SdkExamples/PointZilla/CsvWriter.cs‎

Lines changed: 54 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,15 @@
1-
using System.Collections.Generic;
1+
using System;
2+
using System.Collections.Generic;
23
using System.IO;
34
using System.Linq;
45
using System.Reflection;
56
using Aquarius.TimeSeries.Client.ServiceModels.Acquisition;
67
using Humanizer;
78
using NodaTime;
89
using NodaTime.Text;
10+
using PointZilla.PointReaders;
911
using ServiceStack.Logging;
12+
using PublishNote = Aquarius.TimeSeries.Client.ServiceModels.Publish.Note;
1013

1114
namespace PointZilla
1215
{
@@ -38,6 +41,12 @@ public void WritePoints(List<TimeSeriesPoint> points, List<TimeSeriesNote> notes
3841
Directory.CreateDirectory(dir);
3942
}
4043

44+
var publishNotes = notes
45+
.Select(Convert)
46+
.ToList();
47+
48+
var notesLookup = new MetadataLookup<PublishNote>(publishNotes);
49+
4150
using (var writer = new StreamWriter(csvPath))
4251
{
4352
var offsetPattern = OffsetPattern.CreateWithInvariantCulture("m");
@@ -58,16 +67,30 @@ public void WritePoints(List<TimeSeriesPoint> points, List<TimeSeriesNote> notes
5867
writer.WriteLine($"#");
5968
writer.WriteLine($"# CSV data starts at line 15.");
6069
writer.WriteLine($"#");
61-
writer.WriteLine($"ISO 8601 UTC, Value, Grade, Qualifiers");
70+
71+
var optionalNotesHeader = Context.SaveNotesMode == SaveNotesMode.WithPoints
72+
? ", Notes"
73+
: string.Empty;
74+
75+
writer.WriteLine($"ISO 8601 UTC, Value, Grade, Qualifiers{optionalNotesHeader}");
6276

6377
foreach (var point in points)
6478
{
6579
var time = point.Time ?? Instant.MinValue;
6680

67-
writer.WriteLine($"{InstantPattern.ExtendedIsoPattern.Format(time)}, {point.Value:G12}, {point.GradeCode}, {FormatQualifiers(point.Qualifiers)}");
81+
var line = $"{InstantPattern.ExtendedIsoPattern.Format(time)}, {point.Value:G12}, {point.GradeCode}, {FormatQualifiers(point.Qualifiers)}";
82+
83+
if (Context.SaveNotesMode == SaveNotesMode.WithPoints)
84+
{
85+
var pointNotes = string.Join("\r\n", notesLookup.GetMany(time.ToDateTimeOffset()).Select(note => note.NoteText));
86+
87+
line += $", {CsvEscapedColumn(pointNotes)}";
88+
}
89+
90+
writer.WriteLine(line);
6891
}
6992

70-
if (!Context.IgnoreNotes && notes.Any())
93+
if (Context.SaveNotesMode == SaveNotesMode.SeparateCsv)
7194
{
7295
var notesCsvPath = Path.ChangeExtension(csvPath, ".Notes.csv");
7396

@@ -93,37 +116,32 @@ public void WritePoints(List<TimeSeriesPoint> points, List<TimeSeriesNote> notes
93116
if (!note.TimeRange.HasValue)
94117
continue;
95118

96-
writer.WriteLine($"{InstantPattern.ExtendedIsoPattern.Format(note.TimeRange.Value.Start)}, {InstantPattern.ExtendedIsoPattern.Format(note.TimeRange.Value.End)}, {CsvEscapedColumn(note.NoteText)}");
119+
notesWriter.WriteLine($"{InstantPattern.ExtendedIsoPattern.Format(note.TimeRange.Value.Start)}, {InstantPattern.ExtendedIsoPattern.Format(note.TimeRange.Value.End)}, {CsvEscapedColumn(note.NoteText)}");
97120
}
98121
}
99122
}
100123
}
101124
}
102125

103-
private static string CsvEscapedColumn(string text)
126+
private static PublishNote Convert(TimeSeriesNote note)
104127
{
105-
return !CharactersRequiringEscaping.Any(text.Contains)
106-
? text
107-
: $"\"{text.Replace("\"", "\"\"")}\"";
128+
return new PublishNote
129+
{
130+
StartTime = note.TimeRange?.Start.ToDateTimeOffset() ?? DateTimeOffset.MinValue,
131+
EndTime = note.TimeRange?.End.ToDateTimeOffset() ?? DateTimeOffset.MaxValue,
132+
NoteText = note.NoteText
133+
};
108134
}
109135

110-
private static readonly char[] CharactersRequiringEscaping = new[]
111-
{
112-
',',
113-
'"',
114-
'\n',
115-
'\r'
116-
};
117-
118136
public static void SetPointZillaCsvFormat(Context context)
119137
{
120138
// Match PointZilla Export format below
121139

122140
// # CSV data starts at line 15.
123141
// #
124-
// ISO 8601 UTC, Value, Grade, Qualifiers
125-
// 2015-12-04T00:01:00Z, 3.523200823975, 500,
126-
// 2015-12-04T00:02:00Z, 3.525279357147, 500,
142+
// ISO 8601 UTC, Value, Grade, Qualifiers, Notes
143+
// 2015-12-04T00:01:00Z, 3.523200823975, 500, ,
144+
// 2015-12-04T00:02:00Z, 3.525279357147, 500, ,
127145

128146
context.CsvSkipRows = 0;
129147
context.CsvComment = "#";
@@ -134,6 +152,7 @@ public static void SetPointZillaCsvFormat(Context context)
134152
context.CsvValueField = Field.Parse("Value", nameof(context.CsvValueField));
135153
context.CsvGradeField = Field.Parse("Grade", nameof(context.CsvGradeField));
136154
context.CsvQualifiersField = Field.Parse("Qualifiers", nameof(context.CsvQualifiersField));
155+
context.CsvNotesField = Field.Parse("Notes", nameof(context.CsvNotesField));
137156
context.CsvIgnoreInvalidRows = true;
138157
context.CsvRealign = false;
139158
}
@@ -194,20 +213,31 @@ private static (string StartText, string EndText) CreatePeriod(Instant start, In
194213
start == Instant.MinValue ? "StartOfRecord" : InstantPattern.ExtendedIsoPattern.Format(start),
195214
end == Instant.MaxValue ? "EndOfRecord" : InstantPattern.ExtendedIsoPattern.Format(end)
196215
);
197-
198216
}
199217

200218
private static string FormatQualifiers(List<string> qualifiers)
201219
{
202220
if (qualifiers == null || !qualifiers.Any())
203221
return string.Empty;
204222

205-
if (qualifiers.Count == 1)
206-
return qualifiers.First();
223+
return CsvEscapedColumn(string.Join(",", qualifiers));
224+
}
207225

208-
return $"\"{string.Join(",", qualifiers)}\"";
226+
private static string CsvEscapedColumn(string text)
227+
{
228+
return !CharactersRequiringEscaping.Any(text.Contains)
229+
? text
230+
: $"\"{text.Replace("\"", "\"\"")}\"";
209231
}
210232

233+
private static readonly char[] CharactersRequiringEscaping = new[]
234+
{
235+
',',
236+
'"',
237+
'\n',
238+
'\r'
239+
};
240+
211241
private static string SanitizeFilename(string s)
212242
{
213243
return Path.GetInvalidFileNameChars().Aggregate(s, (current, ch) => current.Replace(ch, '_'));

‎TimeSeries/PublicApis/SdkExamples/PointZilla/PointReaders/CsvNotesReader.cs‎

Lines changed: 117 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,117 @@
1+
using System.Collections.Generic;
2+
using System.IO;
3+
using Aquarius.TimeSeries.Client.ServiceModels.Acquisition;
4+
using Microsoft.VisualBasic.FileIO;
5+
using NodaTime;
6+
7+
namespace PointZilla.PointReaders
8+
{
9+
public class CsvNotesReader : CsvReaderBase
10+
{
11+
public CsvNotesReader(Context context)
12+
: base(context)
13+
{
14+
}
15+
16+
public List<TimeSeriesNote> LoadNotes()
17+
{
18+
if (string.IsNullOrEmpty(Context.CsvNotesFile))
19+
return new List<TimeSeriesNote>();
20+
21+
if (!File.Exists(Context.CsvNotesFile))
22+
throw new ExpectedException($"File '{Context.CsvNotesFile}' does not exist.");
23+
24+
return LoadNotes(Context.CsvNotesFile);
25+
}
26+
27+
private List<TimeSeriesNote> LoadNotes(string path)
28+
{
29+
var notes = new List<TimeSeriesNote>();
30+
31+
var csvDelimiter = string.IsNullOrEmpty(Context.CsvDelimiter)
32+
? ","
33+
: Context.CsvDelimiter;
34+
35+
var parser = new TextFieldParser(path)
36+
{
37+
TextFieldType = FieldType.Delimited,
38+
Delimiters = new[] { csvDelimiter },
39+
TrimWhiteSpace = true,
40+
HasFieldsEnclosedInQuotes = true
41+
};
42+
43+
if (!string.IsNullOrWhiteSpace(Context.CsvComment))
44+
{
45+
parser.CommentTokens = new[] { Context.CsvComment };
46+
}
47+
48+
var skipCount = Context.CsvSkipRows;
49+
50+
var parseHeaderRow = Context.CsvHasHeaderRow;
51+
52+
while (!parser.EndOfData)
53+
{
54+
var lineNumber = parser.LineNumber;
55+
56+
var fields = parser.ReadFields();
57+
if (fields == null) continue;
58+
59+
if (skipCount > 0)
60+
{
61+
--skipCount;
62+
continue;
63+
}
64+
65+
if (parseHeaderRow)
66+
{
67+
ValidateHeaderFields(fields, new List<Field>
68+
{
69+
Context.NoteStartField,
70+
Context.NoteEndField,
71+
Context.NoteTextField,
72+
});
73+
parseHeaderRow = false;
74+
75+
if (Context.CsvHasHeaderRow)
76+
continue;
77+
}
78+
79+
var note = ParseNote(fields);
80+
81+
if (note == null)
82+
{
83+
if (Context.CsvIgnoreInvalidRows) continue;
84+
85+
throw new ExpectedException($"Can't parse '{path}' ({lineNumber}): {string.Join(", ", fields)}");
86+
}
87+
88+
notes.Add(note);
89+
}
90+
91+
return notes;
92+
}
93+
94+
private TimeSeriesNote ParseNote(string[] fields)
95+
{
96+
Instant? start = null;
97+
Instant? end = null;
98+
var noteText = default(string);
99+
100+
ParseField(fields, Context.NoteStartField.ColumnIndex, text => start = ParseInstant(text));
101+
ParseField(fields, Context.NoteEndField.ColumnIndex, text => end = ParseInstant(text));
102+
ParseField(fields, Context.NoteTextField.ColumnIndex, text => noteText = text);
103+
104+
if (!start.HasValue || !end.HasValue || string.IsNullOrWhiteSpace(noteText))
105+
return null;
106+
107+
if (end < start)
108+
return null;
109+
110+
return new TimeSeriesNote
111+
{
112+
TimeRange = new Interval(start.Value, end.Value),
113+
NoteText = noteText
114+
};
115+
}
116+
}
117+
}

0 commit comments

Comments
 (0)