From 7e50cce31040cfed8f7fde5f9e75052896f07a48 Mon Sep 17 00:00:00 2001 From: Daniel Walder Date: Tue, 30 Nov 2021 19:34:45 +1000 Subject: [PATCH] Expose the tokeniser to the public API --- Sledge.Formats/Tokens/ITokenReader.cs | 12 ++++ .../ValveSymbols.cs => Tokens/Symbols.cs} | 8 +-- Sledge.Formats/Tokens/Token.cs | 35 ++++++++++ .../ValveTokenType.cs => Tokens/TokenType.cs} | 5 +- .../ValveTokeniser.cs => Tokens/Tokeniser.cs} | 67 ++++++++++++------- .../Valve/SerialisedObjectFormatter.cs | 27 ++++---- Sledge.Formats/Valve/ValveToken.cs | 27 -------- 7 files changed, 113 insertions(+), 68 deletions(-) create mode 100644 Sledge.Formats/Tokens/ITokenReader.cs rename Sledge.Formats/{Valve/ValveSymbols.cs => Tokens/Symbols.cs} (92%) create mode 100644 Sledge.Formats/Tokens/Token.cs rename Sledge.Formats/{Valve/ValveTokenType.cs => Tokens/TokenType.cs} (53%) rename Sledge.Formats/{Valve/ValveTokeniser.cs => Tokens/Tokeniser.cs} (62%) delete mode 100644 Sledge.Formats/Valve/ValveToken.cs diff --git a/Sledge.Formats/Tokens/ITokenReader.cs b/Sledge.Formats/Tokens/ITokenReader.cs new file mode 100644 index 0000000..a56ed15 --- /dev/null +++ b/Sledge.Formats/Tokens/ITokenReader.cs @@ -0,0 +1,12 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Text; + +namespace Sledge.Formats.Tokens +{ + public interface ITokenReader + { + Token Read(char start, TextReader reader); + } +} diff --git a/Sledge.Formats/Valve/ValveSymbols.cs b/Sledge.Formats/Tokens/Symbols.cs similarity index 92% rename from Sledge.Formats/Valve/ValveSymbols.cs rename to Sledge.Formats/Tokens/Symbols.cs index 3b61cfe..4a9744a 100644 --- a/Sledge.Formats/Valve/ValveSymbols.cs +++ b/Sledge.Formats/Tokens/Symbols.cs @@ -1,7 +1,7 @@ // ReSharper disable MemberCanBePrivate.Global -namespace Sledge.Formats.Valve +namespace Sledge.Formats.Tokens { - public static class ValveSymbols + public static class Symbols { public const char Bang = '!'; public const char At = '@'; @@ -37,8 +37,8 @@ public static class ValveSymbols public const char Greater = '>'; public const char Question = '?'; - public static char[] All = new [] - { + // ReSharper disable once UnusedMember.Global + public static char[] All = { Bang, At, Hash, diff --git a/Sledge.Formats/Tokens/Token.cs b/Sledge.Formats/Tokens/Token.cs new file mode 100644 index 0000000..d19d9be --- /dev/null +++ b/Sledge.Formats/Tokens/Token.cs @@ -0,0 +1,35 @@ +using System; + +namespace Sledge.Formats.Tokens +{ + public class Token + { + public TokenType Type { get; } + public string CustomType { get; } + public string Value { get; } + public int Line { get; set; } + public int Column { get; set; } + + public char Symbol + { + get + { + if (Type != TokenType.Symbol || Value == null || Value.Length != 1) throw new ArgumentException($"Not a symbol: {Type}({Value})"); + return Value[0]; + } + } + + public Token(TokenType type, string value = null) + { + Type = type; + Value = value; + } + + public Token(string customType, string value = null) + { + Type = TokenType.Custom; + CustomType = customType; + Value = value; + } + } +} diff --git a/Sledge.Formats/Valve/ValveTokenType.cs b/Sledge.Formats/Tokens/TokenType.cs similarity index 53% rename from Sledge.Formats/Valve/ValveTokenType.cs rename to Sledge.Formats/Tokens/TokenType.cs index 5043715..2e7c9c1 100644 --- a/Sledge.Formats/Valve/ValveTokenType.cs +++ b/Sledge.Formats/Tokens/TokenType.cs @@ -1,11 +1,12 @@ -namespace Sledge.Formats.Valve +namespace Sledge.Formats.Tokens { - internal enum ValveTokenType + public enum TokenType { Invalid, Symbol, Name, String, + Custom, End } } \ No newline at end of file diff --git a/Sledge.Formats/Valve/ValveTokeniser.cs b/Sledge.Formats/Tokens/Tokeniser.cs similarity index 62% rename from Sledge.Formats/Valve/ValveTokeniser.cs rename to Sledge.Formats/Tokens/Tokeniser.cs index 1c5882b..186e330 100644 --- a/Sledge.Formats/Valve/ValveTokeniser.cs +++ b/Sledge.Formats/Tokens/Tokeniser.cs @@ -1,24 +1,32 @@ -using System; -using System.Collections.Generic; +using System.Collections.Generic; using System.IO; -using System.Text; using System.Linq; +using System.Text; -namespace Sledge.Formats.Valve +namespace Sledge.Formats.Tokens { - internal static class ValveTokeniser + public class Tokeniser { - public static IEnumerable Tokenise(string text, IEnumerable symbols) + private readonly HashSet _symbolSet; + public List CustomReaders { get; } + + public Tokeniser(IEnumerable symbols) + { + _symbolSet = new HashSet(symbols.Select(x => (int) x)); + CustomReaders = new List(); + } + + public IEnumerable Tokenise(string text) { using (var reader = new StringReader(text)) { - foreach (var t in Tokenise(reader, symbols)) yield return t; + foreach (var t in Tokenise(reader)) yield return t; } } - internal static IEnumerable Tokenise(TextReader input, IEnumerable symbols) + public IEnumerable Tokenise(TextReader input) { - var symbolSet = new HashSet(symbols.Select(x => (int) x)); + var custom = CustomReaders.Any(); var line = 1; var col = 0; int b; @@ -64,30 +72,43 @@ internal static IEnumerable Tokenise(TextReader input, IEnumerable= 'a' && b <= 'z' || (b >= 'A' && b <= 'Z') || b == '_') t = TokenName(b, input); - else t = new ValveToken(ValveTokenType.Invalid, $"Unexpected token: {(char) b}"); + else t = new Token(TokenType.Invalid, $"Unexpected token: {(char) b}"); t.Line = line; t.Column = col; yield return t; - if (t.Type == ValveTokenType.Invalid) + if (t.Type == TokenType.Invalid) { yield break; } } - yield return new ValveToken(ValveTokenType.End); + yield return new Token(TokenType.End); + } + + private bool ReadCustom(int start, TextReader input, out Token token) + { + foreach (var reader in CustomReaders) + { + token = reader.Read((char) start, input); + if (token != null) return true; + } + + token = null; + return false; } - private static ValveToken TokenString(TextReader input) + private static Token TokenString(TextReader input) { var sb = new StringBuilder(); int b; @@ -96,12 +117,12 @@ private static ValveToken TokenString(TextReader input) // Newline in string (not allowed) if (b == '\n') { - return new ValveToken(ValveTokenType.Invalid, "String cannot contain a newline"); + return new Token(TokenType.Invalid, "String cannot contain a newline"); } // End of string else if (b == '"') { - return new ValveToken(ValveTokenType.String, sb.ToString()); + return new Token(TokenType.String, sb.ToString()); } // Escaped character else if (b == '\\') @@ -109,7 +130,7 @@ private static ValveToken TokenString(TextReader input) // Read the next character b = input.Read(); // EOF reached - if (b < 0) return new ValveToken(ValveTokenType.Invalid, "Unexpected end of file while reading string value"); + if (b < 0) return new Token(TokenType.Invalid, "Unexpected end of file while reading string value"); // Some common escaped characters else if (b == 'n') sb.Append('\n'); // newline else if (b == 'r') sb.Append('\r'); // return @@ -124,10 +145,10 @@ private static ValveToken TokenString(TextReader input) } } - return new ValveToken(ValveTokenType.Invalid, "Unexpected end of file while reading string value"); + return new Token(TokenType.Invalid, "Unexpected end of file while reading string value"); } - private static ValveToken TokenName(int first, TextReader input) + private static Token TokenName(int first, TextReader input) { var name = ((char) first).ToString(); int b; @@ -144,7 +165,7 @@ private static ValveToken TokenName(int first, TextReader input) } } - return new ValveToken(ValveTokenType.Name, name); + return new Token(TokenType.Name, name); } } } \ No newline at end of file diff --git a/Sledge.Formats/Valve/SerialisedObjectFormatter.cs b/Sledge.Formats/Valve/SerialisedObjectFormatter.cs index 449ae8a..a98ba7f 100644 --- a/Sledge.Formats/Valve/SerialisedObjectFormatter.cs +++ b/Sledge.Formats/Valve/SerialisedObjectFormatter.cs @@ -3,6 +3,7 @@ using System.IO; using System.Linq; using System.Text; +using Sledge.Formats.Tokens; namespace Sledge.Formats.Valve { @@ -102,10 +103,12 @@ private static void Print(SerialisedObject obj, TextWriter tw, int tabs = 0) #region Parser private static readonly char[] Symbols = { - ValveSymbols.OpenBrace, - ValveSymbols.CloseBrace + Tokens.Symbols.OpenBrace, + Tokens.Symbols.CloseBrace }; + private static readonly Tokeniser Tokeniser = new Tokeniser(Symbols); + /// /// Parse a structure from a stream /// @@ -116,7 +119,7 @@ public static IEnumerable Parse(TextReader reader) SerialisedObject current = null; var stack = new Stack(); - var tokens = ValveTokeniser.Tokenise(reader, Symbols); + var tokens = Tokeniser.Tokenise(reader); using (var it = tokens.GetEnumerator()) { while (it.MoveNext()) @@ -124,14 +127,14 @@ public static IEnumerable Parse(TextReader reader) var t = it.Current; switch (t?.Type) { - case ValveTokenType.Invalid: + case TokenType.Invalid: throw new Exception($"Parsing error (line {t.Line}, column {t.Column}): {t.Value}"); - case ValveTokenType.Symbol: - if (t.Symbol == ValveSymbols.OpenBrace) + case TokenType.Symbol: + if (t.Symbol == Tokens.Symbols.OpenBrace) { throw new Exception($"Parsing error (line {t.Line}, column {t.Column}): Structure must have a name"); } - else if (t.Symbol == ValveSymbols.CloseBrace) + else if (t.Symbol == Tokens.Symbols.CloseBrace) { if (current == null) throw new Exception($"Parsing error (line {t.Line}, column {t.Column}): No structure to close"); if (stack.Count == 0) @@ -152,8 +155,8 @@ public static IEnumerable Parse(TextReader reader) { throw new ArgumentOutOfRangeException(); } - case ValveTokenType.Name: - if (!it.MoveNext() || it.Current == null || it.Current.Type != ValveTokenType.Symbol || it.Current.Symbol != ValveSymbols.OpenBrace) + case TokenType.Name: + if (!it.MoveNext() || it.Current == null || it.Current.Type != TokenType.Symbol || it.Current.Symbol != Tokens.Symbols.OpenBrace) { throw new Exception($"Parsing error (line {t.Line}, column {t.Column}): Expected structure open brace"); } @@ -168,14 +171,14 @@ public static IEnumerable Parse(TextReader reader) current = next; } break; - case ValveTokenType.String: + case TokenType.String: if (current == null) throw new Exception($"Parsing error (line {t.Line}, column {t.Column}): No structure to add key/values to"); var key = t.Value; - if (!it.MoveNext() || it.Current == null || it.Current.Type != ValveTokenType.String) throw new Exception($"Parsing error (line {t.Line}, column {t.Column}): Expected string value to follow key"); + if (!it.MoveNext() || it.Current == null || it.Current.Type != TokenType.String) throw new Exception($"Parsing error (line {t.Line}, column {t.Column}): Expected string value to follow key"); var value = it.Current.Value; current.Properties.Add(new KeyValuePair(key, value)); break; - case ValveTokenType.End: + case TokenType.End: if (current != null) throw new Exception($"Parsing error (line {t.Line}, column {t.Column}): Unterminated structure at end of file"); yield break; default: diff --git a/Sledge.Formats/Valve/ValveToken.cs b/Sledge.Formats/Valve/ValveToken.cs deleted file mode 100644 index fec4fe2..0000000 --- a/Sledge.Formats/Valve/ValveToken.cs +++ /dev/null @@ -1,27 +0,0 @@ -using System; - -namespace Sledge.Formats.Valve -{ - internal class ValveToken - { - public ValveTokenType Type { get; } - public string Value { get; } - public int Line { get; set; } - public int Column { get; set; } - - public char Symbol - { - get - { - if (Type != ValveTokenType.Symbol || Value == null || Value.Length != 1) throw new ArgumentException($"Not a symbol: {Type}({Value})"); - return Value[0]; - } - } - - public ValveToken(ValveTokenType type, string value = null) - { - Type = type; - Value = value; - } - } -} \ No newline at end of file