Expose the tokeniser to the public API

Sledge.Formats/Tokens/ITokenReader.cs

@@ -0,0 +1,12 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Text;
+
+namespace Sledge.Formats.Tokens
+{
+    public interface ITokenReader
+    {
+        Token Read(char start, TextReader reader);
+    }
+}

Sledge.Formats/Valve/ValveSymbols.cs → Sledge.Formats/Tokens/Symbols.cs

@@ -1,7 +1,7 @@
 // ReSharper disable MemberCanBePrivate.Global
-namespace Sledge.Formats.Valve
+namespace Sledge.Formats.Tokens
 {
-    public static class ValveSymbols
+    public static class Symbols
     {
         public const char Bang = '!';
         public const char At = '@';
@@ -37,8 +37,8 @@ public static class ValveSymbols
         public const char Greater = '>';
         public const char Question = '?';
 
-        public static char[] All = new []
-        {
+        // ReSharper disable once UnusedMember.Global
+        public static char[] All = {
             Bang, 
             At, 
             Hash, 

Sledge.Formats/Tokens/Token.cs

@@ -0,0 +1,35 @@
+using System;
+
+namespace Sledge.Formats.Tokens
+{
+    public class Token
+    {
+        public TokenType Type { get; }
+        public string CustomType { get; }
+        public string Value { get; }
+        public int Line { get; set; }
+        public int Column { get; set; }
+
+        public char Symbol
+        {
+            get
+            {
+                if (Type != TokenType.Symbol || Value == null || Value.Length != 1) throw new ArgumentException($"Not a symbol: {Type}({Value})");
+                return Value[0];
+            }
+        }
+
+        public Token(TokenType type, string value = null)
+        {
+            Type = type;
+            Value = value;
+        }
+
+        public Token(string customType, string value = null)
+        {
+            Type = TokenType.Custom;
+            CustomType = customType;
+            Value = value;
+        }
+    }
+}

Sledge.Formats/Valve/ValveTokenType.cs → Sledge.Formats/Tokens/TokenType.cs

@@ -1,11 +1,12 @@
-namespace Sledge.Formats.Valve
+namespace Sledge.Formats.Tokens
 {
-    internal enum ValveTokenType
+    public enum TokenType
     {
         Invalid,
         Symbol,
         Name,
         String,
+        Custom,
         End
     }
 }

Sledge.Formats/Valve/ValveTokeniser.cs → Sledge.Formats/Tokens/Tokeniser.cs

@@ -1,24 +1,32 @@
-using System;
-using System.Collections.Generic;
+using System.Collections.Generic;
 using System.IO;
-using System.Text;
 using System.Linq;
+using System.Text;
 
-namespace Sledge.Formats.Valve
+namespace Sledge.Formats.Tokens
 {
-    internal static class ValveTokeniser
+    public class Tokeniser
     {
-        public static IEnumerable<ValveToken> Tokenise(string text, IEnumerable<char> symbols)
+        private readonly HashSet<int> _symbolSet;
+        public List<ITokenReader> CustomReaders { get; }
+
+        public Tokeniser(IEnumerable<char> symbols)
+        {
+            _symbolSet = new HashSet<int>(symbols.Select(x => (int) x));
+            CustomReaders = new List<ITokenReader>();
+        }
+
+        public IEnumerable<Token> Tokenise(string text)
         {
             using (var reader = new StringReader(text))
             {
-                foreach (var t in Tokenise(reader, symbols)) yield return t;
+                foreach (var t in Tokenise(reader)) yield return t;
             }
         }
 
-        internal static IEnumerable<ValveToken> Tokenise(TextReader input, IEnumerable<char> symbols)
+        public IEnumerable<Token> Tokenise(TextReader input)
         {
-            var symbolSet = new HashSet<int>(symbols.Select(x => (int) x));
+            var custom = CustomReaders.Any();
             var line = 1;
             var col = 0;
             int b;
@@ -64,30 +72,43 @@ internal static IEnumerable<ValveToken> Tokenise(TextReader input, IEnumerable<c
                     }
 
                     // It's not a comment, so it's invalid
-                    yield return new ValveToken(ValveTokenType.Invalid, $"Unexpected token: {(char) b}") {Line = line, Column = col};
+                    yield return new Token(TokenType.Invalid, $"Unexpected token: {(char) b}") {Line = line, Column = col};
                 }
 
-                ValveToken t;
-                if (b == '"') t = TokenString(input);
-                else if (symbolSet.Contains(b)) t = new ValveToken(ValveTokenType.Symbol, ((char) b).ToString());
+                Token t;
+                if (custom && ReadCustom(b, input, out var ct)) t = ct;
+                else if (b == '"') t = TokenString(input);
+                else if (_symbolSet.Contains(b)) t = new Token(TokenType.Symbol, ((char) b).ToString());
                 else if (b >= 'a' && b <= 'z' || (b >= 'A' && b <= 'Z') || b == '_') t = TokenName(b, input);
-                else t = new ValveToken(ValveTokenType.Invalid, $"Unexpected token: {(char) b}");
+                else t = new Token(TokenType.Invalid, $"Unexpected token: {(char) b}");
 
                 t.Line = line;
                 t.Column = col;
 
                 yield return t;
 
-                if (t.Type == ValveTokenType.Invalid)
+                if (t.Type == TokenType.Invalid)
                 {
                     yield break;
                 }
             }
 
-            yield return new ValveToken(ValveTokenType.End);
+            yield return new Token(TokenType.End);
+        }
+
+        private bool ReadCustom(int start, TextReader input, out Token token)
+        {
+            foreach (var reader in CustomReaders)
+            {
+                token = reader.Read((char) start, input);
+                if (token != null) return true;
+            }
+
+            token = null;
+            return false;
         }
 
-        private static ValveToken TokenString(TextReader input)
+        private static Token TokenString(TextReader input)
         {
             var sb = new StringBuilder();
             int b;
@@ -96,20 +117,20 @@ private static ValveToken TokenString(TextReader input)
                 // Newline in string (not allowed)
                 if (b == '\n')
                 {
-                    return new ValveToken(ValveTokenType.Invalid, "String cannot contain a newline");
+                    return new Token(TokenType.Invalid, "String cannot contain a newline");
                 }
                 // End of string
                 else if (b == '"')
                 {
-                    return new ValveToken(ValveTokenType.String, sb.ToString());
+                    return new Token(TokenType.String, sb.ToString());
                 }
                 // Escaped character
                 else if (b == '\\')
                 {
                     // Read the next character
                     b = input.Read();
                     // EOF reached
-                    if (b < 0) return new ValveToken(ValveTokenType.Invalid, "Unexpected end of file while reading string value");
+                    if (b < 0) return new Token(TokenType.Invalid, "Unexpected end of file while reading string value");
                     // Some common escaped characters
                     else if (b == 'n') sb.Append('\n'); // newline
                     else if (b == 'r') sb.Append('\r'); // return
@@ -124,10 +145,10 @@ private static ValveToken TokenString(TextReader input)
                 }
             }
 
-            return new ValveToken(ValveTokenType.Invalid, "Unexpected end of file while reading string value");
+            return new Token(TokenType.Invalid, "Unexpected end of file while reading string value");
         }
 
-        private static ValveToken TokenName(int first, TextReader input)
+        private static Token TokenName(int first, TextReader input)
         {
             var name = ((char) first).ToString();
             int b;
@@ -144,7 +165,7 @@ private static ValveToken TokenName(int first, TextReader input)
                 }
             }
 
-            return new ValveToken(ValveTokenType.Name, name);
+            return new Token(TokenType.Name, name);
         }
     }
 }

Sledge.Formats/Valve/SerialisedObjectFormatter.cs

@@ -3,6 +3,7 @@
 using System.IO;
 using System.Linq;
 using System.Text;
+using Sledge.Formats.Tokens;
 
 namespace Sledge.Formats.Valve
 {
@@ -102,10 +103,12 @@ private static void Print(SerialisedObject obj, TextWriter tw, int tabs = 0)
         #region Parser
 
         private static readonly char[] Symbols = {
-            ValveSymbols.OpenBrace,
-            ValveSymbols.CloseBrace
+            Tokens.Symbols.OpenBrace,
+            Tokens.Symbols.CloseBrace
         };
 
+        private static readonly Tokeniser Tokeniser = new Tokeniser(Symbols);
+
         /// <summary>
         /// Parse a structure from a stream
         /// </summary>
@@ -116,22 +119,22 @@ public static IEnumerable<SerialisedObject> Parse(TextReader reader)
             SerialisedObject current = null;
             var stack = new Stack<SerialisedObject>();
 
-            var tokens = ValveTokeniser.Tokenise(reader, Symbols);
+            var tokens = Tokeniser.Tokenise(reader);
             using (var it = tokens.GetEnumerator())
             {
                 while (it.MoveNext())
                 {
                     var t = it.Current;
                     switch (t?.Type)
                     {
-                        case ValveTokenType.Invalid:
+                        case TokenType.Invalid:
                             throw new Exception($"Parsing error (line {t.Line}, column {t.Column}): {t.Value}");
-                        case ValveTokenType.Symbol:
-                            if (t.Symbol == ValveSymbols.OpenBrace)
+                        case TokenType.Symbol:
+                            if (t.Symbol == Tokens.Symbols.OpenBrace)
                             {
                                 throw new Exception($"Parsing error (line {t.Line}, column {t.Column}): Structure must have a name");
                             }
-                            else if (t.Symbol == ValveSymbols.CloseBrace)
+                            else if (t.Symbol == Tokens.Symbols.CloseBrace)
                             {
                                 if (current == null) throw new Exception($"Parsing error (line {t.Line}, column {t.Column}): No structure to close");
                                 if (stack.Count == 0)
@@ -152,8 +155,8 @@ public static IEnumerable<SerialisedObject> Parse(TextReader reader)
                             {
                                 throw new ArgumentOutOfRangeException();
                             }
-                        case ValveTokenType.Name:
-                            if (!it.MoveNext() || it.Current == null || it.Current.Type != ValveTokenType.Symbol || it.Current.Symbol != ValveSymbols.OpenBrace)
+                        case TokenType.Name:
+                            if (!it.MoveNext() || it.Current == null || it.Current.Type != TokenType.Symbol || it.Current.Symbol != Tokens.Symbols.OpenBrace)
                             {
                                 throw new Exception($"Parsing error (line {t.Line}, column {t.Column}): Expected structure open brace");
                             }
@@ -168,14 +171,14 @@ public static IEnumerable<SerialisedObject> Parse(TextReader reader)
                                 current = next;
                             }
                             break;
-                        case ValveTokenType.String:
+                        case TokenType.String:
                             if (current == null) throw new Exception($"Parsing error (line {t.Line}, column {t.Column}): No structure to add key/values to");
                             var key = t.Value;
-                            if (!it.MoveNext() || it.Current == null || it.Current.Type != ValveTokenType.String) throw new Exception($"Parsing error (line {t.Line}, column {t.Column}): Expected string value to follow key");
+                            if (!it.MoveNext() || it.Current == null || it.Current.Type != TokenType.String) throw new Exception($"Parsing error (line {t.Line}, column {t.Column}): Expected string value to follow key");
                             var value = it.Current.Value;
                             current.Properties.Add(new KeyValuePair<string, string>(key, value));
                             break;
-                        case ValveTokenType.End:
+                        case TokenType.End:
                             if (current != null) throw new Exception($"Parsing error (line {t.Line}, column {t.Column}): Unterminated structure at end of file");
                             yield break;
                         default: