Make the tokeniser a bit more flexible

Sledge.Formats/Valve/SerialisedObjectFormatter.cs

@@ -101,6 +101,11 @@ private static void Print(SerialisedObject obj, TextWriter tw, int tabs = 0)
 
         #region Parser
 
+        private static readonly char[] Symbols = {
+            ValveSymbols.OpenBrace,
+            ValveSymbols.CloseBrace
+        };
+
         /// <summary>
         /// Parse a structure from a stream
         /// </summary>
@@ -111,7 +116,7 @@ public static IEnumerable<SerialisedObject> Parse(TextReader reader)
             SerialisedObject current = null;
             var stack = new Stack<SerialisedObject>();
 
-            var tokens = ValveTokeniser.Tokenise(reader);
+            var tokens = ValveTokeniser.Tokenise(reader, Symbols);
             using (var it = tokens.GetEnumerator())
             {
                 while (it.MoveNext())
@@ -121,24 +126,37 @@ public static IEnumerable<SerialisedObject> Parse(TextReader reader)
                     {
                         case ValveTokenType.Invalid:
                             throw new Exception($"Parsing error (line {t.Line}, column {t.Column}): {t.Value}");
-                        case ValveTokenType.Open:
-                            throw new Exception($"Parsing error (line {t.Line}, column {t.Column}): Structure must have a name");
-                        case ValveTokenType.Close:
-                            if (current == null) throw new Exception($"Parsing error (line {t.Line}, column {t.Column}): No structure to close");
-                            if (stack.Count == 0)
+                        case ValveTokenType.Symbol:
+                            if (t.Symbol == ValveSymbols.OpenBrace)
+                            {
+                                throw new Exception($"Parsing error (line {t.Line}, column {t.Column}): Structure must have a name");
+                            }
+                            else if (t.Symbol == ValveSymbols.CloseBrace)
                             {
-                                yield return current;
-                                current = null;
+                                if (current == null) throw new Exception($"Parsing error (line {t.Line}, column {t.Column}): No structure to close");
+                                if (stack.Count == 0)
+                                {
+                                    yield return current;
+                                    current = null;
+                                }
+                                else
+                                {
+                                    var prev = stack.Pop();
+                                    prev.Children.Add(current);
+                                    current = prev;
+                                }
+
+                                break;
                             }
                             else
                             {
-                                var prev = stack.Pop();
-                                prev.Children.Add(current);
-                                current = prev;
+                                throw new ArgumentOutOfRangeException();
                             }
-                            break;
                         case ValveTokenType.Name:
-                            if (!it.MoveNext() || it.Current == null || it.Current.Type != ValveTokenType.Open) throw new Exception($"Parsing error (line {t.Line}, column {t.Column}): Expected structure open brace");
+                            if (!it.MoveNext() || it.Current == null || it.Current.Type != ValveTokenType.Symbol || it.Current.Symbol != ValveSymbols.OpenBrace)
+                            {
+                                throw new Exception($"Parsing error (line {t.Line}, column {t.Column}): Expected structure open brace");
+                            }
                             var next = new SerialisedObject(t.Value);
                             if (current == null)
                             {

Sledge.Formats/Valve/ValveSymbols.cs

@@ -0,0 +1,74 @@
+// ReSharper disable MemberCanBePrivate.Global
+namespace Sledge.Formats.Valve
+{
+    public static class ValveSymbols
+    {
+        public const char Bang = '!';
+        public const char At = '@';
+        public const char Hash = '#';
+        public const char Dollar = '$';
+        public const char Percent = '%';
+        public const char Caret = '^';
+        public const char Ampersand = '&';
+        public const char Star = '*';
+
+        public const char OpenParen = '(';
+        public const char CloseParen = ')';
+        public const char OpenBracket = '[';
+        public const char CloseBracket = ']';
+        public const char OpenBrace = '{';
+        public const char CloseBrace = '}';
+
+        public const char Equal = '=';
+        public const char Plus = '+';
+        public const char Minus = '-';
+        public const char Underscore = '_';
+        public const char Backslash = '/';
+        public const char Pipe = '|';
+        public const char Colon = ':';
+        public const char Semicolon = ';';
+        public const char DoubleQuote = '"';
+        public const char SingleQuote = '\'';
+
+        public const char Comma = ',';
+        public const char Dot = ',';
+        public const char Slash = '/';
+        public const char Less = '<';
+        public const char Greater = '>';
+        public const char Question = '?';
+
+        public static char[] All = new []
+        {
+            Bang, 
+            At, 
+            Hash, 
+            Dollar, 
+            Percent, 
+            Caret, 
+            Ampersand, 
+            Star, 
+            OpenParen, 
+            CloseParen, 
+            OpenBracket, 
+            CloseBracket, 
+            OpenBrace, 
+            CloseBrace, 
+            Equal, 
+            Plus, 
+            Minus, 
+            Underscore, 
+            Backslash, 
+            Pipe, 
+            Colon, 
+            Semicolon, 
+            DoubleQuote, 
+            SingleQuote, 
+            Comma, 
+            Dot, 
+            Slash, 
+            Less, 
+            Greater, 
+            Question, 
+        };
+    }
+}

Sledge.Formats/Valve/ValveToken.cs

@@ -1,4 +1,6 @@
-namespace Sledge.Formats.Valve
+using System;
+
+namespace Sledge.Formats.Valve
 {
     internal class ValveToken
     {
@@ -7,6 +9,15 @@ internal class ValveToken
         public int Line { get; set; }
         public int Column { get; set; }
 
+        public char Symbol
+        {
+            get
+            {
+                if (Type != ValveTokenType.Symbol || Value == null || Value.Length != 1) throw new ArgumentException($"Not a symbol: {Type}({Value})");
+                return Value[0];
+            }
+        }
+
         public ValveToken(ValveTokenType type, string value = null)
         {
             Type = type;

Sledge.Formats/Valve/ValveTokenType.cs

@@ -3,8 +3,7 @@
     internal enum ValveTokenType
     {
         Invalid,
-        Open,
-        Close,
+        Symbol,
         Name,
         String,
         End

Sledge.Formats/Valve/ValveTokeniser.cs

@@ -2,21 +2,23 @@
 using System.Collections.Generic;
 using System.IO;
 using System.Text;
+using System.Linq;
 
 namespace Sledge.Formats.Valve
 {
     internal static class ValveTokeniser
     {
-        public static IEnumerable<ValveToken> Tokenise(string text)
+        public static IEnumerable<ValveToken> Tokenise(string text, IEnumerable<char> symbols)
         {
             using (var reader = new StringReader(text))
             {
-                foreach (var t in Tokenise(reader)) yield return t;
+                foreach (var t in Tokenise(reader, symbols)) yield return t;
             }
         }
 
-        internal static IEnumerable<ValveToken> Tokenise(TextReader input)
+        internal static IEnumerable<ValveToken> Tokenise(TextReader input, IEnumerable<char> symbols)
         {
+            var symbolSet = new HashSet<int>(symbols.Select(x => (int) x));
             var line = 1;
             var col = 0;
             int b;
@@ -67,8 +69,7 @@ internal static IEnumerable<ValveToken> Tokenise(TextReader input)
 
                 ValveToken t;
                 if (b == '"') t = TokenString(input);
-                else if (b == '{') t = new ValveToken(ValveTokenType.Open);
-                else if (b == '}') t = new ValveToken(ValveTokenType.Close);
+                else if (symbolSet.Contains(b)) t = new ValveToken(ValveTokenType.Symbol, ((char) b).ToString());
                 else if (b >= 'a' && b <= 'z' || (b >= 'A' && b <= 'Z') || b == '_') t = TokenName(b, input);
                 else t = new ValveToken(ValveTokenType.Invalid, $"Unexpected token: {(char) b}");