Merge pull request #92 from sesteel/master

Removed regex use from plain text tokenizer

Author: sesteel

gradle.properties

@@ -1,6 +1,6 @@
 # Project details
 group=com.readytalk
-version=2.0.0
+version=2.0.1
 
 
 # Optimize the build environment

src/integTest/java/com/readytalk/swt/text/painter/TextPainterIntegTest.java

@@ -31,9 +31,9 @@ public void setup() {
   }
 
   @Test
-  public void setText_ConfiguredToTokenizePlainText_SeventyOneTokens () {
+  public void setText_ConfiguredToTokenizePlainText_SixtyNineTokens () {
     painter.setText(PLAIN_TEXT);
-    assertEquals(71, painter.getTokens().size());
+    assertEquals(69, painter.getTokens().size());
   }
 
   @Test

src/main/java/com/readytalk/swt/text/tokenizer/PlainTextTokenizer.java

@@ -7,12 +7,6 @@
 
 class PlainTextTokenizer implements TextTokenizer {
 
-  private static final String NEWLINE_SAFETY_TOKEN = "<:br:>";
-  private static final String SPACE_SAFETY_TOKEN = "<:sp:>";
-  private static final String SPACE_SPLITABLE_TOKEN = "<:ssp:>";
-  private static final String NEWLINE = "\n";
-  private static final String WHITESPACE_REGEX = "\\s+";
-
   private List<TextToken> tokens = new ArrayList<TextToken>();
 
   @Override
@@ -21,23 +15,38 @@ public TextTokenizer reset() {
     return this;
   }
 
+  /**
+   * Tokenize will accept a string and return a list of TextTokens.
+   * The algorithm used by PlaintextTokenizer#tokenize will convert all
+   * whitespace to a ' ' character.  Furthermore, subsequent whitespace
+   * characters will be collapsed into a single ' ' character.
+   *
+   * @param text
+   * @return List
+   */
   public List<TextToken> tokenize(String text) {
 
     if (text == null || "".equals(text)) {
       return tokens;
     }
 
-    String str = text.replace(NEWLINE, NEWLINE_SAFETY_TOKEN);
-    str = str.replaceAll(WHITESPACE_REGEX, SPACE_SPLITABLE_TOKEN);
-    for (String s : str.split(SPACE_SPLITABLE_TOKEN)) {
-      if (s.contains(NEWLINE_SAFETY_TOKEN)) {
-        tokens.add(new TextToken(TextType.TEXT, s.replace(NEWLINE_SAFETY_TOKEN, NEWLINE)));
+    StringBuffer sb = new StringBuffer();
+    char[] chars = text.toCharArray();
+    for (char c : chars) {
+      if (!Character.isWhitespace(c)) {
+        sb.append(c);
       } else {
-        tokens.add(new TextToken(TextType.TEXT, s));
-        tokens.add(new TextToken(TextType.WHITESPACE, " "));
+        if (sb.length() > 0) {
+          tokens.add(new TextToken(TextType.TEXT, sb.toString()));
+          tokens.add(new TextToken(TextType.WHITESPACE, " "));
+          sb = new StringBuffer();
+        }
       }
     }
 
+    if (sb.length() > 0) {
+      tokens.add(new TextToken(TextType.TEXT, sb.toString()));
+    }
     return tokens;
   }
 }

src/test/java/com/readytalk/swt/text/tokenizer/TextTokenizerTest.java

@@ -21,7 +21,7 @@ public void tokenize_MultipleTokenizeCalls_TokenListContainsMoreElements() {
 		textTokenizer.tokenize(PLAIN_TEXT);
 		textTokenizer.tokenize(PLAIN_TEXT);
 		textTokenizer.tokenize(PLAIN_TEXT);
-		Assert.assertEquals(284, textTokenizer.tokenize(PLAIN_TEXT).size());
+		Assert.assertEquals(276, textTokenizer.tokenize(PLAIN_TEXT).size());
 	}
 
 	@Test
@@ -30,6 +30,6 @@ public void reset_MultipleTokenizeCalls_ResetClearsInternalList() {
 		textTokenizer.tokenize(PLAIN_TEXT);
 		textTokenizer.tokenize(PLAIN_TEXT);
 		textTokenizer.reset();
-		Assert.assertEquals(71, textTokenizer.tokenize(PLAIN_TEXT).size());
+		Assert.assertEquals(69, textTokenizer.tokenize(PLAIN_TEXT).size());
 	}
 }