Merge pull request #1214 from stanfordnlp/fix_xml_invertible

More tests for invertible cleanxml.  Make sure to include the before …

Author: AngledLuffa

src/edu/stanford/nlp/pipeline/CleanXmlAnnotator.java

@@ -581,7 +581,8 @@ private List<CoreLabel> process(Annotation annotation, List<CoreLabel> tokens) {
             CoreLabel previous = newTokens.get(newTokens.size() - 2);
             // Note that the original AfterAnnotation is not needed.
             // The BeforeAnnotation of the removed tokens already covers that.
-            previous.set(CoreAnnotations.AfterAnnotation.class, removedText.toString());
+            // However, the `before` text of the next token needs to be included.
+            previous.set(CoreAnnotations.AfterAnnotation.class, removedText + before);
           }
           removedText = new StringBuilder();
         }

test/src/edu/stanford/nlp/pipeline/CleanXmlAnnotatorTest.java

@@ -219,29 +219,25 @@ public void testEarlyEnd() {
   @Test
   public void testInvertible() {
     String testNoTags = "This sentence should be invertible.";
-    String testTags =
-      "  <xml>  This sentence should  be  invertible.  </xml>  ";
-    String testManyTags =
-      " <xml>   <foo>       <bar>This sentence should  " +
-      "   </bar>be invertible.   </foo>   </xml> ";
+    String[] testInvertibles = { "  <xml>  This sentence should  be  invertible.  </xml>  ",
+                                 " <xml>   <foo>       <bar>This sentence should     </bar>be invertible.   </foo>   </xml> ",
+                                 "  This sentence <xml>should</xml>  be  invertible.  ",
+                                 "  This sentence<xml> should </xml>be  invertible.  ",
+                                 "  This sentence <xml> should </xml>  be  invertible.  " };
 
     Annotation annotation = annotate(testNoTags, ptbInvertible,
                                      cleanXmlAllTags, wtsSplitter);
     checkResult(annotation, testNoTags);
     checkBeforeInvert(annotation, testNoTags);
     checkAfterInvert(annotation, testNoTags);
 
-    annotation = annotate(testTags, ptbInvertible,
-                          cleanXmlAllTags, wtsSplitter);
-    checkResult(annotation, testNoTags);
-    checkBeforeInvert(annotation, testTags);
-    checkAfterInvert(annotation, testTags);
-
-    annotation = annotate(testManyTags, ptbInvertible,
-                          cleanXmlAllTags, wtsSplitter);
-    checkResult(annotation, testNoTags);
-    checkBeforeInvert(annotation, testManyTags);
-    checkAfterInvert(annotation, testManyTags);
+    for (String test : testInvertibles) {
+      annotation = annotate(test, ptbInvertible,
+                            cleanXmlAllTags, wtsSplitter);
+      checkResult(annotation, testNoTags);
+      checkBeforeInvert(annotation, test);
+      checkAfterInvert(annotation, test);
+    }
   }
 
   @Test