Merge pull request #1213 from stanfordnlp/fix_clean_after

Fixes the accumulation of AfterAnnotation text, as reported in #1210

Author: AngledLuffa

src/edu/stanford/nlp/pipeline/CleanXmlAnnotator.java

@@ -579,12 +579,9 @@ private List<CoreLabel> process(Annotation annotation, List<CoreLabel> tokens) {
           }
           if (added && newTokens.size() > 1) {
             CoreLabel previous = newTokens.get(newTokens.size() - 2);
-            String after = previous.get(CoreAnnotations.AfterAnnotation.class);
-            if (after != null) {
-              previous.set(CoreAnnotations.AfterAnnotation.class, after + removedText);
-            } else {
-              previous.set(CoreAnnotations.AfterAnnotation.class, removedText.toString());
-            }
+            // Note that the original AfterAnnotation is not needed.
+            // The BeforeAnnotation of the removed tokens already covers that.
+            previous.set(CoreAnnotations.AfterAnnotation.class, removedText.toString());
           }
           removedText = new StringBuilder();
         }
@@ -650,6 +647,7 @@ private List<CoreLabel> process(Annotation annotation, List<CoreLabel> tokens) {
       currentRemoval = token.get(CoreAnnotations.OriginalTextAnnotation.class);
       if (currentRemoval != null)
         removedText.append(currentRemoval);
+      // We only process the after text if it's the last token
       if (token == tokens.get(tokens.size() - 1)) {
         currentRemoval = token.get(CoreAnnotations.AfterAnnotation.class);
         if (currentRemoval != null)

test/src/edu/stanford/nlp/pipeline/CleanXmlAnnotatorTest.java

@@ -111,7 +111,7 @@ private static void checkResult(Annotation annotation,
     }
   }
 
-  private static void checkInvert(Annotation annotation, String gold) {
+  private static void checkBeforeInvert(Annotation annotation, String gold) {
     List<CoreLabel> annotationLabels =
       annotation.get(CoreAnnotations.TokensAnnotation.class);
     StringBuilder original = new StringBuilder();
@@ -124,6 +124,18 @@ private static void checkInvert(Annotation annotation, String gold) {
     assertEquals(gold, original.toString());
   }
 
+  private static void checkAfterInvert(Annotation annotation, String gold) {
+    List<CoreLabel> annotationLabels =
+      annotation.get(CoreAnnotations.TokensAnnotation.class);
+    StringBuilder original = new StringBuilder();
+    original.append(annotationLabels.get(0).get(CoreAnnotations.BeforeAnnotation.class));
+    for (CoreLabel label : annotationLabels) {
+      original.append(label.get(CoreAnnotations.OriginalTextAnnotation.class));
+      original.append(label.get(CoreAnnotations.AfterAnnotation.class));
+    }
+    assertEquals(gold, original.toString());
+  }
+
   private static void checkContext(CoreLabel label, String... expectedContext) {
     List<String> xmlContext = label.get(CoreAnnotations.XmlContextAnnotation.class);
     assertEquals(expectedContext.length, xmlContext.size());
@@ -216,17 +228,20 @@ public void testInvertible() {
     Annotation annotation = annotate(testNoTags, ptbInvertible,
                                      cleanXmlAllTags, wtsSplitter);
     checkResult(annotation, testNoTags);
-    checkInvert(annotation, testNoTags);
+    checkBeforeInvert(annotation, testNoTags);
+    checkAfterInvert(annotation, testNoTags);
 
     annotation = annotate(testTags, ptbInvertible,
                           cleanXmlAllTags, wtsSplitter);
     checkResult(annotation, testNoTags);
-    checkInvert(annotation, testTags);
+    checkBeforeInvert(annotation, testTags);
+    checkAfterInvert(annotation, testTags);
 
     annotation = annotate(testManyTags, ptbInvertible,
                           cleanXmlAllTags, wtsSplitter);
     checkResult(annotation, testNoTags);
-    checkInvert(annotation, testManyTags);
+    checkBeforeInvert(annotation, testManyTags);
+    checkAfterInvert(annotation, testManyTags);
   }
 
   @Test
@@ -287,7 +302,8 @@ public void testViaCoreNlp() {
     StanfordCoreNLP pipeline = new StanfordCoreNLP(props);
     pipeline.annotate(anno);
 
-    checkInvert(anno, testManyTags);
+    checkBeforeInvert(anno, testManyTags);
+    checkAfterInvert(anno, testManyTags);
     List<CoreLabel> annotationLabels =
       anno.get(CoreAnnotations.TokensAnnotation.class);
     for (int i = 0; i < 3; ++i) {