Merge branch 'dev' of https://github.com/stanfordnlp/CoreNLP into main

Author: J38

.github/workflows/run-tests.yaml

@@ -0,0 +1,28 @@
+name: Run Tests
+on: [push]
+jobs:
+  Run-CoreNLP-Tests:
+    runs-on: self-hosted
+    steps:
+      - run: echo "🎉 The job was automatically triggered by a ${{ github.event_name }} event."
+      - run: echo "🐧 This job is now running on a ${{ runner.os }} server hosted by GitHub!"
+      - run: echo "🔎 The name of your branch is ${{ github.ref }} and your repository is ${{ github.repository }}."
+      - name: Check out repository code
+        uses: actions/checkout@v2
+      - run: echo "💡 The ${{ github.repository }} repository has been cloned to the runner."
+      - run: echo "🖥️ The workflow is now ready to test your code on the runner."
+      - name: Setup
+        run: |
+          pwd
+          ant clean compile
+          echo $CLASSPATH
+      - name: Run Unit Tests
+        if: always()
+        run: |
+          ant test
+      - name: Run Basic ITests
+        if: always()
+        run: |
+          ant itest
+      - run: echo "All tests finished!"
+      - run: echo "🍏 This job's status is ${{ job.status }}."

doc/corenlp/pom-full.xml

@@ -86,7 +86,7 @@
     <dependency>
       <groupId>xom</groupId>
       <artifactId>xom</artifactId>
-      <version>1.3.2</version>
+      <version>1.3.7</version>
     </dependency>
 
     <dependency>
@@ -128,7 +128,7 @@
     <dependency>
       <groupId>com.google.protobuf</groupId>
       <artifactId>protobuf-java</artifactId>
-      <version>3.9.2</version>
+      <version>3.11.4</version>
     </dependency>
 
     <dependency>

itest/src/edu/stanford/nlp/dcoref/DcorefSlowITest.java

@@ -13,7 +13,7 @@
 import junit.framework.TestCase;
 
 import edu.stanford.nlp.util.StringUtils;
-
+import edu.stanford.nlp.util.TestPaths;
 
 public class DcorefSlowITest extends TestCase {
 
@@ -29,19 +29,19 @@ protected void makePropsFile(String path, String workDir, String scorer) throws
     pw.println("dcoref.maxdist = -1");
     pw.println("dcoref.replicate.conll = true");
     pw.println("dcoref.conll.scorer = " + scorer);
-    pw.println("dcoref.conll2011 = /u/scr/nlp/data/conll-2011/v2/data/dev/data/english/annotations");
+    pw.println(String.format("dcoref.conll2011 = %s/conll-2011/v2/data/dev/data/english/annotations", TestPaths.testHome()));
     pw.println("dcoref.logFile = "+workDir + File.separator + "log.txt");
     pw.close();
   }
 
   public void testDcorefCoNLLResultV4() throws Exception {
-    double finalScore = runDcoref("/u/scr/nlp/data/conll-2011/scorer/v4/scorer.pl");
+    double finalScore = runDcoref(String.format("%s/conll-2011/scorer/v4/scorer.pl", TestPaths.testHome()));
     System.out.printf("Final Score (CoNLL 2011, scorer v4): (MUC+B^3+ceafe)/3 = %.2f%n", finalScore);
     assertEquals(59.3, finalScore, 0.3); // 2016-07: 59.45
   }
 
   public void testDcorefCoNLLResultV801() throws Exception {
-    double finalScore = runDcoref("/u/scr/nlp/data/conll-2012/scorer/v8.01/scorer.pl");
+    double finalScore = runDcoref(String.format("%s/conll-2012/scorer/v8.01/scorer.pl", TestPaths.testHome()));
     System.out.printf("Final Score (CoNLL 2011, scorer v8): (MUC+B^3+ceafe)/3 = %.2f%n", finalScore);
     assertEquals(54.0, finalScore, 0.3); // 2016-07: 54.13
   }

itest/src/edu/stanford/nlp/ie/NERBenchmarkSlowITest.java

@@ -33,6 +33,7 @@
 import edu.stanford.nlp.sequences.CoNLLDocumentReaderAndWriter;
 import edu.stanford.nlp.sequences.SeqClassifierFlags;
 import edu.stanford.nlp.util.logging.Redwood;
+import edu.stanford.nlp.util.TestPaths;
 
 
 /**
@@ -45,7 +46,7 @@
 public class NERBenchmarkSlowITest {
     private static Redwood.RedwoodChannels log = Redwood.channels(NERBenchmarkSlowITest.class);
     // Conll paths
-    private static final String CONLL_BASE_DIR = "/u/nlp/data/ner/conll/";
+    private static final String CONLL_BASE_DIR = String.format("%s/ner/conll/", TestPaths.testHome());
     private static final String CONLL_TRAIN = CONLL_BASE_DIR + "eng.train";
     private static final String CONLL_DEV = CONLL_BASE_DIR + "eng.testa";
     private static final String CONLL_TEST = CONLL_BASE_DIR + "eng.testb";
@@ -54,13 +55,13 @@ public class NERBenchmarkSlowITest {
     private static final String CONLL_OUTPUT_TEST = "conll_output_test.txt";
 
     // Onto paths
-    private static final String ONTO_BASE_DIR = "/u/nlp/data/ner/ontonotes/";
+    private static final String ONTO_BASE_DIR = String.format("%s/ner/ontonotes/", TestPaths.testHome());
     private static final String ONTO_DEV = ONTO_BASE_DIR + "onto-3class-dev.tsv";
     private static final String ONTO_TEST = ONTO_BASE_DIR + "onto-3class-test.tsv";
 
     // TODO: use the model directly to run the test
     /** official CoNLL NER evaluation script **/
-    private static final String CONLL_EVAL = "/u/nlp/data/ner/benchmark/eval_conll.sh";
+    private static final String CONLL_EVAL = String.format("%s/ner/benchmark/eval_conll.sh", TestPaths.testHome());
     // private static final String CONLL_EVAL = (new File("projects/core/scripts/ner/eval_conll_cmd.sh").exists() ?
     //                                           "projects/core/scripts/ner/eval_conll_cmd.sh" :
     //                                           "../../scripts/ner/eval_conll_cmd.sh");

itest/src/edu/stanford/nlp/ie/NERServerITest.java

@@ -17,6 +17,7 @@
 import edu.stanford.nlp.ie.crf.CRFClassifier;
 import edu.stanford.nlp.io.IOUtils;
 import edu.stanford.nlp.net.Ports;
+import edu.stanford.nlp.util.TestPaths;
 
 /**
  * Tests several operations on the NERServer.
@@ -39,10 +40,9 @@
 public class NERServerITest {
   private static CRFClassifier crf = null;
 
-  private static final String englishCRFPath =
-    "/u/nlp/data/ner/goodClassifiers/english.all.3class.nodistsim.crf.ser.gz";
+  private static final String englishCRFPath = "edu/stanford/nlp/models/ner/english.all.3class.distsim.crf.ser.gz";
   private static final String englishTestFile =
-    "/u/nlp/data/ner/column_data/conll.testa";
+    String.format("%s/ner/column_data/conll.testa", TestPaths.testHome());
   private static String loadedQueryFile = null;
 
   private static final String CHARSET = "UTF-8";

itest/src/edu/stanford/nlp/ie/NumberSequenceClassifierITest.java

@@ -224,7 +224,7 @@ public void testOrdinal() {
     { "DATE" , "DATE", "DATE", "DATE" },
     { "DATE" , "DATE" },
     { "DATE" , "DATE", "DATE" },
-    { "DATE" , "DATE", "DATE", "DATE" },
+    { "DATE" , "DATE", "DATE" },
     { "DATE" , "DATE" },
     { "DATE" },
     { "DATE" },
@@ -242,7 +242,7 @@ public void testOrdinal() {
     { "1923-02-05" , "1923-02-05", "1923-02-05", "1923-02-05" },
     { "XXXX-03-03" , "XXXX-03-03" },
     { "2005-07-18" , "2005-07-18", "2005-07-18" },
-    { "XX05-09-18" , "XX05-09-18", "XX05-09-18", "XX05-09-18" },
+    { "XX05-09-18" , "XX05-09-18", "XX05-09-18" },
     { "XXXX-01-13" , "XXXX-01-13" },
     { "2009-07-19" },
     { "2007-06-16" },

itest/src/edu/stanford/nlp/ie/crf/CRFClassifierITest.java

@@ -22,6 +22,7 @@
 import edu.stanford.nlp.stats.Counters;
 import edu.stanford.nlp.util.BenchmarkingHelper;
 import edu.stanford.nlp.util.Pair;
+import edu.stanford.nlp.util.TestPaths;
 import edu.stanford.nlp.util.Triple;
 
 
@@ -34,8 +35,8 @@ public class CRFClassifierITest {
   private static final String nerPath = "edu/stanford/nlp/models/ner/english.all.3class.distsim.crf.ser.gz";
 
   private static final String caselessPath = "edu/stanford/nlp/models/ner/english.all.3class.caseless.distsim.crf.ser.gz";
-  // private static final String caselessPath = "/u/nlp/data/ner/goodClassifiers/english.all.3class.caseless.distsim.crf.ser.gz"; // now works!
-  // private static final String caselessPath = "/u/nlp/data/ner/classifiers-2014-08-31/english.all.3class.caseless.distsim.crf.ser.gz";
+  // private static final String caselessPath = String.format("%s/ner/goodClassifiers/english.all.3class.caseless.distsim.crf.ser.gz", TestPaths.testHome()); // now works!
+  // private static final String caselessPath = String.format("%s/ner/classifiers-2014-08-31/english.all.3class.caseless.distsim.crf.ser.gz", TestPaths.testHome());
 
   /* The extra spaces and tab (after fate) are there to test space preservation.
    * Each item of the top level array is an array of 7 Strings:
@@ -330,7 +331,7 @@ public void testCRF() {
         System.getProperty("ner.caseless.model", caselessPath));
 
     try {
-      Triple<Double, Double, Double> prf = crfCaseless.classifyAndWriteAnswers("/u/nlp/data/ner/column_data/ritter.3class.test", true);
+      Triple<Double, Double, Double> prf = crfCaseless.classifyAndWriteAnswers(String.format("%s/ner/column_data/ritter.3class.test", TestPaths.testHome()), true);
       Counter<String> results = new ClassicCounter<>();
       results.setCount("NER F1", prf.third());
       Counter<String> lowResults = new ClassicCounter<>();

itest/src/edu/stanford/nlp/ie/crf/ThreadedCRFClassifierITest.java

@@ -5,6 +5,8 @@
 import org.junit.Before;
 import org.junit.Test;
 
+import edu.stanford.nlp.util.TestPaths;
+
 /**
  * Test that the CRFClassifier works when multiple classifiers are run
  * in multiple threads.
@@ -19,15 +21,13 @@ public class ThreadedCRFClassifierITest {
     "edu/stanford/nlp/models/ner/german.distsim.crf.ser.gz";
   /** -- We're no longer supporting this one
   private String german2 =
-    "/u/nlp/data/ner/goodClassifiers/german.dewac_175m_600.crf.ser.gz";
+    String.format("%s/ner/goodClassifiers/german.dewac_175m_600.crf.ser.gz", TestPaths.testHome());
   */
-  private static final String germanTestFile = "/u/nlp/data/german/ner/2016/deu.io.f15.utf8.testa";
+  private static final String germanTestFile = String.format("%s/german/ner/2016/deu.io.f15.utf8.testa", TestPaths.testHome());
 
-  private static final String english1 =
-    "/u/nlp/data/ner/goodClassifiers/english.all.3class.nodistsim.crf.ser.gz";
-  private static final String english2 =
-    "/u/nlp/data/ner/goodClassifiers/english.conll.4class.distsim.crf.ser.gz";
-  private static final String englishTestFile = "/u/nlp/data/ner/column_data/conll.4class.testa";
+  private static final String english1 = "edu/stanford/nlp/models/ner/english.all.3class.distsim.crf.ser.gz";
+  private static final String english2 = "edu/stanford/nlp/models/ner/english.conll.4class.distsim.crf.ser.gz";
+  private static final String englishTestFile = String.format("%s/ner/column_data/conll.4class.testa", TestPaths.testHome());
 
   private static final String germanEncoding = "utf-8";
   private static final String englishEncoding = "utf-8";

itest/src/edu/stanford/nlp/ie/crf/ThreadedSegmenterITest.java

@@ -5,6 +5,8 @@
 import org.junit.Before;
 import org.junit.Test;
 
+import edu.stanford.nlp.util.TestPaths;
+
 /** 
  * Test that the CRFClassifier works when multiple classifiers are run
  * in multiple threads, in Chinese Segmentation mode.
@@ -21,21 +23,21 @@ public class ThreadedSegmenterITest {
   Properties props;
 
   static final String crf1 = 
-    "/u/nlp/data/gale/segtool/stanford-seg/classifiers-2010/pk-chris6.lex.gz";
+    String.format("%s/gale/segtool/stanford-seg/classifiers-2010/pk-chris6.lex.gz", TestPaths.testHome());
 
-  static final String crf2 = "/u/nlp/data/gale/segtool/stanford-seg/classifiers-2010/05202008-ctb6.processed-chris6.lex.gz";
+  static final String crf2 = String.format("%s/gale/segtool/stanford-seg/classifiers-2010/05202008-ctb6.processed-chris6.lex.gz", TestPaths.testHome());
 
   @Before
   public void setUp() {
     props = new Properties();
     props.setProperty("sighanCorporaDict", 
-                      "/u/nlp/data/gale/segtool/stanford-seg/data");
+                      String.format("%s/gale/segtool/stanford-seg/data", TestPaths.testHome()));
     props.setProperty("testFile",
-                      "/u/nlp/segtool/stanford-seg/data/Sighan2006/CTB_train_test/test/CTB.utf8.simp");
+                      String.format("%s/gale/segtool/stanford-seg/data/Sighan2006/CTB_train_test/test/CTB.utf8.simp", TestPaths.testHome()));
     props.setProperty("inputEncoding", "utf-8");
     props.setProperty("sighanPostProcessing", "true");
     props.setProperty("serDictionary", 
-                      "/u/nlp/data/gale/segtool/stanford-seg/classifiers/dict-chris6.ser.gz");
+                      String.format("%s/gale/segtool/stanford-seg/classifiers/dict-chris6.ser.gz", TestPaths.testHome()));
     props.setProperty("keepAllWhitespaces", "false");
   }
 

itest/src/edu/stanford/nlp/international/german/process/GermanTokenizerPostProcessorITest.java

@@ -49,4 +49,20 @@ public void testPostProcessor() {
     testExample(abbreviationExample,abbreviationExampleGoldTokens);
   }
 
+  /**
+   * You probably can't tell in your editor, but the input has 4 characters for 
+   * <pre>für</pre>
+   * and the output has 3
+   */
+  public void testUmlauts() {
+    String fur = "für";
+    assertEquals(4, fur.length());
+
+    String furry = "für";
+    assertEquals(3, furry.length());
+
+    String umlautExample = "Welcher der Befunde ist " + fur + " eine Gehirnerkrankung typisch?";
+    List<String> umlautGoldTokens = Arrays.asList("Welcher", "der", "Befunde", "ist", furry, "eine", "Gehirnerkrankung", "typisch", "?");
+    testExample(umlautExample, umlautGoldTokens);
+  }
 }