Merge pull request #1107 from stanfordnlp/jflex

Jflex 1.8.2

Author: AngledLuffa

.classpath

@@ -15,7 +15,7 @@
 	<classpathentry exported="true" kind="lib" path="lib/junit-4.13.1.jar" sourcepath="libsrc/junit-4.13.1-sources.jar"/>
 	<classpathentry exported="true" kind="lib" path="lib/javax.servlet.jar" sourcepath="libsrc/tomcat/apache-tomcat-7.0.12-src.zip"/>
 	<classpathentry exported="true" kind="lib" path="lib/AppleJavaExtensions.jar"/>
-	<classpathentry exported="true" kind="lib" path="lib/jflex-1.6.1.jar" sourcepath="libsrc/jflex-1.6.1-src.zip"/>
+	<classpathentry exported="true" kind="lib" path="lib/jflex-1.8.2.jar" sourcepath="libsrc/jflex-1.8.2-src.zip"/>
 	<classpathentry exported="true" kind="lib" path="lib/xom-1.3.2.jar" sourcepath="libsrc/xom-1.3.2-sources.jar"/>
 	<classpathentry exported="true" kind="lib" path="lib/joda-time.jar" sourcepath="libsrc/joda-time-2.10.5-sources.jar"/>
 	<classpathentry exported="true" kind="lib" path="lib/jollyday-0.4.9.jar" sourcepath="libsrc/jollyday-0.4.9-sources.jar"/>

build.xml

@@ -77,7 +77,7 @@
 
     <taskdef classname="jflex.anttask.JFlexTask" name="jflex">
       <classpath>
-        <pathelement location="${project.core}/lib/jflex-1.6.1.jar"/>
+        <pathelement location="${project.core}/lib/jflex-full-1.8.2.jar"/>
       </classpath>
     </taskdef>
 

lib/README

@@ -218,15 +218,15 @@ LAST UPDATE: 2013-11-12
 LAST UPDATE BY: John Bauer
 
 -----------------------------------------------------------------------
-jflex-1.6.1.jar
+jflex-full-1.8.2.jar
 
-ORIGINAL JAR NAME: jflex-1.6.1.jar
+ORIGINAL JAR NAME: jflex-full-1.8.2.jar
 
-VERSION: 1.6.1
+VERSION: 1.8.2
 
-RELEASE DATE: 2015-03-16
+RELEASE DATE: 2020-05-03
 
-SOURCE AVAILABLE: yes, jflex-1.6.1-src.zip
+SOURCE AVAILABLE: yes, jflex-1.8.2-src.zip
 
 DESCRIPTION: compiles .flex files into .java
 
@@ -235,9 +235,9 @@ URL: http://jflex.de/
 USED BY: Compiling PTBTokenizer, etc.
 Not needed by CoreNLP distributions
 
-LAST UPDATE: 2017-05-06
+LAST UPDATE: 2020-11-04
 
-LAST UPDATE BY: Chris Manning
+LAST UPDATE BY: John Bauer
 
 -----------------------------------------------------------------------
 log4j-1.2.16.jar

lib/jflex-1.6.1.jar

@@ -289,14 +289,14 @@ import edu.stanford.nlp.util.PropertiesUtils;
     if (invertible) {
       String str = prevWordAfter.toString();
       prevWordAfter.setLength(0);
-      CoreLabel word = (CoreLabel) tokenFactory.makeToken(txt, yychar, yylength());
+      CoreLabel word = (CoreLabel) tokenFactory.makeToken(txt, Math.toIntExact(yychar), yylength());
       word.set(CoreAnnotations.OriginalTextAnnotation.class, originalText);
       word.set(CoreAnnotations.BeforeAnnotation.class, str);
       prevWord.set(CoreAnnotations.AfterAnnotation.class, str);
       prevWord = word;
       return word;
     } else {
-      return tokenFactory.makeToken(txt, yychar, yylength());
+      return tokenFactory.makeToken(txt, Math.toIntExact(yychar), yylength());
     }
   }
 

lib/jflex-full-1.8.2.jar

@@ -253,7 +253,7 @@ import edu.stanford.nlp.util.logging.Redwood;
 
   private Object getNext(String txt, String originalText, String annotation) {
     txt = LexerUtils.removeSoftHyphens(txt);
-    Label w = (Label) tokenFactory.makeToken(txt, yychar, yylength());
+    Label w = (Label) tokenFactory.makeToken(txt, Math.toIntExact(yychar), yylength());
     if (invertible || annotation != null) {
       CoreLabel word = (CoreLabel) w;
       if (invertible) {