File tree Expand file tree Collapse file tree 5 files changed +9
-9
lines changed
Expand file tree Collapse file tree 5 files changed +9
-9
lines changed Original file line number Diff line number Diff line change @@ -289,14 +289,14 @@ import edu.stanford.nlp.util.PropertiesUtils;
289289 if (invertible) {
290290 String str = prevWordAfter. toString();
291291 prevWordAfter. setLength(0 );
292- CoreLabel word = (CoreLabel ) tokenFactory. makeToken(txt, yychar, yylength());
292+ CoreLabel word = (CoreLabel ) tokenFactory. makeToken(txt, Math . toIntExact( yychar) , yylength());
293293 word. set(CoreAnnotations . OriginalTextAnnotation . class, originalText);
294294 word. set(CoreAnnotations . BeforeAnnotation . class, str);
295295 prevWord. set(CoreAnnotations . AfterAnnotation . class, str);
296296 prevWord = word;
297297 return word;
298298 } else {
299- return tokenFactory. makeToken(txt, yychar, yylength());
299+ return tokenFactory. makeToken(txt, Math . toIntExact( yychar) , yylength());
300300 }
301301 }
302302
Original file line number Diff line number Diff line change @@ -253,7 +253,7 @@ import edu.stanford.nlp.util.logging.Redwood;
253253
254254 private Object getNext(String txt, String originalText, String annotation) {
255255 txt = LexerUtils . removeSoftHyphens(txt);
256- Label w = (Label ) tokenFactory. makeToken(txt, yychar, yylength());
256+ Label w = (Label ) tokenFactory. makeToken(txt, Math . toIntExact( yychar) , yylength());
257257 if (invertible || annotation != null ) {
258258 CoreLabel word = (CoreLabel ) w;
259259 if (invertible) {
Original file line number Diff line number Diff line change @@ -269,7 +269,7 @@ import edu.stanford.nlp.util.logging.Redwood;
269269
270270 private Object getNext(String txt, String originalText, String annotation) {
271271 txt = LexerUtils . removeSoftHyphens(txt);
272- Label w = (Label ) tokenFactory. makeToken(txt, yychar, yylength());
272+ Label w = (Label ) tokenFactory. makeToken(txt, Math . toIntExact( yychar) , yylength());
273273 if (invertible || annotation != null ) {
274274 CoreLabel word = (CoreLabel ) w;
275275 if (invertible) {
Original file line number Diff line number Diff line change @@ -489,14 +489,14 @@ import edu.stanford.nlp.util.logging.Redwood;
489489 if (invertible) {
490490 String str = prevWordAfter. toString();
491491 prevWordAfter. setLength(0 );
492- CoreLabel word = (CoreLabel ) tokenFactory. makeToken(txt, yychar, yylength());
492+ CoreLabel word = (CoreLabel ) tokenFactory. makeToken(txt, Math . toIntExact( yychar) , yylength());
493493 word. set(CoreAnnotations . OriginalTextAnnotation . class, originalText);
494494 word. set(CoreAnnotations . BeforeAnnotation . class, str);
495495 prevWord. set(CoreAnnotations . AfterAnnotation . class, str);
496496 prevWord = word;
497497 return word;
498498 } else {
499- Object word = tokenFactory. makeToken(txt, yychar, yylength());
499+ Object word = tokenFactory. makeToken(txt, Math . toIntExact( yychar) , yylength());
500500 if (word instanceof CoreLabel ) {
501501 prevWord = (CoreLabel ) word;
502502 }
Original file line number Diff line number Diff line change @@ -53,8 +53,8 @@ TEXT = [^ \t\u1680\u2000-\u2006\u2008-\u200A\u205F\u3000\r\n\u0085\u2028\u2029\u
5353
5454%%
5555
56- {CR} { return tokenFactory. makeToken(NEWLINE , yychar, yylength()); }
57- {OTHERSEP} { return tokenFactory. makeToken(NEWLINE , yychar, yylength()); }
56+ {CR} { return tokenFactory. makeToken(NEWLINE , Math . toIntExact( yychar) , yylength()); }
57+ {OTHERSEP} { return tokenFactory. makeToken(NEWLINE , Math . toIntExact( yychar) , yylength()); }
5858{SPACES} { }
59- {TEXT} { return tokenFactory. makeToken(yytext(), yychar, yylength()); }
59+ {TEXT} { return tokenFactory. makeToken(yytext(), Math . toIntExact( yychar) , yylength()); }
6060<<EOF>> { return null ; }
You can’t perform that action at this time.
0 commit comments