Merge pull request microsoft#5 from Roopan-Microsoft/bug_fix_psl_7834

Bug fix psl 7834

Author: UtkarshMishra-Microsoft

scripts/data_utils.py

@@ -146,28 +146,31 @@ def find_imgs(string):
 
     def split_text(self, text: str) -> List[str]:
         content_dict, masked_text = self.mask_urls_and_imgs(text)
-        start_tag = self._table_tags["table_open"]
-        end_tag = self._table_tags["table_close"]
-        splits = masked_text.split(start_tag)
+        start_tags = [self._table_tags["table_open"], "<tabular>"]
+        end_tags = [self._table_tags["table_close"], "</tabular>"]
+        splits = masked_text
+        for start_tag in start_tags:
+            splits = splits.split(start_tag)
 
         final_chunks = self.chunk_rest(splits[0]) # the first split is before the first table tag so it is regular text
 
         table_caption_prefix = ""
         if len(final_chunks)>0:
             table_caption_prefix += self.extract_caption(final_chunks[-1]) # extracted from the last chunk before the table
         for part in splits[1:]:
-            table, rest = part.split(end_tag)
-            table = start_tag + table + end_tag 
-            minitables = self.chunk_table(table, table_caption_prefix)
-            final_chunks.extend(minitables)
-
-            if rest.strip()!="":
-                text_minichunks = self.chunk_rest(rest)
-                final_chunks.extend(text_minichunks)
-                table_caption_prefix = self.extract_caption(text_minichunks[-1])
-            else:
-                table_caption_prefix = ""
-            
+            for end_tag in part:
+                if end_tag in part:
+                   table, rest = part.split(end_tag)
+                   table = start_tags[0] + table + end_tags[0] 
+                   minitables = self.chunk_table(table, table_caption_prefix)
+                   final_chunks.extend(minitables)
+
+                if rest.strip()!="":
+                   text_minichunks = self.chunk_rest(rest)
+                   final_chunks.extend(text_minichunks)
+                   table_caption_prefix = self.extract_caption(text_minichunks[-1])
+                else:
+                   table_caption_prefix = ""
 
         final_final_chunks = [chunk for chunk, chunk_size in merge_chunks_serially(final_chunks, self._chunk_size, content_dict)]