[BugFix]: Exporting when --do_train and --do_eval are not set (#843) (#853)

* [BugFixes]: To supported exporting when `--do_train` and `--do_eval` are not set
* Use `get_eval_dataloader()` instead of `get_val_dataloader`

For `question_answering.py`
* Always create a eval_dataset if num_export_samples>0

* Address suggestions from initial code review

TODO:
* Propagate same fixes to all transformer tasks

* Propagate changes to MLM

* Propagate changes to text classification

* Propagate changes to token classification

Co-authored-by: Mark Kurtz <mark.kurtz@neuralmagic.com>

Author: rahul-tuli

src/sparseml/transformers/masked_language_modeling.py

@@ -635,7 +635,8 @@ def group_texts(examples):
             train_dataset = train_dataset.select(range(data_args.max_train_samples))
 
     compute_metrics = None
-    if training_args.do_eval:
+    make_eval_dataset = training_args.do_eval or data_args.num_export_samples > 0
+    if make_eval_dataset:
         if "validation" not in tokenized_datasets:
             raise ValueError("--do_eval requires a validation dataset")
         eval_dataset = tokenized_datasets["validation"]
@@ -687,7 +688,7 @@ def compute_metrics(eval_preds):
         args=training_args,
         data_args=data_args,
         train_dataset=train_dataset if training_args.do_train else None,
-        eval_dataset=eval_dataset if training_args.do_eval else None,
+        eval_dataset=eval_dataset if make_eval_dataset else None,
         tokenizer=tokenizer,
         data_collator=data_collator,
         compute_metrics=compute_metrics if training_args.do_eval else None,

src/sparseml/transformers/question_answering.py

@@ -475,9 +475,10 @@ def main():
 
     # Preprocessing the datasets.
     # Preprocessing is slighlty different for training and evaluation.
+    make_eval_dataset = training_args.do_eval or data_args.num_export_samples > 0
     if training_args.do_train:
         column_names = raw_datasets["train"].column_names
-    elif training_args.do_eval:
+    elif make_eval_dataset:
         column_names = raw_datasets["validation"].column_names
     else:
         column_names = raw_datasets["test"].column_names
@@ -666,7 +667,7 @@ def prepare_validation_features(examples):
 
         return tokenized_examples
 
-    if training_args.do_eval:
+    if make_eval_dataset:
         if "validation" not in raw_datasets:
             raise ValueError("--do_eval requires a validation dataset")
         eval_examples = raw_datasets["validation"]
@@ -777,7 +778,7 @@ def compute_metrics(p: EvalPrediction):
         args=training_args,
         data_args=data_args,
         train_dataset=train_dataset if training_args.do_train else None,
-        eval_dataset=eval_dataset if training_args.do_eval else None,
+        eval_dataset=eval_dataset if make_eval_dataset else None,
         eval_examples=eval_examples if training_args.do_eval else None,
         tokenizer=tokenizer,
         data_collator=data_collator,

src/sparseml/transformers/sparsification/trainer.py

@@ -502,7 +502,7 @@ def save_sample_inputs_outputs(
         device = self.model.device
 
         try:
-            dataloader = self.get_val_dataloader()
+            dataloader = self.get_eval_dataloader()
         except Exception:
             dataloader = self.get_train_dataloader()
 

src/sparseml/transformers/text_classification.py

@@ -649,7 +649,8 @@ def preprocess_function(examples):
         if data_args.max_train_samples is not None:
             train_dataset = train_dataset.select(range(data_args.max_train_samples))
 
-    if training_args.do_eval:
+    make_eval_dataset = training_args.do_eval or data_args.num_export_samples > 0
+    if make_eval_dataset:
         if (
             "validation" not in raw_datasets
             and "validation_matched" not in raw_datasets
@@ -725,7 +726,7 @@ def compute_metrics(p: EvalPrediction):
         args=training_args,
         data_args=data_args,
         train_dataset=train_dataset if training_args.do_train else None,
-        eval_dataset=eval_dataset if training_args.do_eval else None,
+        eval_dataset=eval_dataset if make_eval_dataset else None,
         tokenizer=tokenizer,
         data_collator=data_collator,
         compute_metrics=compute_metrics,

src/sparseml/transformers/token_classification.py

@@ -560,7 +560,8 @@ def tokenize_and_align_labels(examples):
                 desc="Running tokenizer on train dataset",
             )
 
-    if training_args.do_eval:
+    make_eval_dataset = training_args.do_eval or data_args.num_export_samples > 0
+    if make_eval_dataset:
         if "validation" not in raw_datasets:
             raise ValueError("--do_eval requires a validation dataset")
         eval_dataset = raw_datasets["validation"]
@@ -648,7 +649,7 @@ def compute_metrics(p):
         args=training_args,
         data_args=data_args,
         train_dataset=train_dataset if training_args.do_train else None,
-        eval_dataset=eval_dataset if training_args.do_eval else None,
+        eval_dataset=eval_dataset if make_eval_dataset else None,
         tokenizer=tokenizer,
         data_collator=data_collator,
         compute_metrics=compute_metrics,