Skip to content

Commit 36f00a7

Browse files
patrickfleithpatrickfleith
authored
Merge pull request #85 from patrickfleith/feature/82-data-inspector
Feature/82 data inspector
2 parents 344c027 + 888551a commit 36f00a7

4 files changed

Lines changed: 730 additions & 1 deletion

File tree

‎datafast/datasets.py‎

Lines changed: 119 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -71,12 +71,130 @@ class JudgeLLMOutput(BaseModel):
7171

7272

7373
class DatasetBase(ABC):
74-
"""Abstract base class for all dataset generators."""
74+
"""
75+
Abstract base class for all dataset generators.
76+
77+
Methods
78+
-------
79+
inspect():
80+
Launch a Gradio app to visually browse the dataset (self.data_rows).
81+
Requires gradio to be installed (pip install gradio).
82+
Provides Next/Previous navigation through dataset examples.
83+
"""
7584

7685
def __init__(self, config):
7786
self.config = config
7887
self.data_rows = []
7988

89+
def inspect(self, random: bool = False) -> None:
90+
"""
91+
Launch an interactive Gradio app to visually inspect the generated dataset.
92+
93+
This method redirects to specialized inspectors in datafast.inspectors module,
94+
which provide tailored visualization for each dataset type.
95+
96+
Args:
97+
random: If True, examples will be shown in random order instead of sequential order.
98+
Default is False (sequential order).
99+
100+
Raises:
101+
ImportError: If gradio is not installed.
102+
ValueError: If the dataset type is not supported by any specialized inspector.
103+
"""
104+
import warnings
105+
from importlib import import_module
106+
107+
try:
108+
# Test if Gradio is installed
109+
import gradio as gr
110+
except ImportError as e:
111+
raise ImportError("Gradio is required for .inspect(). Install with 'pip install gradio'.") from e
112+
113+
if not self.data_rows:
114+
raise ValueError("No data rows to inspect. Generate or load data first.")
115+
116+
try:
117+
# Import inspectors dynamically to prevent import cycles
118+
inspectors = import_module('datafast.inspectors')
119+
120+
# Get the class name without module prefix and convert CamelCase to snake_case
121+
class_name = self.__class__.__name__
122+
123+
# Convert CamelCase to snake_case (e.g., ClassificationDataset -> classification_dataset)
124+
import re
125+
snake_case = re.sub(r'(?<!^)(?=[A-Z])', '_', class_name).lower()
126+
inspector_name = f"inspect_{snake_case}"
127+
128+
if hasattr(inspectors, inspector_name):
129+
# Call the appropriate specialized inspector
130+
inspector_func = getattr(inspectors, inspector_name)
131+
inspector_func(self, random=random)
132+
else:
133+
# Fall back to generic JSON display with a warning
134+
warnings.warn(
135+
f"No specialized inspector found for {class_name}. "
136+
"Using generic inspector. Consider adding a specialized inspector "
137+
"in datafast.inspectors module.",
138+
UserWarning
139+
)
140+
self._generic_inspect(random=random)
141+
except Exception as e:
142+
# If there's any error in the process, fall back to generic inspector
143+
warnings.warn(f"Error using specialized inspector: {e}. Falling back to generic inspector.")
144+
self._generic_inspect(random=random)
145+
146+
def _generic_inspect(self, random: bool = False) -> None:
147+
"""Generic inspector that displays dataset rows as JSON.
148+
149+
Args:
150+
random: If True, examples will be shown in random order instead of sequential. Default is False.
151+
"""
152+
import gradio as gr
153+
import numpy as np
154+
155+
# Convert data rows to dicts for display
156+
examples = [row.model_dump() if hasattr(row, 'model_dump') else row.dict() if hasattr(row, 'dict') else row for row in self.data_rows]
157+
total = len(examples)
158+
159+
# Generate random order indices if random is True
160+
if random and total > 1:
161+
import numpy as np
162+
# Create a permutation of indices
163+
random_indices = np.random.permutation(total)
164+
display_order = list(random_indices)
165+
ordering_label = "(Random Order)"
166+
else:
167+
# Sequential order
168+
display_order = list(range(total))
169+
ordering_label = ""
170+
171+
def show_example(idx: int) -> tuple[str, dict]:
172+
idx = max(0, min(idx, total - 1))
173+
# Get the actual example based on the display order
174+
example_idx = display_order[idx]
175+
return f"Example {idx+1} / {total} {ordering_label}", examples[example_idx]
176+
177+
with gr.Blocks() as demo:
178+
idx_state = gr.State(0)
179+
gr.Markdown("# Dataset Inspector (Generic)")
180+
idx_label = gr.Markdown()
181+
data_view = gr.JSON()
182+
with gr.Row():
183+
prev_btn = gr.Button("Previous")
184+
next_btn = gr.Button("Next")
185+
186+
def update_example(idx):
187+
label, example = show_example(idx)
188+
return label, example, idx
189+
190+
prev_btn.click(lambda idx: max(0, idx-1), idx_state, idx_state).then(update_example, idx_state, [idx_label, data_view, idx_state])
191+
next_btn.click(lambda idx: min(total-1, idx+1), idx_state, idx_state).then(update_example, idx_state, [idx_label, data_view, idx_state])
192+
193+
# Initial display
194+
demo.load(update_example, idx_state, [idx_label, data_view, idx_state])
195+
196+
demo.launch()
197+
80198
@abstractmethod
81199
def generate(self, llms=None):
82200
"""Main method to generate the dataset."""

‎datafast/examples/inspect_dataset_example.py‎

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
"""
2+
Example script showing how to generate a dataset and launch the visual inspector.
3+
4+
Run with:
5+
python -m datafast.examples.inspect_dataset_example
6+
7+
Requires:
8+
- OpenAI API key in secrets.env or environment
9+
- gradio package (pip install gradio)
10+
"""
11+
from datafast.datasets import ClassificationDataset
12+
from datafast.schema.config import ClassificationDatasetConfig, PromptExpansionConfig
13+
from dotenv import load_dotenv
14+
15+
# Load API keys from environment or secrets.env
16+
load_dotenv("secrets.env")
17+
18+
# Configure the dataset generation
19+
config = ClassificationDatasetConfig(
20+
classes=[
21+
{"name": "positive", "description": "Text expressing positive emotions or approval"},
22+
{"name": "negative", "description": "Text expressing negative emotions or criticism"},
23+
],
24+
num_samples_per_prompt=2, # Small number for quick demo
25+
output_file="outdoor_activities_sentiments.jsonl", # Optional, will save generated data
26+
languages={
27+
"en": "English",
28+
},
29+
prompts=[
30+
(
31+
"Generate {num_samples} reviews in {language_name} which are diverse "
32+
"and representative of a '{label_name}' sentiment class. "
33+
"{label_description}. The reviews should be brief and in the "
34+
"context of {{context}}."
35+
)
36+
],
37+
expansion=PromptExpansionConfig(
38+
placeholders={
39+
"context": ["hiking trail review", "kayaking trip review"],
40+
},
41+
combinatorial=True # Will generate combinations of all placeholders
42+
)
43+
)
44+
45+
# Set up LLM providers
46+
from datafast.llms import OpenAIProvider, AnthropicProvider, GeminiProvider
47+
48+
providers = [
49+
OpenAIProvider(model_id="gpt-4.1-nano"),
50+
# Uncomment to use additional providers
51+
# AnthropicProvider(model_id="claude-3-5-haiku-latest"),
52+
# GeminiProvider(model_id="gemini-2.0-flash"),
53+
]
54+
55+
def main():
56+
# Generate the dataset
57+
dataset = ClassificationDataset(config)
58+
num_expected_rows = dataset.get_num_expected_rows(providers)
59+
print(f"Expected number of rows to generate: {num_expected_rows}")
60+
61+
# Generate data (comment out if loading existing data)
62+
print("Generating dataset...")
63+
dataset.generate(providers)
64+
print(f"Generated {len(dataset.data_rows)} examples")
65+
66+
# Launch the interactive inspector
67+
print("\nLaunching dataset inspector...")
68+
print("(Close the browser window or press Ctrl+C to exit)")
69+
print("Showing examples in random order")
70+
dataset.inspect(random=True)
71+
72+
if __name__ == "__main__":
73+
main()

‎datafast/examples/show_dataset_examples.py‎

Lines changed: 160 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,160 @@
1+
"""
2+
Demo: Show an example for each dataset type using the Gradio inspectors.
3+
4+
Run with:
5+
python show_dataset_examples.py
6+
7+
Requires: gradio
8+
"""
9+
from datafast.inspectors import (
10+
inspect_classification_dataset,
11+
inspect_mcq_dataset,
12+
inspect_preference_dataset,
13+
inspect_raw_dataset,
14+
inspect_ultrachat_dataset,
15+
)
16+
from datafast.schema.data_rows import (
17+
TextClassificationRow,
18+
MCQRow,
19+
PreferenceRow,
20+
TextRow,
21+
ChatRow,
22+
)
23+
from datafast.datasets import (
24+
ClassificationDataset,
25+
MCQDataset,
26+
PreferenceDataset,
27+
RawDataset,
28+
UltrachatDataset,
29+
)
30+
from datafast.schema.config import (
31+
ClassificationDatasetConfig,
32+
MCQDatasetConfig,
33+
PreferenceDatasetConfig,
34+
)
35+
36+
# --- Classification Example ---
37+
classification_row = TextClassificationRow(
38+
text="The trail is blocked by a fallen tree.",
39+
label="trail_obstruction",
40+
model_id="gpt-4.1-nano",
41+
metadata={"language": "en"},
42+
)
43+
classification_dataset = ClassificationDataset(
44+
ClassificationDatasetConfig(classes=[{"name": "trail_obstruction", "description": "Obstruction on the trail."}])
45+
)
46+
classification_row2 = TextClassificationRow(
47+
text="The trail is well maintained and easy to follow.",
48+
label="positive_conditions",
49+
model_id="claude-3-5-haiku-latest",
50+
metadata={"language": "en"},
51+
)
52+
classification_dataset.data_rows = [classification_row, classification_row2]
53+
54+
# --- MCQ Example ---
55+
mcq_row = MCQRow(
56+
source_document="The Eiffel Tower is in Paris.",
57+
question="Where is the Eiffel Tower located?",
58+
correct_answer="Paris",
59+
incorrect_answer_1="London",
60+
incorrect_answer_2="Berlin",
61+
incorrect_answer_3="Rome",
62+
model_id="gemini-2.0-flash",
63+
metadata={"language": "en"},
64+
)
65+
mcq_config = MCQDatasetConfig(
66+
text_column="source_document",
67+
local_file_path="dummy.jsonl", # Required by config, not used in this demo
68+
)
69+
mcq_dataset = MCQDataset(mcq_config)
70+
mcq_row2 = MCQRow(
71+
source_document="The Amazon River is the second longest river in the world.",
72+
question="Which river is the second longest in the world?",
73+
correct_answer="Amazon River",
74+
incorrect_answer_1="Nile River",
75+
incorrect_answer_2="Yangtze River",
76+
incorrect_answer_3="Mississippi River",
77+
model_id="gpt-4.1-nano",
78+
metadata={"language": "en"},
79+
)
80+
mcq_dataset.data_rows = [mcq_row, mcq_row2]
81+
82+
# --- Preference Example ---
83+
preference_row = PreferenceRow(
84+
input_document="Describe a recent Mars mission.",
85+
question="What was the main goal of the Mars 2020 mission?",
86+
chosen_response="To search for signs of ancient life and collect samples.",
87+
rejected_response="To launch a satellite.",
88+
chosen_model_id="claude-3-5-haiku-latest",
89+
rejected_model_id="gpt-4.1-nano",
90+
chosen_response_score=9,
91+
rejected_response_score=3,
92+
chosen_response_assessment="Accurate and detailed.",
93+
rejected_response_assessment="Too generic.",
94+
metadata={"language": "en"},
95+
)
96+
preference_dataset = PreferenceDataset(PreferenceDatasetConfig(input_documents=["Describe a recent Mars mission."]))
97+
preference_row2 = PreferenceRow(
98+
input_document="Describe the Voyager 1 mission.",
99+
question="What is Voyager 1 known for?",
100+
chosen_response="It is the farthest human-made object from Earth, exploring interstellar space.",
101+
rejected_response="It is a Mars rover.",
102+
chosen_model_id="gemini-2.0-flash",
103+
rejected_model_id="gpt-4.1-nano",
104+
chosen_response_score=10,
105+
rejected_response_score=2,
106+
chosen_response_assessment="Factually correct and detailed.",
107+
rejected_response_assessment="Incorrect mission.",
108+
metadata={"language": "en"},
109+
)
110+
preference_dataset.data_rows = [preference_row, preference_row2]
111+
112+
# --- RawDataset Example ---
113+
from datafast.schema.data_rows import TextRow
114+
from datafast.schema.config import RawDatasetConfig, UltrachatDatasetConfig
115+
116+
raw_row1 = TextRow(
117+
text="SpaceX launched a new batch of Starlink satellites.",
118+
text_source="human",
119+
metadata={"date": "2025-06-30", "topic": "space"}
120+
)
121+
raw_row2 = TextRow(
122+
text="The James Webb Space Telescope captured new images of a distant galaxy.",
123+
text_source="synthetic",
124+
metadata={"date": "2025-06-29", "topic": "astronomy"}
125+
)
126+
raw_config = RawDatasetConfig(document_types=["news_article", "science_report"], topics=["space", "astronomy"])
127+
raw_dataset = RawDataset(raw_config)
128+
raw_dataset.data_rows = [raw_row1, raw_row2]
129+
130+
# --- UltrachatDataset Example ---
131+
from datafast.schema.data_rows import ChatRow
132+
ultrachat_row1 = ChatRow(
133+
opening_question="How can we reduce space debris?",
134+
persona="space policy expert",
135+
messages=[{"role": "user", "content": "What are current efforts to clean up space debris?"}, {"role": "assistant", "content": "There are several ongoing projects, such as RemoveDEBRIS and ClearSpace-1."}],
136+
model_id="gpt-4.1-nano",
137+
metadata={"language": "en"}
138+
)
139+
ultrachat_row2 = ChatRow(
140+
opening_question="What is the importance of the Moon missions?",
141+
persona="lunar geologist",
142+
messages=[{"role": "user", "content": "Why do we keep returning to the Moon?"}, {"role": "assistant", "content": "The Moon offers scientific insights and is a stepping stone for Mars exploration."}],
143+
model_id="gemini-2.0-flash",
144+
metadata={"language": "en"}
145+
)
146+
ultrachat_config = UltrachatDatasetConfig()
147+
ultrachat_dataset = UltrachatDataset(ultrachat_config)
148+
ultrachat_dataset.data_rows = [ultrachat_row1, ultrachat_row2]
149+
150+
if __name__ == "__main__":
151+
# print("Showing ClassificationDataset example...")
152+
# inspect_classification_dataset(classification_dataset)
153+
# print("Showing MCQDataset example...")
154+
# inspect_mcq_dataset(mcq_dataset)
155+
# print("Showing PreferenceDataset example...")
156+
# inspect_preference_dataset(preference_dataset)
157+
# print("Showing RawDataset example...")
158+
# inspect_raw_dataset(raw_dataset)
159+
# print("Showing UltrachatDataset example...")
160+
# inspect_ultrachat_dataset(ultrachat_dataset)

0 commit comments

Comments
 (0)