Skip to content

Commit 56165f0

Browse files
galzilbergalzilber
authored
fix(datasets): add files support to datasets (#836)
1 parent ff80677 commit 56165f0

File tree

19 files changed

+3585
-77
lines changed

19 files changed

+3585
-77
lines changed

‎packages/sample-app/src/sample_dataset_files.ts‎

Lines changed: 156 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,156 @@
1+
import * as traceloop from "@traceloop/node-server-sdk";
2+
3+
const main = async () => {
4+
traceloop.initialize({
5+
appName: "sample_dataset_files",
6+
apiKey: process.env.TRACELOOP_API_KEY,
7+
disableBatch: true,
8+
traceloopSyncEnabled: true,
9+
});
10+
11+
try {
12+
await traceloop.waitForInitialization();
13+
} catch (error) {
14+
console.error(
15+
"Failed to initialize Traceloop SDK:",
16+
error instanceof Error ? error.message : String(error),
17+
);
18+
process.exit(1);
19+
}
20+
21+
const client = traceloop.getClient();
22+
if (!client) {
23+
console.error("Failed to initialize Traceloop client");
24+
return;
25+
}
26+
27+
console.log("Dataset with Attachments Sample");
28+
console.log("================================\n");
29+
30+
try {
31+
// 1. Create a new dataset for documents
32+
console.log("Creating dataset...");
33+
const dataset = await client.datasets.create({
34+
name: `documents-dataset-${Date.now()}`,
35+
description: "Dataset with file attachments",
36+
});
37+
console.log(`Created dataset: ${dataset.name} (slug: ${dataset.slug})\n`);
38+
39+
// 2. Add columns including a file column
40+
console.log("Adding columns...");
41+
await dataset.addColumn([
42+
{ name: "title", type: "string", required: true },
43+
{ name: "category", type: "string" },
44+
{ name: "document", type: "file" },
45+
{ name: "processed", type: "boolean" },
46+
]);
47+
console.log("Added 4 columns (including file type)\n");
48+
49+
// 3. Add row with external URL attachment
50+
console.log("Adding row with external URL attachment...");
51+
const externalRows = await dataset.addRows([
52+
{
53+
title: "Sample PDF Document",
54+
category: "documentation",
55+
document: traceloop.attachment.url(
56+
"https://www.w3.org/WAI/WCAG21/Techniques/pdf/img/table-word.jpg",
57+
{
58+
fileType: "image",
59+
metadata: { source: "w3.org" },
60+
},
61+
),
62+
processed: false,
63+
},
64+
]);
65+
console.log(`Added row with external attachment\n`);
66+
67+
// Check the attachment reference
68+
const externalRow = externalRows[0];
69+
const externalRef = externalRow.getAttachment("document");
70+
if (externalRef) {
71+
console.log(` Storage type: ${externalRef.storageType}`);
72+
console.log(` File type: ${externalRef.fileType}`);
73+
console.log(` URL: ${externalRef.url}\n`);
74+
}
75+
76+
// 4. Add row with buffer attachment
77+
console.log("Adding row with buffer attachment...");
78+
const textContent = "This is sample text content for the document.";
79+
const bufferRows = await dataset.addRows([
80+
{
81+
title: "Text Document",
82+
category: "notes",
83+
document: traceloop.attachment.buffer(
84+
Buffer.from(textContent),
85+
"notes.txt",
86+
{
87+
contentType: "text/plain",
88+
metadata: { author: "sample-app" },
89+
},
90+
),
91+
processed: true,
92+
},
93+
]);
94+
console.log(`Added row with buffer attachment\n`);
95+
96+
const bufferRow = bufferRows[0];
97+
const bufferRef = bufferRow.getAttachment("document");
98+
if (bufferRef) {
99+
console.log(` Storage type: ${bufferRef.storageType}`);
100+
console.log(` File type: ${bufferRef.fileType}\n`);
101+
}
102+
103+
// 5. Add row without attachment, then set it later
104+
console.log("Adding row and setting attachment later...");
105+
const emptyRows = await dataset.addRows([
106+
{
107+
title: "Document to update",
108+
category: "pending",
109+
document: null,
110+
processed: false,
111+
},
112+
]);
113+
114+
const rowToUpdate = emptyRows[0];
115+
const updatedRef = await rowToUpdate.setAttachment(
116+
"document",
117+
traceloop.attachment.url("https://example.com/updated-document.pdf", {
118+
fileType: "file",
119+
}),
120+
);
121+
console.log(`Set attachment on existing row`);
122+
console.log(` Storage type: ${updatedRef.storageType}\n`);
123+
124+
// 6. Get all rows and display summary
125+
console.log("Dataset summary:");
126+
const allRows = await dataset.getRows();
127+
console.log(` Total rows: ${allRows.length}`);
128+
129+
for (const row of allRows) {
130+
const att = row.getAttachment("document");
131+
console.log(
132+
` - ${row.data.title}: ${att ? att.storageType : "no attachment"}`,
133+
);
134+
}
135+
136+
// 7. Clean up
137+
console.log("\nCleaning up...");
138+
await client.datasets.delete(dataset.slug);
139+
console.log(`Deleted dataset: ${dataset.slug}`);
140+
141+
console.log("\nSample completed successfully!");
142+
} catch (error) {
143+
console.error(
144+
"Error:",
145+
error instanceof Error ? error.message : String(error),
146+
);
147+
if (error instanceof Error && error.stack) {
148+
console.error("Stack:", error.stack);
149+
}
150+
}
151+
};
152+
153+
main().catch((error) => {
154+
console.error("Application failed:", error.message);
155+
process.exit(1);
156+
});

‎packages/traceloop-sdk/package.json‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,7 @@
8282
"cross-fetch": "^4.1.0",
8383
"eventsource": "^3.0.2",
8484
"fetch-retry": "^6.0.0",
85+
"mime-types": "^3.0.2",
8586
"nunjucks": "^3.2.4",
8687
"papaparse": "^5.5.3",
8788
"supports-color": "^10.0.0",
@@ -106,6 +107,7 @@
106107
"@pollyjs/core": "^6.0.6",
107108
"@pollyjs/persister-fs": "^6.0.6",
108109
"@qdrant/js-client-rest": "^1.15.0",
110+
"@types/mime-types": "^3.0.1",
109111
"@types/mocha": "^10.0.10",
110112
"@types/node": "^24.0.15",
111113
"@types/papaparse": "^5.3.16",

0 commit comments

Comments
 (0)