Fix ignore files in file upload and added tests (#1012)

<!-- CURSOR_SUMMARY -->
> [!NOTE]
> Ensure file uploads honor ignore patterns (incl. .dockerignore),
refactor file discovery/tar streaming in JS/Python SDKs, and add
comprehensive tests.
> 
> - **File upload behavior**:
> - JS/TS and Python uploads now pass `ignorePatterns` (merged from
`fileIgnorePatterns` and `.dockerignore`) to tar creation, ensuring
ignored files aren’t uploaded.
> - **Refactor/Utilities**:
> - Rename `getAllFilesForFilesHash` -> `getAllFilesInPath` with
optional directory inclusion in both SDKs.
> - Implement `tar_file_stream` (JS and Python) to build archives from
`getAllFilesInPath`, with `noDirRecurse` and symlink control.
>   - Hashing functions updated to use `getAllFilesInPath`.
> - Type tweak: allow `None` in Python `stack_traces` during build wait.
> - **Integration**:
> - JS `uploadFile` and Python async/sync `upload_file` now use tar
streaming with ignore patterns; JS `Template` wires ignore patterns into
uploads.
> - **Tests**:
> - Add extensive tests for `getAllFilesInPath` and `tar_file_stream` in
JS and Python, covering ignore patterns, directories, sorting, and
symlinks.
>   - Remove obsolete tests relying on old function names.
> - **Meta**:
>   - Changeset entries for `@e2b/python-sdk` and `e2b` (patch).
> 
> <sup>Written by [Cursor
Bugbot](https://cursor.com/dashboard?tab=bugbot) for commit
2f83444. This will update automatically
on new commits. Configure
[here](https://cursor.com/dashboard?tab=bugbot).</sup>
<!-- /CURSOR_SUMMARY -->

Author: mishushakov

.changeset/old-carrots-greet.md

@@ -0,0 +1,6 @@
+---
+'@e2b/python-sdk': patch
+'e2b': patch
+---
+
+fixes ignore files in file upload

packages/js-sdk/src/template/buildApi.ts

@@ -91,15 +91,18 @@ export async function uploadFile(
     fileName: string
     fileContextPath: string
     url: string
+    ignorePatterns: string[]
     resolveSymlinks: boolean
   },
   stackTrace: string | undefined
 ) {
-  const { fileName, url, fileContextPath, resolveSymlinks } = options
+  const { fileName, url, fileContextPath, ignorePatterns, resolveSymlinks } =
+    options
   try {
     const { contentLength, uploadStream } = await tarFileStreamUpload(
       fileName,
       fileContextPath,
+      ignorePatterns,
       resolveSymlinks
     )
 

packages/js-sdk/src/template/index.ts

@@ -925,6 +925,10 @@ export class TemplateBase
               fileName: src,
               fileContextPath: this.fileContextPath.toString(),
               url,
+              ignorePatterns: [
+                ...this.fileIgnorePatterns,
+                ...readDockerignore(this.fileContextPath.toString()),
+              ],
               resolveSymlinks: instruction.resolveSymlinks ?? RESOLVE_SYMLINKS,
             },
             stackTrace

packages/js-sdk/src/template/utils.ts

@@ -32,10 +32,11 @@ export function readDockerignore(contextPath: string): string[] {
  * @param ignorePatterns Ignore patterns
  * @returns Array of files
  */
-export async function getAllFilesForFilesHash(
+export async function getAllFilesInPath(
   src: string,
   contextPath: string,
-  ignorePatterns: string[]
+  ignorePatterns: string[],
+  includeDirectories: boolean = true
 ) {
   const { glob } = await dynamicGlob()
   const files = new Map<string, Path>()
@@ -50,7 +51,9 @@ export async function getAllFilesForFilesHash(
   for (const file of globFiles) {
     if (file.isDirectory()) {
       // For directories, add the directory itself and all files inside it
-      files.set(file.fullpath(), file)
+      if (includeDirectories) {
+        files.set(file.fullpath(), file)
+      }
       const dirFiles = await glob(
         path.join(path.relative(contextPath, file.fullpath()), '**/*'),
         {
@@ -97,7 +100,7 @@ export async function calculateFilesHash(
 
   hash.update(content)
 
-  const files = await getAllFilesForFilesHash(src, contextPath, ignorePatterns)
+  const files = await getAllFilesInPath(src, contextPath, ignorePatterns, true)
 
   if (files.length === 0) {
     const error = new Error(`No files found in ${srcPath}`)
@@ -233,25 +236,37 @@ export function padOctal(mode: number): string {
  *
  * @param fileName Glob pattern for files to include
  * @param fileContextPath Base directory for resolving file paths
+ * @param ignorePatterns Ignore patterns to exclude from the archive
  * @param resolveSymlinks Whether to follow symbolic links
  * @returns A readable stream of the gzipped tar archive
  */
 export async function tarFileStream(
   fileName: string,
   fileContextPath: string,
+  ignorePatterns: string[],
   resolveSymlinks: boolean
 ) {
-  const { globSync } = await dynamicGlob()
   const { create } = await dynamicTar()
-  const files = globSync(fileName, { cwd: fileContextPath })
+
+  const allFiles = await getAllFilesInPath(
+    fileName,
+    fileContextPath,
+    ignorePatterns,
+    true
+  )
+
+  const filePaths = allFiles.map((file) =>
+    path.relative(fileContextPath, file.fullpath())
+  )
 
   return create(
     {
       gzip: true,
       cwd: fileContextPath,
       follow: resolveSymlinks,
+      noDirRecurse: true,
     },
-    files
+    filePaths
   )
 }
 
@@ -266,12 +281,14 @@ export async function tarFileStream(
 export async function tarFileStreamUpload(
   fileName: string,
   fileContextPath: string,
+  ignorePatterns: string[],
   resolveSymlinks: boolean
 ) {
   // First pass: calculate the compressed size
   const sizeCalculationStream = await tarFileStream(
     fileName,
     fileContextPath,
+    ignorePatterns,
     resolveSymlinks
   )
   let contentLength = 0
@@ -284,6 +301,7 @@ export async function tarFileStreamUpload(
     uploadStream: await tarFileStream(
       fileName,
       fileContextPath,
+      ignorePatterns,
       resolveSymlinks
     ),
   }

…te/utils/getAllFilesForFilesHash.test.ts …template/utils/getAllFilesInPath.test.tspackages/js-sdk/tests/template/utils/getAllFilesForFilesHash.test.ts renamed to packages/js-sdk/tests/template/utils/getAllFilesInPath.test.ts packages/js-sdk/tests/template/utils/getAllFilesForFilesHash.test.ts renamed to packages/js-sdk/tests/template/utils/getAllFilesInPath.test.ts

@@ -1,9 +1,9 @@
 import { expect, test, describe, beforeAll, afterAll, beforeEach } from 'vitest'
 import { writeFile, mkdir, rm } from 'fs/promises'
 import { join } from 'path'
-import { getAllFilesForFilesHash } from '../../../src/template/utils'
+import { getAllFilesInPath } from '../../../src/template/utils'
 
-describe('getAllFilesForFilesHash', () => {
+describe('getAllFilesInPath', () => {
   const testDir = join(__dirname, 'folder')
 
   beforeAll(async () => {
@@ -26,7 +26,7 @@ describe('getAllFilesForFilesHash', () => {
     await writeFile(join(testDir, 'file2.txt'), 'content2')
     await writeFile(join(testDir, 'file3.js'), 'content3')
 
-    const files = await getAllFilesForFilesHash('*.txt', testDir, [])
+    const files = await getAllFilesInPath('*.txt', testDir, [])
 
     expect(files).toHaveLength(2)
     expect(files.some((f) => f.fullpath().endsWith('file1.txt'))).toBe(true)
@@ -51,7 +51,7 @@ describe('getAllFilesForFilesHash', () => {
     )
     await writeFile(join(testDir, 'README.md'), 'readme content')
 
-    const files = await getAllFilesForFilesHash('src', testDir, [])
+    const files = await getAllFilesInPath('src', testDir, [])
 
     expect(files).toHaveLength(6) // 3 files + 3 directories (src, components, utils)
     expect(files.some((f) => f.fullpath().endsWith('index.ts'))).toBe(true)
@@ -67,7 +67,7 @@ describe('getAllFilesForFilesHash', () => {
     await writeFile(join(testDir, 'temp.txt'), 'temp content')
     await writeFile(join(testDir, 'backup.txt'), 'backup content')
 
-    const files = await getAllFilesForFilesHash('*.txt', testDir, [
+    const files = await getAllFilesInPath('*.txt', testDir, [
       'temp*',
       'backup*',
     ])
@@ -105,7 +105,7 @@ describe('getAllFilesForFilesHash', () => {
       'spec content'
     )
 
-    const files = await getAllFilesForFilesHash('src', testDir, [
+    const files = await getAllFilesInPath('src', testDir, [
       '**/*.test.*',
       '**/*.spec.*',
     ])
@@ -126,7 +126,7 @@ describe('getAllFilesForFilesHash', () => {
     await mkdir(join(testDir, 'empty'), { recursive: true })
     await writeFile(join(testDir, 'file.txt'), 'content')
 
-    const files = await getAllFilesForFilesHash('empty', testDir, [])
+    const files = await getAllFilesInPath('empty', testDir, [])
 
     expect(files).toHaveLength(1) // The empty directory itself
   })
@@ -138,7 +138,7 @@ describe('getAllFilesForFilesHash', () => {
     await writeFile(join(testDir, 'dir1', 'file2.txt'), 'content2')
     await writeFile(join(testDir, 'file3.txt'), 'content3')
 
-    const files = await getAllFilesForFilesHash('*', testDir, [])
+    const files = await getAllFilesInPath('*', testDir, [])
 
     expect(files).toHaveLength(4) // 3 files + 1 directory
     expect(files.some((f) => f.fullpath().endsWith('file1.txt'))).toBe(true)
@@ -166,7 +166,7 @@ describe('getAllFilesForFilesHash', () => {
       'css content'
     )
 
-    const files = await getAllFilesForFilesHash('src/**/*', testDir, [])
+    const files = await getAllFilesInPath('src/**/*', testDir, [])
 
     expect(files).toHaveLength(6) // 4 files + 2 directories (components, utils)
     expect(files.some((f) => f.fullpath().endsWith('index.ts'))).toBe(true)
@@ -181,7 +181,7 @@ describe('getAllFilesForFilesHash', () => {
     await writeFile(join(testDir, 'file3.tsx'), 'tsx content')
     await writeFile(join(testDir, 'file4.css'), 'css content')
 
-    const files = await getAllFilesForFilesHash('*.ts', testDir, [])
+    const files = await getAllFilesInPath('*.ts', testDir, [])
 
     expect(files).toHaveLength(1)
     expect(files.some((f) => f.fullpath().endsWith('file1.ts'))).toBe(true)
@@ -192,7 +192,7 @@ describe('getAllFilesForFilesHash', () => {
     await writeFile(join(testDir, 'apple.txt'), 'a content')
     await writeFile(join(testDir, 'banana.txt'), 'b content')
 
-    const files = await getAllFilesForFilesHash('*.txt', testDir, [])
+    const files = await getAllFilesInPath('*.txt', testDir, [])
 
     expect(files).toHaveLength(3)
     // Files are sorted by full path, not just filename
@@ -203,7 +203,7 @@ describe('getAllFilesForFilesHash', () => {
   test('should handle no matching files', async () => {
     await writeFile(join(testDir, 'file.txt'), 'content')
 
-    const files = await getAllFilesForFilesHash('*.js', testDir, [])
+    const files = await getAllFilesInPath('*.js', testDir, [])
 
     expect(files).toHaveLength(0)
   })
@@ -232,7 +232,7 @@ describe('getAllFilesForFilesHash', () => {
     await writeFile(join(testDir, 'dist', 'bundle.js'), 'bundle content')
     await writeFile(join(testDir, 'README.md'), 'readme content')
 
-    const files = await getAllFilesForFilesHash('src', testDir, [
+    const files = await getAllFilesInPath('src', testDir, [
       '**/tests/**',
       '**/*.spec.*',
     ])