Improve cache hit consistency across environments (#1010)

Author: mishushakov

.changeset/sixty-mails-cheat.md

@@ -0,0 +1,6 @@
+---
+'@e2b/python-sdk': patch
+'e2b': patch
+---
+
+improve cache hit consistency

packages/js-sdk/src/template/utils.ts

@@ -71,7 +71,8 @@ export async function getAllFilesForFilesHash(
 
 /**
  * Calculate a hash of files being copied to detect changes for cache invalidation.
- * The hash includes file content, metadata (mode, uid, gid, size, mtime), and relative paths.
+ * The hash includes file content, metadata (mode, size), and relative paths.
+ * Note: uid, gid, and mtime are excluded to ensure stable hashes across environments.
  *
  * @param src Source path pattern for files to copy
  * @param dest Destination path where files will be copied
@@ -106,13 +107,11 @@ export async function calculateFilesHash(
     throw error
   }
 
-  // Hash stats
+  // Hash stats - only include stable metadata (mode, size)
+  // Exclude uid, gid, and mtime to ensure consistent hashes across environments
   const hashStats = (stats: fs.Stats) => {
     hash.update(stats.mode.toString())
-    hash.update(stats.uid.toString())
-    hash.update(stats.gid.toString())
     hash.update(stats.size.toString())
-    hash.update(stats.mtimeMs.toString())
   }
 
   // Process files recursively

packages/js-sdk/tests/setup.ts

@@ -12,21 +12,21 @@ interface SandboxFixture {
 interface BuildTemplateFixture {
   buildTemplate: (
     template: TemplateClass,
-    skipCache?: boolean,
+    options?: { skipCache?: boolean },
     onBuildLogs?: (logEntry: LogEntry) => void
   ) => Promise<void>
 }
 
 function buildTemplate(
   template: TemplateClass,
-  skipCache?: boolean,
+  options?: { skipCache?: boolean },
   onBuildLogs?: (logEntry: LogEntry) => void
 ) {
   return Template.build(template, {
     alias: `e2b-test-${randomUUID()}`,
     cpuCount: 1,
     memoryMB: 1024,
-    skipCache: skipCache,
+    skipCache: options?.skipCache,
     onBuildLogs: onBuildLogs,
   })
 }

packages/js-sdk/tests/template/build.test.ts

@@ -35,12 +35,13 @@ buildTemplateTest('build template', async ({ buildTemplate }) => {
     .setWorkdir('/app')
     .setStartCmd('echo "Hello, world!"', waitForTimeout(10_000))
 
-  await buildTemplate(template, undefined, defaultBuildLogger())
+  await buildTemplate(template, { skipCache: true }, defaultBuildLogger())
 })
 
 buildTemplateTest('build template with symlinks', async ({ buildTemplate }) => {
   const template = Template()
     .fromImage('ubuntu:22.04')
+    .skipCache()
     .copy('folder/*', 'folder', { forceUpload: true })
     .runCmd('cat folder/symlink.txt')
 
@@ -52,6 +53,7 @@ buildTemplateTest(
   async ({ buildTemplate }) => {
     const template = Template()
       .fromImage('ubuntu:22.04')
+      .skipCache()
       .copy('folder/symlink.txt', 'folder/symlink.txt', {
         forceUpload: true,
         resolveSymlinks: true,

packages/js-sdk/tests/template/methods/aptInstall.test.ts

@@ -2,7 +2,10 @@ import { Template } from '../../../src'
 import { buildTemplateTest } from '../../setup'
 
 buildTemplateTest('apt install', async ({ buildTemplate }) => {
-  const template = Template().fromImage('ubuntu:24.04').aptInstall(['vim'])
+  const template = Template()
+    .fromImage('ubuntu:24.04')
+    .skipCache()
+    .aptInstall(['vim'])
 
   await buildTemplate(template)
 })
@@ -12,6 +15,7 @@ buildTemplateTest(
   async ({ buildTemplate }) => {
     const template = Template()
       .fromImage('ubuntu:24.04')
+      .skipCache()
       .aptInstall(['vim'], { noInstallRecommends: true })
 
     await buildTemplate(template)

packages/js-sdk/tests/template/methods/bunInstall.test.ts

@@ -4,6 +4,7 @@ import { buildTemplateTest } from '../../setup'
 buildTemplateTest('bun install', async ({ buildTemplate }) => {
   const template = Template()
     .fromBunImage('1.3')
+    .skipCache()
     .bunInstall(['lodash', 'axios'])
 
   await buildTemplate(template)
@@ -12,6 +13,7 @@ buildTemplateTest('bun install', async ({ buildTemplate }) => {
 buildTemplateTest('bun install global', async ({ buildTemplate }) => {
   const template = Template()
     .fromBunImage('1.3')
+    .skipCache()
     .bunInstall(['tsx'], { g: true })
 
   await buildTemplate(template)
@@ -20,6 +22,7 @@ buildTemplateTest('bun install global', async ({ buildTemplate }) => {
 buildTemplateTest('bun install dev', async ({ buildTemplate }) => {
   const template = Template()
     .fromBunImage('1.3')
+    .skipCache()
     .bunInstall(['typescript'], { dev: true })
 
   await buildTemplate(template)

packages/js-sdk/tests/template/methods/fromMethods.test.ts

@@ -21,67 +21,67 @@ afterAll(async () => {
 
 buildTemplateTest('fromBaseImage', async ({ buildTemplate }) => {
   const template = Template().fromBaseImage()
-  await buildTemplate(template)
+  await buildTemplate(template, { skipCache: true })
 })
 
 buildTemplateTest('fromDebianImage', async ({ buildTemplate }) => {
   const template = Template().fromDebianImage()
-  await buildTemplate(template)
+  await buildTemplate(template, { skipCache: true })
 })
 
 buildTemplateTest('fromDebianImage with variant', async ({ buildTemplate }) => {
   const template = Template().fromDebianImage('bookworm')
-  await buildTemplate(template)
+  await buildTemplate(template, { skipCache: true })
 })
 
 buildTemplateTest('fromUbuntuImage', async ({ buildTemplate }) => {
   const template = Template().fromUbuntuImage()
-  await buildTemplate(template)
+  await buildTemplate(template, { skipCache: true })
 })
 
 buildTemplateTest('fromUbuntuImage with variant', async ({ buildTemplate }) => {
   const template = Template().fromUbuntuImage('24.04')
-  await buildTemplate(template)
+  await buildTemplate(template, { skipCache: true })
 })
 
 buildTemplateTest('fromPythonImage', async ({ buildTemplate }) => {
   const template = Template().fromPythonImage()
-  await buildTemplate(template)
+  await buildTemplate(template, { skipCache: true })
 })
 
 buildTemplateTest('fromPythonImage with variant', async ({ buildTemplate }) => {
   const template = Template().fromPythonImage('3.12')
-  await buildTemplate(template)
+  await buildTemplate(template, { skipCache: true })
 })
 
 buildTemplateTest('fromNodeImage', async ({ buildTemplate }) => {
   const template = Template().fromNodeImage()
-  await buildTemplate(template)
+  await buildTemplate(template, { skipCache: true })
 })
 
 buildTemplateTest('fromNodeImage with variant', async ({ buildTemplate }) => {
   const template = Template().fromNodeImage('24')
-  await buildTemplate(template)
+  await buildTemplate(template, { skipCache: true })
 })
 
 buildTemplateTest('fromBunImage', async ({ buildTemplate }) => {
   const template = Template().fromBunImage()
-  await buildTemplate(template)
+  await buildTemplate(template, { skipCache: true })
 })
 
 buildTemplateTest('fromBunImage with variant', async ({ buildTemplate }) => {
   const template = Template().fromBunImage('1.3')
-  await buildTemplate(template)
+  await buildTemplate(template, { skipCache: true })
 })
 
 buildTemplateTest('fromImage', async ({ buildTemplate }) => {
   const template = Template().fromImage('ubuntu:22.04')
-  await buildTemplate(template)
+  await buildTemplate(template, { skipCache: true })
 })
 
 buildTemplateTest('fromTemplate', async ({ buildTemplate }) => {
   const template = Template().fromTemplate('base')
-  await buildTemplate(template)
+  await buildTemplate(template, { skipCache: true })
 })
 
 buildTemplateTest('fromDockerfile', async ({ buildTemplate }) => {
@@ -91,5 +91,5 @@ COPY package.json .
 RUN npm install`
 
   const template = Template({ fileContextPath }).fromDockerfile(dockerfile)
-  await buildTemplate(template)
+  await buildTemplate(template, { skipCache: true })
 })

packages/js-sdk/tests/template/methods/makeSymlink.test.ts

@@ -4,6 +4,7 @@ import { buildTemplateTest } from '../../setup'
 buildTemplateTest('make symlink', async ({ buildTemplate }) => {
   const template = Template()
     .fromImage('ubuntu:22.04')
+    .skipCache()
     .makeSymlink('.bashrc', '.bashrc.local')
     .runCmd('test "$(readlink .bashrc.local)" = ".bashrc"')
 
@@ -13,6 +14,7 @@ buildTemplateTest('make symlink', async ({ buildTemplate }) => {
 buildTemplateTest('make symlink (force)', async ({ buildTemplate }) => {
   const template = Template()
     .fromImage('ubuntu:22.04')
+    .skipCache()
     .makeSymlink('.bashrc', '.bashrc.local')
     .makeSymlink('.bashrc', '.bashrc.local', { force: true }) // Overwrite existing symlink
     .runCmd('test "$(readlink .bashrc.local)" = ".bashrc"')

packages/js-sdk/tests/template/methods/npmInstall.test.ts

@@ -4,6 +4,7 @@ import { buildTemplateTest } from '../../setup'
 buildTemplateTest('npm install', async ({ buildTemplate }) => {
   const template = Template()
     .fromNodeImage('24')
+    .skipCache()
     .npmInstall(['lodash', 'axios'])
 
   await buildTemplate(template)
@@ -12,6 +13,7 @@ buildTemplateTest('npm install', async ({ buildTemplate }) => {
 buildTemplateTest('npm install global', async ({ buildTemplate }) => {
   const template = Template()
     .fromNodeImage('24')
+    .skipCache()
     .npmInstall(['tsx'], { g: true })
 
   await buildTemplate(template)
@@ -20,6 +22,7 @@ buildTemplateTest('npm install global', async ({ buildTemplate }) => {
 buildTemplateTest('npm install dev', async ({ buildTemplate }) => {
   const template = Template()
     .fromNodeImage('24')
+    .skipCache()
     .npmInstall(['typescript'], { dev: true })
 
   await buildTemplate(template)

packages/js-sdk/tests/template/methods/pipInstall.test.ts

@@ -4,6 +4,7 @@ import { buildTemplateTest } from '../../setup'
 buildTemplateTest('pip install', async ({ buildTemplate }) => {
   const template = Template()
     .fromPythonImage('3.13.7-trixie')
+    .skipCache()
     .pipInstall(['six', 'pyyaml'])
 
   await buildTemplate(template)
@@ -12,6 +13,7 @@ buildTemplateTest('pip install', async ({ buildTemplate }) => {
 buildTemplateTest('pip install (user)', async ({ buildTemplate }) => {
   const template = Template()
     .fromPythonImage('3.13.7-trixie')
+    .skipCache()
     .pipInstall(['six', 'pyyaml'], { g: false })
 
   await buildTemplate(template)