Skip to content

Commit dee8d1f

Browse files
jdaltonjdalton
committed
test(e2e): route every state-touching CLI call through executeCliInScratch
Audit + harden of the e2e suite's isolation guarantees. The Phase 1 helper change pinned npm/pnpm/yarn/pip/cargo/gradle dirs into the scratch HOME; this commit makes sure every test that COULD touch the dev's filesystem, npm cache, real Socket account, or org-side state actually USES that helper. Files updated (per-file conversion of executeCliCommand → executeCliInScratch for state-touching calls): - scan.e2e.test.mts: all auth-required read calls (list/view/metadata/ report/diff) + the --org fake_org error paths now scratch. - organization.e2e.test.mts: list/policy/quota, --org real / --org trash, config-driven org resolution. - package.e2e.test.mts: every score/shallow against the real API. - analytics.e2e.test.mts: default/org/repo/time-window calls plus the unknown-repo error path (which hits the API for validation). - manifest.e2e.test.mts: every no-args generator invocation (auto, conda, gradle, kotlin, scala) — the generators walk cwd looking for build files; scratch cwd keeps them out of socket-cli/. - repos.e2e.test.mts: read-only error paths + the destructive round-trip (create / update / view / del + afterAll cleanup). - config.e2e.test.mts: `config get defaultOrg` no longer reads the dev's real config even with --config '{}' injection. - package-managers.e2e.test.mts: npm info, npx cowsay, raw-npm info, raw-npx cowsay, raw-npm (no args), cdxgen, all organization dependencies API calls. - audit-fix-ci.e2e.test.mts: audit-log no-args (real API hit). - auth.e2e.test.mts: whoami. - threat-feed.e2e.test.mts: every non-help/dry-run call. What stays on executeCliCommand (intentional): - --help / --dry-run calls (argparser short-circuits before any IO). - Argparse-error checks like `analytics --whatnow` or `config` with no subcommand (exit 2 before any IO). - `oops` (no args, deliberate failure, no IO). - The single beforeAll() read in organization.e2e that needs the dev's real defaultOrg — uses `isolateConfig: false` explicitly. Suite still collects 315 tests across 15 files. Skip behavior unchanged when RUN_E2E_TESTS is unset.
1 parent a54e760 commit dee8d1f

11 files changed

Lines changed: 139 additions & 116 deletions

‎packages/cli/test/e2e/analytics.e2e.test.mts‎

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -34,52 +34,52 @@ describe('socket analytics (e2e)', () => {
3434
})
3535
})
3636

37-
describe('default / org / repo / time-window (auth required)', () => {
37+
describe('default / org / repo / time-window (auth required, scratch-isolated)', () => {
3838
it.skipIf(!RUN)('analytics (default scope) exits 0', async () => {
39-
const result = await executeCliCommand(['analytics'])
39+
const result = await executeCliInScratch(['analytics'])
4040
expect(result.code).toBe(0)
4141
})
4242

4343
it.skipIf(!RUN)('analytics --markdown exits 0', async () => {
44-
const result = await executeCliCommand(['analytics', '--markdown'])
44+
const result = await executeCliInScratch(['analytics', '--markdown'])
4545
expect(result.code).toBe(0)
4646
})
4747

4848
it.skipIf(!RUN)('analytics --json conforms to contract', async () => {
49-
const result = await executeCliCommand(['analytics', '--json'])
49+
const result = await executeCliInScratch(['analytics', '--json'])
5050
expect(result.code).toBe(0)
5151
validateSocketJsonContract(result.stdout, 0)
5252
})
5353

5454
it.skipIf(!RUN)('analytics org --json conforms to contract', async () => {
55-
const result = await executeCliCommand(['analytics', 'org', '--json'])
55+
const result = await executeCliInScratch(['analytics', 'org', '--json'])
5656
expect(result.code).toBe(0)
5757
validateSocketJsonContract(result.stdout, 0)
5858
})
5959

6060
it.skipIf(!RUN)('analytics repo socket-cli --json conforms to contract', async () => {
61-
const result = await executeCliCommand(['analytics', 'repo', 'socket-cli', '--json'])
61+
const result = await executeCliInScratch(['analytics', 'repo', 'socket-cli', '--json'])
6262
expect(result.code).toBe(0)
6363
validateSocketJsonContract(result.stdout, 0)
6464
})
6565

6666
it.skipIf(!RUN)('analytics org 7 --markdown exits 0', async () => {
67-
const result = await executeCliCommand(['analytics', 'org', '7', '--markdown'])
67+
const result = await executeCliInScratch(['analytics', 'org', '7', '--markdown'])
6868
expect(result.code).toBe(0)
6969
})
7070

7171
it.skipIf(!RUN)(
7272
'analytics repo socket-cli 30 --markdown exits 0',
7373
async () => {
74-
const result = await executeCliCommand([
74+
const result = await executeCliInScratch([
7575
'analytics', 'repo', 'socket-cli', '30', '--markdown',
7676
])
7777
expect(result.code).toBe(0)
7878
},
7979
)
8080

8181
it.skipIf(!RUN)('analytics 90 --json conforms to contract', async () => {
82-
const result = await executeCliCommand(['analytics', '90', '--json'])
82+
const result = await executeCliInScratch(['analytics', '90', '--json'])
8383
expect(result.code).toBe(0)
8484
validateSocketJsonContract(result.stdout, 0)
8585
})
@@ -116,7 +116,7 @@ describe('socket analytics (e2e)', () => {
116116
it.skipIf(!RUN)(
117117
'analytics repo veryunknownrepo --json (unknown repo) exits 1',
118118
async () => {
119-
const result = await executeCliCommand([
119+
const result = await executeCliInScratch([
120120
'analytics', 'repo', 'veryunknownrepo', '--json',
121121
])
122122
expect(result.code).toBe(1)

‎packages/cli/test/e2e/audit-fix-ci.e2e.test.mts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ describe('socket audit-log (e2e, auth required)', () => {
3030
})
3131

3232
it.skipIf(!RUN)('audit-log exits 0', async () => {
33-
const result = await executeCliCommand(['audit-log'])
33+
const result = await executeCliInScratch(['audit-log'])
3434
expect(result.code).toBe(0)
3535
})
3636
})

‎packages/cli/test/e2e/auth.e2e.test.mts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -80,7 +80,7 @@ describe('socket whoami (e2e, auth required)', () => {
8080
})
8181

8282
it.skipIf(!RUN || !hasAuth)('whoami exits 0 with auth present', async () => {
83-
const result = await executeCliCommand(['whoami'])
83+
const result = await executeCliInScratch(['whoami'])
8484
expect(result.code).toBe(0)
8585
})
8686
})

‎packages/cli/test/e2e/config.e2e.test.mts‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,9 @@ describe('socket config (e2e)', () => {
5252
})
5353

5454
it.skipIf(!RUN)('config get defaultOrg exits 0', async () => {
55-
const result = await executeCliCommand(['config', 'get', 'defaultOrg'])
55+
// Scratch HOME so even a stray `~/.config/socket/...` read can't leak
56+
// into the test assertion.
57+
const result = await executeCliInScratch(['config', 'get', 'defaultOrg'])
5658
expect(result.code).toBe(0)
5759
})
5860
})

‎packages/cli/test/e2e/manifest.e2e.test.mts‎

Lines changed: 11 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,10 @@
1010
import { describe, expect, it } from 'vitest'
1111

1212
import { ENV } from '../../src/constants/env.mts'
13-
import { executeCliCommand } from '../helpers/cli-execution.mts'
13+
import {
14+
executeCliCommand,
15+
executeCliInScratch,
16+
} from '../helpers/cli-execution.mts'
1417

1518
const RUN = ENV.RUN_E2E_TESTS
1619

@@ -34,7 +37,9 @@ describe('socket manifest (e2e)', () => {
3437

3538
describe('manifest auto', () => {
3639
it.skipIf(!RUN)('manifest auto (no path) exits 1', async () => {
37-
const result = await executeCliCommand(['manifest', 'auto'])
40+
// Scratch cwd so the generator's filesystem walk doesn't traverse the
41+
// dev's socket-cli repo looking for build files.
42+
const result = await executeCliInScratch(['manifest', 'auto'])
3843
expect(result.code).toBe(1)
3944
})
4045

@@ -51,7 +56,7 @@ describe('socket manifest (e2e)', () => {
5156

5257
describe('manifest conda', () => {
5358
it.skipIf(!RUN)('manifest conda (no env file) exits 1', async () => {
54-
const result = await executeCliCommand(['manifest', 'conda'])
59+
const result = await executeCliInScratch(['manifest', 'conda'])
5560
expect(result.code).toBe(1)
5661
})
5762

@@ -68,7 +73,7 @@ describe('socket manifest (e2e)', () => {
6873

6974
describe('manifest gradle', () => {
7075
it.skipIf(!RUN)('manifest gradle (no project) exits 1', async () => {
71-
const result = await executeCliCommand(['manifest', 'gradle'])
76+
const result = await executeCliInScratch(['manifest', 'gradle'])
7277
expect(result.code).toBe(1)
7378
})
7479

@@ -88,7 +93,7 @@ describe('socket manifest (e2e)', () => {
8893

8994
describe('manifest kotlin', () => {
9095
it.skipIf(!RUN)('manifest kotlin (no project) exits 1', async () => {
91-
const result = await executeCliCommand(['manifest', 'kotlin'])
96+
const result = await executeCliInScratch(['manifest', 'kotlin'])
9297
expect(result.code).toBe(1)
9398
})
9499

@@ -105,7 +110,7 @@ describe('socket manifest (e2e)', () => {
105110

106111
describe('manifest scala', () => {
107112
it.skipIf(!RUN)('manifest scala (no project) exits 1', async () => {
108-
const result = await executeCliCommand(['manifest', 'scala'])
113+
const result = await executeCliInScratch(['manifest', 'scala'])
109114
expect(result.code).toBe(1)
110115
})
111116

‎packages/cli/test/e2e/organization.e2e.test.mts‎

Lines changed: 24 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -121,39 +121,39 @@ describe('socket organization (e2e)', () => {
121121
})
122122
})
123123

124-
describe('list / policy / quota (auth required)', () => {
124+
describe('list / policy / quota (auth required, scratch-isolated)', () => {
125125
it.skipIf(!RUN || !hasAuth)('organization list exits 0', async () => {
126-
const result = await executeCliCommand(['organization', 'list'])
126+
const result = await executeCliInScratch(['organization', 'list'])
127127
expect(result.code).toBe(0)
128128
})
129129

130130
it.skipIf(!RUN || !hasAuth)('organization policy license exits 0', async () => {
131-
const result = await executeCliCommand(['organization', 'policy', 'license'])
131+
const result = await executeCliInScratch(['organization', 'policy', 'license'])
132132
expect(result.code).toBe(0)
133133
})
134134

135135
it.skipIf(!RUN || !hasAuth)('organization policy security exits 0', async () => {
136-
const result = await executeCliCommand(['organization', 'policy', 'security'])
136+
const result = await executeCliInScratch(['organization', 'policy', 'security'])
137137
expect(result.code).toBe(0)
138138
})
139139

140140
it.skipIf(!RUN || !hasAuth)('organization quota exits 0', async () => {
141-
const result = await executeCliCommand(['organization', 'quota'])
141+
const result = await executeCliInScratch(['organization', 'quota'])
142142
expect(result.code).toBe(0)
143143
})
144144

145145
it.skipIf(!RUN || !hasAuth)(
146146
'organization policy security --markdown exits 0',
147147
async () => {
148-
const result = await executeCliCommand(['organization', 'policy', 'security', '--markdown'])
148+
const result = await executeCliInScratch(['organization', 'policy', 'security', '--markdown'])
149149
expect(result.code).toBe(0)
150150
},
151151
)
152152

153153
it.skipIf(!RUN || !hasAuth)(
154154
'organization policy security --json conforms to contract',
155155
async () => {
156-
const result = await executeCliCommand(['organization', 'policy', 'security', '--json'])
156+
const result = await executeCliInScratch(['organization', 'policy', 'security', '--json'])
157157
expect(result.code).toBe(0)
158158
validateSocketJsonContract(result.stdout, 0)
159159
},
@@ -162,29 +162,29 @@ describe('socket organization (e2e)', () => {
162162
it.skipIf(!RUN || !hasAuth)(
163163
'organization policy license --markdown exits 0',
164164
async () => {
165-
const result = await executeCliCommand(['organization', 'policy', 'license', '--markdown'])
165+
const result = await executeCliInScratch(['organization', 'policy', 'license', '--markdown'])
166166
expect(result.code).toBe(0)
167167
},
168168
)
169169

170170
it.skipIf(!RUN || !hasAuth)(
171171
'organization policy license --json conforms to contract',
172172
async () => {
173-
const result = await executeCliCommand(['organization', 'policy', 'license', '--json'])
173+
const result = await executeCliInScratch(['organization', 'policy', 'license', '--json'])
174174
expect(result.code).toBe(0)
175175
validateSocketJsonContract(result.stdout, 0)
176176
},
177177
)
178178
})
179179

180-
describe('--org <real-default-org> (auth required)', () => {
180+
describe('--org <real-default-org> (auth required, scratch-isolated)', () => {
181181
it.skipIf(!RUN || !hasAuth)(
182182
'organization policy security --org <real> exits 0',
183183
async () => {
184184
if (!defaultOrg) {
185185
return
186186
}
187-
const result = await executeCliCommand([
187+
const result = await executeCliInScratch([
188188
'organization', 'policy', 'security', '--org', defaultOrg,
189189
])
190190
expect(result.code).toBe(0)
@@ -197,19 +197,19 @@ describe('socket organization (e2e)', () => {
197197
if (!defaultOrg) {
198198
return
199199
}
200-
const result = await executeCliCommand([
200+
const result = await executeCliInScratch([
201201
'organization', 'policy', 'license', '--org', defaultOrg,
202202
])
203203
expect(result.code).toBe(0)
204204
},
205205
)
206206
})
207207

208-
describe('--org trash (invalid org, auth required)', () => {
208+
describe('--org trash (invalid org, auth required, scratch-isolated)', () => {
209209
it.skipIf(!RUN || !hasAuth)(
210210
'organization policy security --org trash exits 1',
211211
async () => {
212-
const result = await executeCliCommand([
212+
const result = await executeCliInScratch([
213213
'organization', 'policy', 'security', '--org', 'trash',
214214
])
215215
expect(result.code).toBe(1)
@@ -219,7 +219,7 @@ describe('socket organization (e2e)', () => {
219219
it.skipIf(!RUN || !hasAuth)(
220220
'organization policy security --org trash --markdown exits 1',
221221
async () => {
222-
const result = await executeCliCommand([
222+
const result = await executeCliInScratch([
223223
'organization', 'policy', 'security', '--org', 'trash', '--markdown',
224224
])
225225
expect(result.code).toBe(1)
@@ -229,7 +229,7 @@ describe('socket organization (e2e)', () => {
229229
it.skipIf(!RUN || !hasAuth)(
230230
'organization policy security --org trash --json conforms to error contract',
231231
async () => {
232-
const result = await executeCliCommand([
232+
const result = await executeCliInScratch([
233233
'organization', 'policy', 'security', '--org', 'trash', '--json',
234234
])
235235
expect(result.code).toBe(1)
@@ -240,7 +240,7 @@ describe('socket organization (e2e)', () => {
240240
it.skipIf(!RUN || !hasAuth)(
241241
'organization policy license --org trash exits 1',
242242
async () => {
243-
const result = await executeCliCommand([
243+
const result = await executeCliInScratch([
244244
'organization', 'policy', 'license', '--org', 'trash',
245245
])
246246
expect(result.code).toBe(1)
@@ -250,7 +250,7 @@ describe('socket organization (e2e)', () => {
250250
it.skipIf(!RUN || !hasAuth)(
251251
'organization policy license --org trash --markdown exits 1',
252252
async () => {
253-
const result = await executeCliCommand([
253+
const result = await executeCliInScratch([
254254
'organization', 'policy', 'license', '--org', 'trash', '--markdown',
255255
])
256256
expect(result.code).toBe(1)
@@ -260,7 +260,7 @@ describe('socket organization (e2e)', () => {
260260
it.skipIf(!RUN || !hasAuth)(
261261
'organization policy license --org trash --json conforms to error contract',
262262
async () => {
263-
const result = await executeCliCommand([
263+
const result = await executeCliInScratch([
264264
'organization', 'policy', 'license', '--org', 'trash', '--json',
265265
])
266266
expect(result.code).toBe(1)
@@ -269,13 +269,13 @@ describe('socket organization (e2e)', () => {
269269
)
270270
})
271271

272-
describe('config-driven org resolution (auth required)', () => {
272+
describe('config-driven org resolution (auth required, scratch-isolated)', () => {
273273
it.skipIf(!RUN || !hasAuth)(
274274
'policy security with no defaultOrg in config exits 1 (--no-interactive)',
275275
async () => {
276276
// No `defaultOrg` in the injected config and --no-interactive prevents
277277
// the CLI from prompting; failure is the expected outcome.
278-
const result = await executeCliCommand(
278+
const result = await executeCliInScratch(
279279
['organization', 'policy', 'security', '--json', '--no-interactive'],
280280
{ config: {} },
281281
)
@@ -287,7 +287,7 @@ describe('socket organization (e2e)', () => {
287287
it.skipIf(!RUN || !hasAuth)(
288288
'policy license with no defaultOrg in config exits 1 (--no-interactive)',
289289
async () => {
290-
const result = await executeCliCommand(
290+
const result = await executeCliInScratch(
291291
['organization', 'policy', 'license', '--json', '--no-interactive'],
292292
{ config: {} },
293293
)
@@ -299,7 +299,7 @@ describe('socket organization (e2e)', () => {
299299
it.skipIf(!RUN || !hasAuth)(
300300
'policy security with defaultOrg=fake_org in config exits 1 (--no-interactive)',
301301
async () => {
302-
const result = await executeCliCommand(
302+
const result = await executeCliInScratch(
303303
['organization', 'policy', 'security', '--json', '--no-interactive'],
304304
{ config: { defaultOrg: 'fake_org' } },
305305
)
@@ -311,7 +311,7 @@ describe('socket organization (e2e)', () => {
311311
it.skipIf(!RUN || !hasAuth)(
312312
'policy license with defaultOrg=fake_org in config exits 1 (--no-interactive)',
313313
async () => {
314-
const result = await executeCliCommand(
314+
const result = await executeCliInScratch(
315315
['organization', 'policy', 'license', '--json', '--no-interactive'],
316316
{ config: { defaultOrg: 'fake_org' } },
317317
)

0 commit comments

Comments
 (0)