fix: only needed artifacts are pushed

Author: skoshx

.gitignore

@@ -2,6 +2,7 @@ dist
 node_modules
 coverage
 .DS_Store
+.env
 /test/transform/generated-*/
 /test/transform/ecosystem-repos
 # Transform

package.json

@@ -43,6 +43,7 @@
 	"dependencies": {
 		"@babel/parser": "^7.21.4",
 		"@babel/traverse": "^7.21.4",
+		"@babel/types": "^7.22.5",
 		"c12": "^1.4.1",
 		"copy-anything": "^3.0.5",
 		"fast-equals": "^5.0.1",
@@ -57,7 +58,6 @@
 		"unplugin": "^1.3.1"
 	},
 	"devDependencies": {
-		"@babel/types": "^7.21.4",
 		"@happy-dom/global-registrator": "^9.20.3",
 		"@nuxtjs/eslint-config-typescript": "latest",
 		"@rollup/plugin-commonjs": "^25.0.0",
@@ -72,6 +72,7 @@
 		"@vitest/browser": "^0.30.1",
 		"acorn": "8.8.2",
 		"degit": "^2.8.4",
+		"dotenv": "^16.3.1",
 		"eslint": "8.36.0",
 		"eslint-config-prettier": "latest",
 		"eslint-plugin-prettier": "latest",

pnpm-lock.yaml

@@ -157,6 +157,13 @@ export type Artifact = {
 	functionId?: string
 }
 
+export type DatabaseArtifact = Artifact & {
+	id: string
+	createdAt: string
+	projectId: string
+	functionId: null | undefined | string
+}
+
 export type ArtifactCacheEntry = {
 	timestamp: number
 	uploadStatus: 'not-uploaded' | 'uploaded' | 'upload-failed'

src/core/types.ts

@@ -17,7 +17,7 @@ export async function batchedArtifactsUpload(
 
 	const uploadedArtifactBatches = await Promise.all(
 		batches.map(async (batch) => {
-			const { data, error } = await post<Artifact>(
+			const { data, error } = await post<string[]>(
 				`${FLYTRAP_API_BASE}/api/v1/artifacts/${projectId}`,
 				JSON.stringify({
 					artifacts: batch
@@ -35,5 +35,5 @@ export async function batchedArtifactsUpload(
 			return data
 		})
 	)
-	return uploadedArtifactBatches
+	return uploadedArtifactBatches.reduce((acc, curr) => [...acc, ...curr], [] as string[])
 }

src/transform/artifacts/batchedArtifactsUpload.ts

@@ -2,7 +2,7 @@ import { dirname, join } from 'path'
 import { homedir } from 'os'
 import { deepEqual } from 'fast-equals'
 import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'fs'
-import { Artifact, CacheFile } from '../../exports'
+import { Artifact, CacheFile, DatabaseArtifact } from '../../exports'
 import { cwd } from 'process'
 import { formatBytes, get } from '../../core/util'
 import { FLYTRAP_API_BASE } from '../../core/config'
@@ -57,7 +57,13 @@ const _populateCache = async (projectId: string, secretApiKey: string): Promise<
 	return cache
 }
 
-async function _getArtifactsToUpload(
+function _removeDatabaseKeys(artifact: DatabaseArtifact): Artifact {
+	const { id, createdAt, projectId, ...rest } = artifact
+	if (rest.functionId === null) delete rest.functionId
+	return rest
+}
+
+export async function _getArtifactsToUpload(
 	projectId: string,
 	secretApiKey: string,
 	artifacts: Artifact[]
@@ -70,8 +76,11 @@ async function _getArtifactsToUpload(
 			artifacts[i].functionOrCallId
 		)
 		if (existingArtifactIndex !== -1) {
+			const withoutDatabaseKeys = _removeDatabaseKeys(
+				alreadyUploadedArtifacts[existingArtifactIndex]
+			)
 			// Exists, check if they're the same
-			if (!deepEqual(alreadyUploadedArtifacts[existingArtifactIndex], artifacts[i])) {
+			if (!deepEqual(withoutDatabaseKeys, artifacts[i])) {
 				artifactsToUpload.push(artifacts[i])
 				continue
 			}
@@ -161,8 +170,8 @@ export function getArtifactsToUpload(projectId: string) {
 		.reduce((acc, curr) => [...acc, curr.artifact], [] as Artifact[])
 }
 
-async function _fetchUploadedArtifacts(projectId: string, secretApiKey: string) {
-	const { data, error } = await get<Artifact[]>(
+export async function _fetchUploadedArtifacts(projectId: string, secretApiKey: string) {
+	const { data, error } = await get<DatabaseArtifact[]>(
 		`${FLYTRAP_API_BASE}/api/v1/artifacts/${projectId}`,
 		undefined,
 		{

src/transform/artifacts/cache.ts

@@ -11,13 +11,18 @@ import babelTsParser from 'recast/parsers/babel-ts.js'
 import { Identifier } from '@babel/types'
 import { flytrapTransformArtifacts } from '../src/transform/index'
 import {
+	_fetchUploadedArtifacts,
+	_getArtifactsToUpload,
 	getArtifactsToUpload,
 	getCacheFilePath,
 	markArtifactAsUploaded,
-	upsertArtifact
+	upsertArtifact,
+	upsertArtifacts
 } from '../src/transform/artifacts/cache'
 import { Artifact } from '../src/exports'
 import { rmSync } from 'fs'
+import { config } from 'dotenv'
+config()
 
 describe('Artifacts for functions', () => {
 	const arrowFunctionFixture = `
@@ -261,9 +266,17 @@ it('generates values same as transform', () => {
 })
 
 describe('Cache', () => {
+	const testEnvProjectId = process.env['FLYTRAP_PROJECT_ID']
+	const testEnvSecretApiKey = process.env['FLYTRAP_SECRET_KEY']
+	if (!testEnvProjectId) {
+		throw new Error(`Flytrap Testing Environment project ID not defined.`)
+	}
+	if (!testEnvSecretApiKey) {
+		throw new Error(`Flytrap Testing Environment secret API key not defined.`)
+	}
 	const cacheFilePath = getCacheFilePath('mock-project-id')
 	const mockArtifact: Artifact = {
-		functionOrCallId: '',
+		functionOrCallId: `mock-call-id-${new Date().toISOString()}`,
 		functionOrCallName: '',
 		params: '',
 		scopes: [],
@@ -278,6 +291,33 @@ describe('Cache', () => {
 		rmSync(cacheFilePath, { recursive: true })
 	})
 
+	it('_fetchUploadedArtifacts', async () => {
+		const uploadedArtifacts = await _fetchUploadedArtifacts(testEnvProjectId, testEnvSecretApiKey)
+		expect(
+			uploadedArtifacts.find((u) => u.functionOrCallId === mockArtifact.functionOrCallId) !==
+				undefined
+		)
+	})
+
+	it('_getArtifactsToUpload', async () => {
+		const artifactsToUpload = await _getArtifactsToUpload(testEnvProjectId, testEnvSecretApiKey, [
+			mockArtifact
+		])
+		expect(artifactsToUpload).toEqual([mockArtifact])
+	})
+
+	it('upsertArtifacts', async () => {
+		const response = await upsertArtifacts(testEnvProjectId, testEnvSecretApiKey, [mockArtifact])
+		expect(response.includes(mockArtifact.functionOrCallId))
+	})
+
+	it('_getArtifactsToUpload no longer include the mock artifact', async () => {
+		const artifactsToUpload = await _getArtifactsToUpload(testEnvProjectId, testEnvSecretApiKey, [
+			mockArtifact
+		])
+		expect(artifactsToUpload).toEqual([])
+	})
+
 	it('upserts artifact', () => {
 		upsertArtifact('mock-project-id', mockArtifact)
 	})