feat: populate artifacts cache from the api

Author: skoshx

src/core/encryption.ts

@@ -1,3 +1,6 @@
+import * as importedCrypto from 'crypto'
+const crypto = typeof window !== 'undefined' ? window.crypto : importedCrypto
+
 const MAX_CHUNK_SIZE = 190 // 2048 bits RSA-OAEP key size, minus padding (256 bits)
 const CHUNK_SEPARATOR = '|'
 

src/core/storage.ts

@@ -169,12 +169,6 @@ export const liveFlytrapStorage: FlytrapStorage = {
 			throw errorLog.toString()
 		}
 
-		log.info(
-			'api-calls',
-			`[GET] ${FLYTRAP_API_BASE}/api/v1/captures/${captureId} - Received ${data}`,
-			{ data }
-		)
-
 		const decryptedCapture = await decryptCapture(data, privateKey)
 
 		log.info('storage', `Loaded capture ID "${captureId}".`)
@@ -239,12 +233,6 @@ export const liveFlytrapStorage: FlytrapStorage = {
 			return
 		}
 
-		log.info(
-			'api-calls',
-			`[POST] ${FLYTRAP_API_BASE}/api/v1/captures - Payload size ~${stringifiedPayload.length} bytes. Payload:`,
-			{ payload: newPayload }
-		)
-
 		const { error: captureError } = await post(
 			`${FLYTRAP_API_BASE}/api/v1/captures`,
 			stringifiedPayload,

src/core/types.ts

@@ -11,6 +11,7 @@ export type LogGroup =
 	| 'identify'
 	| 'capture'
 	| 'transform'
+	| 'cache'
 
 export type FlytrapConfig = {
 	projectId: string

src/core/util.ts

@@ -1,5 +1,6 @@
 import { getLoadedConfig } from './config'
 import { FLYTRAP_REPLACE_VALUES, FLYTRAP_UNSERIALIZABLE_VALUE } from './constants'
+import { log } from './logging'
 import { SourceType } from './types'
 
 /**
@@ -41,6 +42,11 @@ export async function request<DataType = unknown, ErrorType = string>(
 	body?: BodyInit,
 	options: RequestInit = {}
 ): Promise<RequestResponse<DataType, ErrorType>> {
+	log.info(
+		'api-calls',
+		`[🐛 ${method}] ${endpoint} - Size ${formatBytes(JSON.stringify(body).length)}.`,
+		{ payload: body }
+	)
 	try {
 		const data = await fetch(endpoint, {
 			...options,
@@ -213,3 +219,9 @@ export function fillUnserializableFlytrapValues(
 	}
 	return input
 }
+
+export const formatBytes = (bytes: number) => {
+	if (bytes === 0) return '0 B'
+	const e = Math.floor(Math.log(bytes) / Math.log(1000))
+	return `${(bytes / Math.pow(1000, e)).toFixed(2)} ${' KMGTP'.charAt(e)}B`
+}

src/transform.ts

@@ -19,7 +19,8 @@ import { getFileExtension } from './transform/util'
 import {
 	getArtifactsToUpload,
 	markArtifactAsUploaded,
-	upsertArtifact
+	upsertArtifact,
+	upsertArtifacts
 } from './transform/artifacts/cache'
 import { Artifact } from './exports'
 
@@ -228,23 +229,9 @@ export const unpluginOptions: UnpluginOptions = {
 				}
 			}
 
-			// Cache artifacts
-			for (let i = 0; i < artifacts.length; i++) {
-				upsertArtifact(config.projectId, artifacts[i])
-			}
-
-			// Upload artifacts marked as `not-uploaded`
-			const artifactsToUpload = getArtifactsToUpload(config.projectId)
-			log.info(
-				'storage',
-				`Created ${artifactsToUpload.length} new artifacts to upload. Size: ${
-					JSON.stringify(artifactsToUpload).length
-				}`
-			)
-
+			// Upload artifacts
 			const { data: uploadedBatches, error } = await tryCatch(
-				// batchedArtifactsUpload(artifacts, config.secretApiKey, config.projectId)
-				batchedArtifactsUpload(artifactsToUpload, config.secretApiKey, config.projectId)
+				upsertArtifacts(config.projectId, config.secretApiKey, artifacts)
 			)
 			if (error || !uploadedBatches) {
 				console.error(
@@ -253,18 +240,6 @@ export const unpluginOptions: UnpluginOptions = {
 				console.error(error)
 				return
 			}
-
-			for (let i = 0; i < uploadedBatches.length; i++) {
-				markArtifactAsUploaded(config.projectId, uploadedBatches[i])
-			}
-			log.info(
-				'api-calls',
-				`Pushed ${artifactsToUpload.length} artifacts in ${
-					uploadedBatches?.length
-				} batches to the Flytrap API. Payload size: ~${
-					JSON.stringify(artifactsToUpload).length
-				} bytes.`
-			)
 		}
 	}
 }

src/transform/artifacts/cache.ts

@@ -4,6 +4,10 @@ import { deepEqual } from 'fast-equals'
 import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'fs'
 import { Artifact, CacheFile } from '../../exports'
 import { cwd } from 'process'
+import { formatBytes, get } from '../../core/util'
+import { FLYTRAP_API_BASE } from '../../core/config'
+import { log } from '../../core/logging'
+import { batchedArtifactsUpload } from './batchedArtifactsUpload'
 
 export const getCacheFilePath = (projectId: string) => {
 	if (process.env.NODE_ENV === 'test') {
@@ -22,6 +26,7 @@ function _createCacheFile(projectId: string): string {
 			createdTimestamp: Date.now(),
 			artifactCacheEntries: []
 		}
+		log.info('cache', `Creating cache file at path ${cacheFilePath}.`)
 		writeFileSync(cacheFilePath, JSON.stringify(cache, null, 2))
 	}
 	return cacheFilePath
@@ -38,6 +43,65 @@ const _getCacheFile = (projectId: string): CacheFile => {
 	return JSON.parse(cacheFileContents) as CacheFile
 }
 
+const _populateCache = async (projectId: string, secretApiKey: string): Promise<CacheFile> => {
+	const artifacts = await _fetchUploadedArtifacts(projectId, secretApiKey)
+	const cache = _getCacheFile(projectId)
+	for (let i = 0; i < artifacts.length; i++) {
+		cache.artifactCacheEntries.push({
+			artifact: artifacts[i],
+			timestamp: Date.now(),
+			uploadStatus: 'uploaded'
+		})
+	}
+	_saveCacheFile(projectId, cache)
+	return cache
+}
+
+async function _getArtifactsToUpload(
+	projectId: string,
+	secretApiKey: string,
+	artifacts: Artifact[]
+) {
+	const alreadyUploadedArtifacts = await _fetchUploadedArtifacts(projectId, secretApiKey)
+	const artifactsToUpload: Artifact[] = []
+	for (let i = 0; i < artifacts.length; i++) {
+		const existingArtifactIndex = _indexOfArtifactWithId(
+			alreadyUploadedArtifacts,
+			artifacts[i].functionOrCallId
+		)
+		if (existingArtifactIndex !== -1) {
+			// Exists, check if they're the same
+			if (!deepEqual(alreadyUploadedArtifacts[existingArtifactIndex], artifacts[i])) {
+				artifactsToUpload.push(artifacts[i])
+				continue
+			}
+		} else {
+			// Doesn't exist in uploaded artifacts
+			artifactsToUpload.push(artifacts[i])
+		}
+	}
+	return artifactsToUpload
+}
+
+export async function upsertArtifacts(
+	projectId: string,
+	secretApiKey: string,
+	artifacts: Artifact[]
+) {
+	const artifactsToUpload = await _getArtifactsToUpload(projectId, secretApiKey, artifacts)
+	// Upload artifacts
+	const uploadedBatches = await batchedArtifactsUpload(artifactsToUpload, secretApiKey, projectId)
+	log.info(
+		'storage',
+		`Pushed ${artifactsToUpload.length} artifacts in ${
+			uploadedBatches?.length
+		} batches to the Flytrap API. Payload Size: ${formatBytes(
+			JSON.stringify(artifactsToUpload).length
+		)}`
+	)
+	return uploadedBatches
+}
+
 const _indexOfArtifactWithId = (haystack: Artifact[], functionOrCallId: string) =>
 	haystack.findIndex((h) => h.functionOrCallId === functionOrCallId)
 
@@ -96,3 +160,21 @@ export function getArtifactsToUpload(projectId: string) {
 		.filter((a) => a.uploadStatus !== 'uploaded')
 		.reduce((acc, curr) => [...acc, curr.artifact], [] as Artifact[])
 }
+
+async function _fetchUploadedArtifacts(projectId: string, secretApiKey: string) {
+	const { data, error } = await get<Artifact[]>(
+		`${FLYTRAP_API_BASE}/v1/artifacts/${projectId}`,
+		undefined,
+		{
+			headers: new Headers({
+				Authorization: `Bearer ${secretApiKey}`,
+				'Content-Type': 'application/json'
+			})
+		}
+	)
+	if (error || data === null) {
+		throw error
+	}
+
+	return data
+}