diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 000000000..6c5049e25 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,10 @@ +version: 2 +updates: + - package-ecosystem: github-actions + directory: / + schedule: + interval: weekly + groups: + github-actions: + patterns: + - "*" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 000000000..3ea567733 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,26 @@ +name: CI + +permissions: + contents: read + +on: + push: + branches: + - main + pull_request: + +jobs: + check: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v6.0.2 + + - name: Setup Deno + uses: denoland/setup-deno@v2.0.4 + with: + deno-version: v2.x + cache: true + + - name: Check formatter, linter, and types + run: deno task check diff --git a/.github/workflows/update.yml b/.github/workflows/update.yml index 0da831b3c..a10b14afd 100644 --- a/.github/workflows/update.yml +++ b/.github/workflows/update.yml @@ -3,26 +3,35 @@ name: Update APIs permissions: contents: write pages: write + id-token: write on: schedule: - cron: "0 0 * * *" workflow_dispatch: +concurrency: + group: pages + cancel-in-progress: false + jobs: update: runs-on: ubuntu-latest + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@v6.0.2 - name: Setup Deno - uses: denoland/setup-deno@v1 + uses: denoland/setup-deno@v2.0.4 with: - deno-version: v1.x + deno-version: v2.x + cache: true - name: Run fetcher script - run: deno run --allow-net --allow-read --allow-write fetcher.ts + run: deno task update - name: Format README.md run: deno fmt README.md @@ -32,12 +41,18 @@ jobs: git config --local user.email "action@github.com" git config --local user.name "GitHub Action" git add . - git commit -m ":card_file_box: Update APIs: $(date +'%Y-%m-%d')" || exit 0 - git push - - - name: Publish to GitHub Pages - uses: peaceiris/actions-gh-pages@v4 + if git diff --cached --quiet; then + echo "No API changes to commit" + else + git commit -m ":card_file_box: Update APIs: $(date +'%Y-%m-%d')" + git push + fi + + - name: Upload Pages artifact + uses: actions/upload-pages-artifact@v5.0.0 with: - github_token: ${{ secrets.GITHUB_TOKEN }} - publish_dir: ./ - force_orphan: true + path: ./ + + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@v5.0.0 diff --git a/deno.json b/deno.json new file mode 100644 index 000000000..4fe058a5c --- /dev/null +++ b/deno.json @@ -0,0 +1,10 @@ +{ + "tasks": { + "update": "deno run --allow-net --allow-read --allow-write fetcher.ts", + "check": "deno fmt --check fetcher.ts .github && deno lint fetcher.ts && deno check fetcher.ts" + }, + "imports": { + "@std/fs/ensure-dir": "jsr:@std/fs@1.0.23/ensure-dir", + "slugify": "https://deno.land/x/slugify@0.3.0/mod.ts" + } +} diff --git a/deno.lock b/deno.lock new file mode 100644 index 000000000..bac3e266e --- /dev/null +++ b/deno.lock @@ -0,0 +1,19 @@ +{ + "version": "5", + "specifiers": { + "jsr:@std/fs@1.0.23": "1.0.23" + }, + "jsr": { + "@std/fs@1.0.23": { + "integrity": "3ecbae4ce4fee03b180fa710caff36bb5adb66631c46a6460aaad49515565a37" + } + }, + "remote": { + "https://deno.land/x/slugify@0.3.0/mod.ts": "a862dd4af7aac637a44bb3a3507acaca4b52386f8bebf4483b5a43e9458af315" + }, + "workspace": { + "dependencies": [ + "jsr:@std/fs@1.0.23" + ] + } +} diff --git a/fetcher.ts b/fetcher.ts index 600ca8ac3..1eca00b39 100644 --- a/fetcher.ts +++ b/fetcher.ts @@ -1,5 +1,5 @@ -import { ensureDir } from "https://deno.land/std@0.224.0/fs/ensure_dir.ts"; -import { slugify } from "https://deno.land/x/slugify/mod.ts"; +import { ensureDir } from "@std/fs/ensure-dir"; +import { slugify } from "slugify"; const indexName = "YCCompany_By_Launch_Date_production"; const applicationId = "45BWZJ1SGC"; @@ -57,9 +57,15 @@ interface LaunchedCompany { question_answers: boolean; } +type CompanyResult = LaunchedCompany & { + url: string; + api: string; +}; + const fetchAllCompanies = async (): Promise => { const algoliaApiKey = await getAlgoliaApiKey(); - const baseUrl = `https://${applicationId.toLowerCase()}-dsn.algolia.net/1/indexes/*/queries`; + const baseUrl = + `https://${applicationId.toLowerCase()}-dsn.algolia.net/1/indexes/*/queries`; const params = new URLSearchParams({ "x-algolia-agent": "Algolia for JavaScript (3.35.1); Browser; JS Helper (3.16.1)", @@ -74,7 +80,8 @@ const fetchAllCompanies = async (): Promise => { requests: [ { indexName, - params: `facets=%5B%22app_answers%22%2C%22app_video_public%22%2C%22batch%22%2C%22demo_day_video_public%22%2C%22highlight_black%22%2C%22highlight_latinx%22%2C%22highlight_women%22%2C%22industries%22%2C%22isHiring%22%2C%22nonprofit%22%2C%22question_answers%22%2C%22regions%22%2C%22subindustry%22%2C%22tags%22%2C%22top_company%22%5D&hitsPerPage=1000&maxValuesPerFacet=1000&query=&tagFilters=`, + params: + `facets=%5B%22app_answers%22%2C%22app_video_public%22%2C%22batch%22%2C%22demo_day_video_public%22%2C%22highlight_black%22%2C%22highlight_latinx%22%2C%22highlight_women%22%2C%22industries%22%2C%22isHiring%22%2C%22nonprofit%22%2C%22question_answers%22%2C%22regions%22%2C%22subindustry%22%2C%22tags%22%2C%22top_company%22%5D&hitsPerPage=1000&maxValuesPerFacet=1000&query=&tagFilters=`, }, ], }), @@ -108,9 +115,12 @@ const fetchAllCompanies = async (): Promise => { requests: [ { indexName, - params: `facets=%5B%22app_answers%22%2C%22app_video_public%22%2C%22batch%22%2C%22demo_day_video_public%22%2C%22highlight_black%22%2C%22highlight_latinx%22%2C%22highlight_women%22%2C%22industries%22%2C%22isHiring%22%2C%22nonprofit%22%2C%22question_answers%22%2C%22regions%22%2C%22subindustry%22%2C%22tags%22%2C%22top_company%22%5D&hitsPerPage=1000&maxValuesPerFacet=1000&query=&tagFilters=&facetFilters=batch:${encodeURIComponent( - batch - )}&page=${page}`, + params: + `facets=%5B%22app_answers%22%2C%22app_video_public%22%2C%22batch%22%2C%22demo_day_video_public%22%2C%22highlight_black%22%2C%22highlight_latinx%22%2C%22highlight_women%22%2C%22industries%22%2C%22isHiring%22%2C%22nonprofit%22%2C%22question_answers%22%2C%22regions%22%2C%22subindustry%22%2C%22tags%22%2C%22top_company%22%5D&hitsPerPage=1000&maxValuesPerFacet=1000&query=&tagFilters=&facetFilters=batch:${ + encodeURIComponent( + batch, + ) + }&page=${page}`, }, ], }), @@ -118,14 +128,18 @@ const fetchAllCompanies = async (): Promise => { if (!batchRes.ok) { const body = await batchRes.text(); throw new Error( - `Algolia batch request failed for ${batch}: ${batchRes.status} ${body}` + `Algolia batch request failed for ${batch}: ${batchRes.status} ${body}`, ); } const batchJson = await batchRes.json(); - const batchHits = batchJson.results?.[0]?.hits as LaunchedCompany[] | undefined; + const batchHits = batchJson.results?.[0]?.hits as + | LaunchedCompany[] + | undefined; if (!batchHits) { - throw new Error(`Algolia batch response did not include hits for ${batch}`); + throw new Error( + `Algolia batch response did not include hits for ${batch}`, + ); } allCompanies = allCompanies.concat(batchHits); fetchedCount += batchHits.length; @@ -141,7 +155,7 @@ await ensureDir("industries"); await ensureDir("batches"); const companies = await fetchAllCompanies(); -const results = companies.map((hit) => { +const results: CompanyResult[] = companies.map((hit) => { if (typeof hit !== "object") throw new Error("Result is not of type object"); if (hit === null) throw new Error("Object is of type null"); if ("_highlightResult" in hit) delete hit._highlightResult; @@ -149,12 +163,14 @@ const results = companies.map((hit) => { return { ...hit, url: `https://www.ycombinator.com/companies/${hit.slug}`, - api: `https://yc-oss.github.io/api/batches/${slugify( - hit.batch ?? "Unspecified", - { - lower: true, - } - )}/${hit.slug}.json`, + api: `https://yc-oss.github.io/api/batches/${ + slugify( + hit.batch ?? "Unspecified", + { + lower: true, + }, + ) + }/${hit.slug}.json`, }; }); @@ -176,7 +192,7 @@ const meta: Record< await Deno.writeTextFile( "companies/all.json", - JSON.stringify(results, null, 2) + "\n" + JSON.stringify(results, null, 2) + "\n", ); const uniqueTags = Array.from(new Set(results.flatMap((result) => result.tags))) @@ -196,7 +212,7 @@ for (const tag of uniqueTags) { ); await Deno.writeTextFile( `tags/${tag.slug}.json`, - JSON.stringify(filteredResults, null, 2) + "\n" + JSON.stringify(filteredResults, null, 2) + "\n", ); meta.tags[tag.slug] = { name: tag.name, @@ -206,7 +222,7 @@ for (const tag of uniqueTags) { } const uniqueIndustries = Array.from( - new Set(results.flatMap((result) => result.industries)) + new Set(results.flatMap((result) => result.industries)), ) .map((industry) => { const slug = slugify(industry, { lower: true }); @@ -224,7 +240,7 @@ for (const industry of uniqueIndustries) { ); await Deno.writeTextFile( `industries/${industry.slug}.json`, - JSON.stringify(filteredResults, null, 2) + "\n" + JSON.stringify(filteredResults, null, 2) + "\n", ); meta.industries[industry.slug] = { name: industry.name, @@ -268,11 +284,11 @@ const uniqueBatches = Array.from(new Set(results.map((result) => result.batch))) for (const batch of uniqueBatches) { const filteredResults = results.filter( - (result) => result.batch === batch.name + (result) => result.batch === batch.name, ); await Deno.writeTextFile( `batches/${batch.slug}.json`, - JSON.stringify(filteredResults, null, 2) + "\n" + JSON.stringify(filteredResults, null, 2) + "\n", ); meta.batches[batch.slug] = { name: batch.name, @@ -281,7 +297,7 @@ for (const batch of uniqueBatches) { }; } -for (const { key, slug, name } of [ +const companyBooleanFilters = [ { key: "top_company", slug: "top", name: "Top companies" }, { key: "highlight_black", @@ -300,11 +316,25 @@ for (const { key, slug, name } of [ }, { key: "nonprofit", slug: "nonprofit", name: "Not-for-profit companies" }, { key: "isHiring", slug: "hiring", name: "Companies currently hiring" }, -]) { +] as const satisfies ReadonlyArray<{ + key: keyof Pick< + CompanyResult, + | "top_company" + | "highlight_black" + | "highlight_latinx" + | "highlight_women" + | "nonprofit" + | "isHiring" + >; + slug: string; + name: string; +}>; + +for (const { key, slug, name } of companyBooleanFilters) { const filteredResults = results.filter((result) => result[key]); await Deno.writeTextFile( `companies/${slug}.json`, - JSON.stringify(filteredResults, null, 2) + "\n" + JSON.stringify(filteredResults, null, 2) + "\n", ); meta.companies[slug] = { name, @@ -315,20 +345,27 @@ for (const { key, slug, name } of [ for (const company of results) { await ensureDir( - `batches/${slugify(company.batch ?? "Unspecified", { - lower: true, - })}` + `batches/${ + slugify(company.batch ?? "Unspecified", { + lower: true, + }) + }`, ); await Deno.writeTextFile( - `batches/${slugify(company.batch ?? "Unspecified", { - lower: true, - })}/${company.slug}.json`, - JSON.stringify(company, null, 2) + "\n" + `batches/${ + slugify(company.batch ?? "Unspecified", { + lower: true, + }) + }/${company.slug}.json`, + JSON.stringify(company, null, 2) + "\n", ); } -const existingMeta = JSON.parse(await Deno.readTextFile("meta.json")); -const newMeta = { +const existingMeta = JSON.parse(await Deno.readTextFile("meta.json")) as Record< + string, + unknown +>; +const newMeta: Record = { last_updated: new Date().toISOString(), readme: "https://github.com/yc-oss/api", ...meta, @@ -344,7 +381,7 @@ if (hasChanges) { console.log("Meta has changed, updating meta.json"); await Deno.writeTextFile( "meta.json", - JSON.stringify(newMeta, null, 2) + "\n" + JSON.stringify(newMeta, null, 2) + "\n", ); const readme = await Deno.readTextFile("README.md"); @@ -352,10 +389,12 @@ if (hasChanges) { text += `\n## â„šī¸ Metadata\n\n`; text += `API endpoint: https://yc-oss.github.io/api/meta.json\n\n`; - text += `- Last updated: ${new Date().toLocaleString("en-US", { - dateStyle: "long", - timeStyle: "short", - })}\n`; + text += `- Last updated: ${ + new Date().toLocaleString("en-US", { + dateStyle: "long", + timeStyle: "short", + }) + }\n`; text += `- Companies: ${results.length}\n`; text += `- Batches: ${uniqueBatches.length}\n`; text += `- Industries: ${uniqueIndustries.length}\n`; @@ -363,26 +402,38 @@ if (hasChanges) { text += `\n## đŸ’ģ APIs\n\n`; - text += `\n### đŸĸ Companies\n\n| List of companies | API endpoint |\n| --------------- | ------------ |\n`; + text += + `\n### đŸĸ Companies\n\n| List of companies | API endpoint |\n| --------------- | ------------ |\n`; for (const slug of Object.keys(meta.companies)) { - text += `| ${meta.companies[slug].name} | https://yc-oss.github.io/api/companies/${slug}.json |\n`; + text += `| ${ + meta.companies[slug].name + } | https://yc-oss.github.io/api/companies/${slug}.json |\n`; } - text += `\n### 🎓 Batches\n\n
\nCompanies per batch\n\n| Batch | Count | API endpoint |\n| ---- | ---- | ------------ |\n`; + text += + `\n### 🎓 Batches\n\n
\nCompanies per batch\n\n| Batch | Count | API endpoint |\n| ---- | ---- | ------------ |\n`; for (const slug of Object.keys(meta.batches)) { - text += `| ${meta.batches[slug].name} | ${meta.batches[slug].count} | https://yc-oss.github.io/api/batches/${slug}.json |\n`; + text += `| ${meta.batches[slug].name} | ${ + meta.batches[slug].count + } | https://yc-oss.github.io/api/batches/${slug}.json |\n`; } text += `
\n`; - text += `\n### 🏭 Industries\n\n
\nCompanies per industry\n\n| Industry | Count | API endpoint |\n| -------- | ---- | ------------ |\n`; + text += + `\n### 🏭 Industries\n\n
\nCompanies per industry\n\n| Industry | Count | API endpoint |\n| -------- | ---- | ------------ |\n`; for (const slug of Object.keys(meta.industries)) { - text += `| ${meta.industries[slug].name} | ${meta.industries[slug].count} | https://yc-oss.github.io/api/industries/${slug}.json |\n`; + text += `| ${meta.industries[slug].name} | ${ + meta.industries[slug].count + } | https://yc-oss.github.io/api/industries/${slug}.json |\n`; } text += `
\n`; - text += `\n### đŸˇī¸ Tags\n\n
\nCompanies per tag\n\n| Tag | Count | API endpoint |\n| --- | ---- | ------------ |\n`; + text += + `\n### đŸˇī¸ Tags\n\n
\nCompanies per tag\n\n| Tag | Count | API endpoint |\n| --- | ---- | ------------ |\n`; for (const slug of Object.keys(meta.tags)) { - text += `| ${meta.tags[slug].name} | ${meta.tags[slug].count} | https://yc-oss.github.io/api/tags/${slug}.json |\n`; + text += `| ${meta.tags[slug].name} | ${ + meta.tags[slug].count + } | https://yc-oss.github.io/api/tags/${slug}.json |\n`; } text += `
\n`; @@ -390,7 +441,7 @@ if (hasChanges) { const newReadme = readme.replace( /[\s\S]*/g, - text + text, ); await Deno.writeTextFile("README.md", newReadme); } else console.log("Meta has not changed");