Skip to content

Commit c8f221d

Browse files
edujuanh3xxitcubic-dev-ai[bot]
authored
Hall of fame lines of code (#40)
* made lines of code instead * Update scripts/fetch-contributors.js Co-authored-by: cubic-dev-ai[bot] <191113872+cubic-dev-ai[bot]@users.noreply.github.com> * Fix 0 id --------- Co-authored-by: Razvan Radulescu <43811028+h3xxit@users.noreply.github.com> Co-authored-by: cubic-dev-ai[bot] <191113872+cubic-dev-ai[bot]@users.noreply.github.com>
1 parent 7c7cb8f commit c8f221d

File tree

5 files changed

+899
-168
lines changed

5 files changed

+899
-168
lines changed

‎scripts/fetch-contributors.js‎

Lines changed: 193 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,15 @@ const GITHUB_TOKEN = process.env.GITHUB_TOKEN;
77
const ORG_NAME = 'universal-tool-calling-protocol';
88
const OUTPUT_FILE = path.join(__dirname, '../src/data/contributors.json');
99

10+
// Configuration for efficient API batching
11+
const BATCH_CONFIG = {
12+
COMMIT_DETAILS_BATCH_SIZE: 20, // Process commits in batches of 20
13+
CONCURRENT_REQUESTS: 5, // Max concurrent API requests
14+
RETRY_ATTEMPTS: 3, // Number of retry attempts for failed requests
15+
RATE_LIMIT_DELAY: 100, // Base delay between requests (ms)
16+
COMMIT_ANALYSIS_LIMIT: 1000, // Max commits to analyze (0 = no limit)
17+
};
18+
1019
const githubApi = async (endpoint) => {
1120
const headers = {
1221
'Accept': 'application/vnd.github.v3+json',
@@ -50,6 +59,82 @@ const fetchContributorsForRepo = async (repoName) => {
5059
}
5160
};
5261

62+
const fetchCommitDetails = async (repoName, commitSha) => {
63+
try {
64+
const commit = await githubApi(`/repos/${ORG_NAME}/${repoName}/commits/${commitSha}`);
65+
return {
66+
additions: commit.stats?.additions || 0,
67+
deletions: commit.stats?.deletions || 0,
68+
total: commit.stats?.total || 0
69+
};
70+
} catch (error) {
71+
console.warn(`Could not fetch commit details for ${commitSha}:`, error.message);
72+
return { additions: 0, deletions: 0, total: 0 };
73+
}
74+
};
75+
76+
// Helper function for batched API calls with retry logic
77+
const batchedApiCall = async (items, apiCallFn, batchSize = BATCH_CONFIG.COMMIT_DETAILS_BATCH_SIZE) => {
78+
const results = [];
79+
const batches = [];
80+
81+
// Split items into batches
82+
for (let i = 0; i < items.length; i += batchSize) {
83+
batches.push(items.slice(i, i + batchSize));
84+
}
85+
86+
console.log(` 📦 Processing ${items.length} items in ${batches.length} batches...`);
87+
88+
for (let batchIndex = 0; batchIndex < batches.length; batchIndex++) {
89+
const batch = batches[batchIndex];
90+
console.log(` ⚡ Processing batch ${batchIndex + 1}/${batches.length} (${batch.length} items)...`);
91+
92+
// Process items in current batch concurrently
93+
const batchPromises = batch.map(async (item) => {
94+
let retryCount = 0;
95+
while (retryCount < BATCH_CONFIG.RETRY_ATTEMPTS) {
96+
try {
97+
const result = await apiCallFn(item);
98+
return result;
99+
} catch (error) {
100+
retryCount++;
101+
if (retryCount === BATCH_CONFIG.RETRY_ATTEMPTS) {
102+
console.warn(` ⚠️ Failed after ${BATCH_CONFIG.RETRY_ATTEMPTS} attempts:`, error.message);
103+
return null;
104+
}
105+
// Exponential backoff for retries
106+
await new Promise(resolve => setTimeout(resolve, BATCH_CONFIG.RATE_LIMIT_DELAY * Math.pow(2, retryCount)));
107+
}
108+
}
109+
});
110+
111+
const batchResults = await Promise.all(batchPromises);
112+
results.push(...batchResults.filter(Boolean)); // Filter out null results from failures
113+
114+
// Rate limiting between batches
115+
if (batchIndex < batches.length - 1) {
116+
await new Promise(resolve => setTimeout(resolve, BATCH_CONFIG.RATE_LIMIT_DELAY));
117+
}
118+
}
119+
120+
return results;
121+
};
122+
123+
// Helper function to create progress reporting
124+
const createProgressReporter = (total, operation) => {
125+
let processed = 0;
126+
return {
127+
update: () => {
128+
processed++;
129+
const percentage = Math.round((processed / total) * 100);
130+
console.log(` 📊 ${operation}: ${processed}/${total} (${percentage}%)`);
131+
},
132+
finish: () => {
133+
console.log(` ✅ ${operation}: Completed all ${total} items`);
134+
}
135+
};
136+
};
137+
53138
const fetchUserPRsAndActivity = async (username, repoName) => {
54139
try {
55140
// Get PRs for this user in this repo
@@ -60,6 +145,61 @@ const fetchUserPRsAndActivity = async (username, repoName) => {
60145
sixMonthsAgo.setMonth(sixMonthsAgo.getMonth() - 6);
61146
const commits = await githubApi(`/repos/${ORG_NAME}/${repoName}/commits?author=${username}&since=${sixMonthsAgo.toISOString()}&per_page=100`);
62147

148+
// Get ALL commits by this user for line change statistics
149+
let allCommits = [];
150+
let page = 1;
151+
let hasMore = true;
152+
153+
console.log(` 📊 Fetching all commits for line statistics...`);
154+
while (hasMore && allCommits.length < (BATCH_CONFIG.COMMIT_ANALYSIS_LIMIT || Infinity)) { // Limit to prevent excessive API calls
155+
try {
156+
const commitPage = await githubApi(`/repos/${ORG_NAME}/${repoName}/commits?author=${username}&per_page=100&page=${page}`);
157+
158+
if (commitPage.length === 0) {
159+
hasMore = false;
160+
} else {
161+
allCommits = allCommits.concat(commitPage);
162+
page++;
163+
164+
// Rate limiting for commit fetching
165+
await new Promise(resolve => setTimeout(resolve, 100));
166+
}
167+
} catch (error) {
168+
console.warn(` Could not fetch commits page ${page} for ${username}:`, error.message);
169+
hasMore = false;
170+
}
171+
}
172+
173+
// Analyze ALL commits with efficient batching (improved from 100 commit limit)
174+
let totalAdditions = 0;
175+
let totalDeletions = 0;
176+
let totalChanges = 0;
177+
178+
// Apply configurable limit if set (0 = no limit)
179+
const commitsToAnalyze = BATCH_CONFIG.COMMIT_ANALYSIS_LIMIT > 0
180+
? allCommits.slice(0, BATCH_CONFIG.COMMIT_ANALYSIS_LIMIT)
181+
: allCommits;
182+
183+
console.log(` 📈 Analyzing ${commitsToAnalyze.length} commits for line changes (improved batching)...`);
184+
185+
if (commitsToAnalyze.length > 0) {
186+
// Use efficient batching instead of sequential processing
187+
const commitDetails = await batchedApiCall(
188+
commitsToAnalyze,
189+
async (commit) => await fetchCommitDetails(repoName, commit.sha),
190+
BATCH_CONFIG.COMMIT_DETAILS_BATCH_SIZE
191+
);
192+
193+
// Aggregate the results
194+
for (const details of commitDetails) {
195+
totalAdditions += details.additions;
196+
totalDeletions += details.deletions;
197+
totalChanges += details.total;
198+
}
199+
200+
console.log(` ✅ Successfully analyzed ${commitDetails.length}/${commitsToAnalyze.length} commits`);
201+
}
202+
63203
// Get PR reviews by this user
64204
const reviews = await githubApi(`/repos/${ORG_NAME}/${repoName}/pulls/comments?per_page=100`);
65205
const userReviews = reviews.filter(review => review.user.login === username);
@@ -68,17 +208,30 @@ const fetchUserPRsAndActivity = async (username, repoName) => {
68208
prs: prs.length,
69209
mergedPrs: prs.filter(pr => pr.merged_at).length,
70210
recentCommits: commits.length,
211+
totalCommits: allCommits.length,
71212
reviews: userReviews.length,
72-
lastActivity: commits.length > 0 ? commits[0].commit.author.date : null
213+
lastActivity: commits.length > 0 ? commits[0].commit.author.date : null,
214+
// Enhanced line change statistics (now analyzes ALL commits up to limit)
215+
totalAdditions,
216+
totalDeletions,
217+
totalChanges,
218+
commitsAnalyzed: commitsToAnalyze.length, // Total commits we attempted to analyze
219+
commitsSuccessfullyAnalyzed: totalChanges > 0 ? commitsToAnalyze.filter(commit => commit).length : 0 // Successful analyses
73220
};
74221
} catch (error) {
75222
console.warn(`Could not fetch detailed activity for ${username} in ${repoName}:`, error.message);
76223
return {
77224
prs: 0,
78225
mergedPrs: 0,
79226
recentCommits: 0,
227+
totalCommits: 0,
80228
reviews: 0,
81-
lastActivity: null
229+
lastActivity: null,
230+
totalAdditions: 0,
231+
totalDeletions: 0,
232+
totalChanges: 0,
233+
commitsAnalyzed: 0,
234+
commitsSuccessfullyAnalyzed: 0
82235
};
83236
}
84237
};
@@ -107,7 +260,13 @@ const aggregateContributors = (contributorsByRepo) => {
107260
existing.totalPrs += contributor.activityData.prs;
108261
existing.totalMergedPrs += contributor.activityData.mergedPrs;
109262
existing.totalRecentCommits += contributor.activityData.recentCommits;
263+
existing.totalCommits += contributor.activityData.totalCommits;
110264
existing.totalReviews += contributor.activityData.reviews;
265+
// Aggregate line change statistics
266+
existing.totalAdditions += contributor.activityData.totalAdditions;
267+
existing.totalDeletions += contributor.activityData.totalDeletions;
268+
existing.totalChanges += contributor.activityData.totalChanges;
269+
existing.commitsAnalyzed += contributor.activityData.commitsAnalyzed;
111270
// Keep the most recent activity date
112271
if (contributor.activityData.lastActivity) {
113272
const activityDate = new Date(contributor.activityData.lastActivity);
@@ -127,8 +286,14 @@ const aggregateContributors = (contributorsByRepo) => {
127286
totalPrs: contributor.activityData?.prs || 0,
128287
totalMergedPrs: contributor.activityData?.mergedPrs || 0,
129288
totalRecentCommits: contributor.activityData?.recentCommits || 0,
289+
totalCommits: contributor.activityData?.totalCommits || 0,
130290
totalReviews: contributor.activityData?.reviews || 0,
131-
lastActivity: contributor.activityData?.lastActivity || null
291+
lastActivity: contributor.activityData?.lastActivity || null,
292+
// New line change statistics
293+
totalAdditions: contributor.activityData?.totalAdditions || 0,
294+
totalDeletions: contributor.activityData?.totalDeletions || 0,
295+
totalChanges: contributor.activityData?.totalChanges || 0,
296+
commitsAnalyzed: contributor.activityData?.commitsAnalyzed || 0
132297
});
133298
}
134299
});
@@ -194,10 +359,16 @@ const enhanceWithUserData = async (contributors) => {
194359
total_prs: contributor.totalPrs,
195360
total_merged_prs: contributor.totalMergedPrs,
196361
total_recent_commits: contributor.totalRecentCommits,
362+
total_commits: contributor.totalCommits,
197363
total_reviews: contributor.totalReviews,
198364
last_activity: contributor.lastActivity,
199365
pr_success_rate: contributor.totalPrs > 0 ?
200-
Math.round((contributor.totalMergedPrs / contributor.totalPrs) * 100) : 0
366+
Math.round((contributor.totalMergedPrs / contributor.totalPrs) * 100) : 0,
367+
// New line change statistics
368+
total_additions: contributor.totalAdditions,
369+
total_deletions: contributor.totalDeletions,
370+
total_changes: contributor.totalChanges,
371+
commits_analyzed: contributor.commitsAnalyzed
201372
});
202373

203374
// Rate limiting: delay between requests
@@ -269,15 +440,19 @@ const calculateAllScores = async (contributorsByRepo, repositories) => {
269440

270441
const main = async () => {
271442
try {
272-
console.log('🚀 Starting contributor data fetch with simplified scoring...');
443+
console.log('🚀 Starting contributor data fetch with enhanced batching and full commit analysis...');
444+
console.log(`📊 Configuration: Analyzing up to ${BATCH_CONFIG.COMMIT_ANALYSIS_LIMIT} commits per contributor (0 = no limit)`);
445+
console.log(`⚡ Batching: ${BATCH_CONFIG.COMMIT_DETAILS_BATCH_SIZE} commits per batch with ${BATCH_CONFIG.RETRY_ATTEMPTS} retry attempts`);
446+
console.log('✨ Improvements: Removed 100-commit limit, added efficient batching, retry logic, and better error handling\n');
273447

274448
/*
275449
* Process Flow:
276450
* 1. Fetch all repositories from the organization
277451
* 2. Fetch all contributors from all repositories
278452
* 3. Calculate scores for all contributors (simplified recent activity scoring)
279-
* 4. Enhance with detailed user information from GitHub
280-
* 5. Generate and save output file
453+
* 4. Fetch detailed line change statistics for each contributor
454+
* 5. Enhance with detailed user information from GitHub
455+
* 6. Generate and save output file with comprehensive metrics
281456
*/
282457

283458
// Step 1: Fetch all repositories
@@ -308,6 +483,10 @@ const main = async () => {
308483
const totalImpactScore = enhancedContributors.reduce((sum, c) => sum + c.impact_score, 0);
309484
const totalContributions = enhancedContributors.reduce((sum, c) => sum + c.contributions, 0);
310485
const totalRecentActivity = enhancedContributors.reduce((sum, c) => sum + c.total_recent_commits, 0);
486+
const totalAdditions = enhancedContributors.reduce((sum, c) => sum + c.total_additions, 0);
487+
const totalDeletions = enhancedContributors.reduce((sum, c) => sum + c.total_deletions, 0);
488+
const totalChanges = enhancedContributors.reduce((sum, c) => sum + c.total_changes, 0);
489+
const totalCommitsAnalyzed = enhancedContributors.reduce((sum, c) => sum + c.commits_analyzed, 0);
311490

312491
// Write to file
313492
fs.writeFileSync(OUTPUT_FILE, JSON.stringify({
@@ -317,6 +496,11 @@ const main = async () => {
317496
total_impact_score: totalImpactScore,
318497
total_recent_activity: totalRecentActivity,
319498
scoring_method: 'simplified_recent_activity',
499+
// New aggregated line change statistics
500+
total_additions: totalAdditions,
501+
total_deletions: totalDeletions,
502+
total_changes: totalChanges,
503+
total_commits_analyzed: totalCommitsAnalyzed,
320504
contributors: enhancedContributors
321505
}, null, 2));
322506

@@ -325,6 +509,8 @@ const main = async () => {
325509
console.log(` 💫 Total impact score: ${totalImpactScore}`);
326510
console.log(` 📈 Total contributions: ${totalContributions}`);
327511
console.log(` 🔥 Recent activity: ${totalRecentActivity} commits (last 6 months)`);
512+
console.log(` 📊 Line changes: +${totalAdditions.toLocaleString()} -${totalDeletions.toLocaleString()} (${totalChanges.toLocaleString()} total)`);
513+
console.log(` 🔍 Commits analyzed: ${totalCommitsAnalyzed.toLocaleString()}`);
328514
console.log(` 🏆 Top contributor: ${enhancedContributors[0]?.name} (${enhancedContributors[0]?.impact_score} impact score)`);
329515

330516
} catch (error) {

0 commit comments

Comments
 (0)