From dc76f78e984748959df40c96b1d1d03ce2d680d9 Mon Sep 17 00:00:00 2001 From: guenhter Date: Sun, 19 Jul 2026 07:39:05 +0200 Subject: [PATCH 1/6] pack_statistics: add run() and unit test; enable serde_json preserve_order --- Cargo.lock | 2 ++ Cargo.toml | 5 +++- src/bin/pack_statistics.rs | 54 ++++++++++++++++++++++++++++++++++++-- 3 files changed, 58 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9b9fa52..2510381 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -408,6 +408,7 @@ dependencies = [ "reqwest", "serde", "serde_json", + "tempfile", "tokio", ] @@ -1507,6 +1508,7 @@ version = "1.0.150" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" dependencies = [ + "indexmap", "itoa", "memchr", "serde", diff --git a/Cargo.toml b/Cargo.toml index 8a91e88..bc03fe9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,7 +9,7 @@ reqwest = { version = "0.13", features = ["json", "native-tls-vendored"] } bytes = "1" chrono = { version = "0.4", features = ["clock"] } serde = { version = "1", features = ["derive"] } -serde_json = "1" +serde_json = { version = "1", features = ["preserve_order"] } clap = { version = "4", features = ["derive", "env"] } flate2 = "1" anyhow = "1" @@ -23,3 +23,6 @@ tokio = { version = "1", features = [ "io-util", "io-std", ] } + +[dev-dependencies] +tempfile = "3" diff --git a/src/bin/pack_statistics.rs b/src/bin/pack_statistics.rs index c6b0821..ae8560e 100644 --- a/src/bin/pack_statistics.rs +++ b/src/bin/pack_statistics.rs @@ -57,8 +57,10 @@ struct Args { // ── Main ───────────────────────────────────────────────────────────────────── fn main() -> Result<()> { - let args = Args::parse(); + run(Args::parse()) +} +fn run(args: Args) -> Result<()> { validate_type(&args.r#type)?; // Collect all matching input files, sorted chronologically. @@ -152,7 +154,7 @@ fn collect_input_files(dir: &PathBuf, stat_type: &str) -> Result> { /// Read one monthly ratings file and write every record to `w`, injecting /// `"month": month` as the first field. Returns the number of records written. -fn pack_file(path: &PathBuf, month: &str, w: &mut BufWriter) -> Result { +fn pack_file(path: &PathBuf, month: &str, w: &mut impl Write) -> Result { let reader = BufReader::new(File::open(path).with_context(|| format!("cannot open {:?}", path))?); let mut count = 0; @@ -203,3 +205,51 @@ fn month_from_path(path: &Path) -> String { } name } + +// ── Tests ───────────────────────────────────────────────────────────────────── + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_pack_two_months() -> Result<()> { + let tmp = tempfile::tempdir()?; + let dir = tmp.path(); + + // Write two dummy monthly input files. + std::fs::write( + dir.join("language-ratings-2024-01-pr-count.jsonl"), + r#"{"language":"Rust","rating":100.0,"percentage":50.0} +{"language":"Go","rating":80.0,"percentage":40.0} +"#, + )?; + std::fs::write( + dir.join("language-ratings-2024-02-pr-count.jsonl"), + r#"{"language":"TypeScript","rating":200.0,"percentage":60.0} +{"language":"Rust","rating":120.0,"percentage":36.0} +"#, + )?; + + run(Args { + r#type: "pr-count".to_string(), + input_dir: dir.to_path_buf(), + output_dir: dir.to_path_buf(), + })?; + + // Read the output file. + let out_path = dir.join("language-ratings-all-pr-count.jsonl"); + let content = std::fs::read_to_string(&out_path)?; + + assert_eq!( + content, + r#"{"month":"2024-01","language":"Rust","rating":100.0,"percentage":50.0} +{"month":"2024-01","language":"Go","rating":80.0,"percentage":40.0} +{"month":"2024-02","language":"TypeScript","rating":200.0,"percentage":60.0} +{"month":"2024-02","language":"Rust","rating":120.0,"percentage":36.0} +"# + ); + + Ok(()) + } +} From a6b86de65f00a8966f9e72811a455569b7d5356a Mon Sep 17 00:00:00 2001 From: guenhter Date: Sun, 19 Jul 2026 07:41:43 +0200 Subject: [PATCH 2/6] docs: add git policy to AGENTS.md generic instructions --- AGENTS.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/AGENTS.md b/AGENTS.md index 21df70e..a995481 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -131,3 +131,7 @@ If port 8080 is already in use, try 8081, 8082, etc. + +### Git Policy + +Never run `git commit`, `git push`, or create pull requests unless the user explicitly requests it. Always leave changes staged or unstaged for the user to review before committing. From a2639b2378a97e154c310b3097e15465fa375f8e Mon Sep 17 00:00:00 2001 From: guenhter Date: Sun, 19 Jul 2026 07:46:24 +0200 Subject: [PATCH 3/6] filter_archive, produce_statistics: add run() and unit tests --- src/bin/filter_archive.rs | 207 +++++++++++++++++++++++++++++++++- src/bin/produce_statistics.rs | 112 +++++++++++++++++- 2 files changed, 316 insertions(+), 3 deletions(-) diff --git a/src/bin/filter_archive.rs b/src/bin/filter_archive.rs index e99f5b7..8e72d27 100644 --- a/src/bin/filter_archive.rs +++ b/src/bin/filter_archive.rs @@ -63,8 +63,10 @@ struct Row { // ── Entry point ─────────────────────────────────────────────────────────────── fn main() -> Result<()> { - let args = Args::parse(); + run(Args::parse()) +} +fn run(args: Args) -> Result<()> { let rows = read_csv(&args.input)?; let total = rows.len(); eprintln!(" [read] {:>8} rows", total); @@ -532,3 +534,206 @@ fn split_csv_line(line: &str) -> Vec { fields.push(field); fields } + +// ── Tests ───────────────────────────────────────────────────────────────────── + +#[cfg(test)] +mod tests { + use super::*; + + // ── helpers ─────────────────────────────────────────────────────────────── + + fn row(actor: &str, repo: &str, event_type: &str, count: u64) -> Row { + Row { + actor: actor.to_string(), + repo: repo.to_string(), + event_type: event_type.to_string(), + action: String::new(), + language: String::new(), + count, + } + } + + fn actors(rows: &[Row]) -> Vec<&str> { + rows.iter().map(|r| r.actor.as_str()).collect() + } + + fn repos(rows: &[Row]) -> Vec<&str> { + rows.iter().map(|r| r.repo.as_str()).collect() + } + + // ── filter_bots ─────────────────────────────────────────────────────────── + + #[test] + fn test_filter_bots() { + let rows = vec![ + row("alice", "a/a", "PushEvent", 1), + row("dependabot[bot]", "a/a", "PullRequestEvent", 1), + row("MyProjectBot", "a/a", "PushEvent", 1), + row("BOT-ci", "a/a", "PushEvent", 1), + ]; + let result = filter_bots(rows); + assert_eq!(actors(&result), vec!["alice"]); + } + + // ── filter_ci_actors ────────────────────────────────────────────────────── + + #[test] + fn test_filter_ci_actors() { + let rows = vec![ + row("alice", "a/a", "PushEvent", 1), + row("github-actions[bot]", "a/a", "PushEvent", 1), + row("Renovate-App", "a/a", "PullRequestEvent", 1), // case-insensitive + row("snyk-io", "a/a", "PushEvent", 1), + row("netlify[bot]", "a/a", "PushEvent", 1), + ]; + let result = filter_ci_actors(rows); + assert_eq!(actors(&result), vec!["alice"]); + } + + // ── filter_single_event_repos ───────────────────────────────────────────── + + #[test] + fn test_filter_single_event_repos() { + let rows = vec![ + row("alice", "active/repo", "PushEvent", 3), + row("bob", "active/repo", "PushEvent", 1), // total=4, kept + row("carol", "quiet/repo", "PushEvent", 1), // total=1, dropped + ]; + let result = filter_single_event_repos(rows); + assert_eq!(repos(&result), vec!["active/repo", "active/repo"]); + } + + // ── filter_empty_repos ──────────────────────────────────────────────────── + + #[test] + fn test_filter_empty_repos() { + let rows = vec![ + row("alice", "code/repo", "PushEvent", 2), + row("alice", "code/repo", "IssuesEvent", 1), // kept: repo has pushes + row("bob", "watch/repo", "WatchEvent", 5), // dropped: no push/PR + row("bob", "watch/repo", "IssuesEvent", 1), // dropped: same repo + ]; + let result = filter_empty_repos(rows); + assert_eq!(repos(&result), vec!["code/repo", "code/repo"]); + } + + // ── filter_high_volume_actors ───────────────────────────────────────────── + + #[test] + fn test_filter_high_volume_actors() { + let rows = vec![ + row("alice", "a/a", "PushEvent", 5), + row("alice", "a/b", "PushEvent", 5), // alice total=10, kept (limit=10) + row("bob", "b/a", "PushEvent", 11), // bob total=11, dropped + ]; + let result = filter_high_volume_actors(rows, 10); + assert_eq!(actors(&result), vec!["alice", "alice"]); + } + + // ── filter_deleted_repos ────────────────────────────────────────────────── + + #[test] + fn test_filter_deleted_repos() { + let sha = "a".repeat(40); + let uuid = "550e8400-e29b-41d4-a716-446655440000"; + let rows = vec![ + row("alice", "normal/repo", "PushEvent", 1), + row("alice", &format!("{sha}/repo"), "PushEvent", 1), + row("alice", &format!("owner/{sha}"), "PushEvent", 1), + row("alice", &format!("{uuid}/repo"), "PushEvent", 1), + row("alice", &format!("owner/{uuid}"), "PushEvent", 1), + ]; + let result = filter_deleted_repos(rows); + assert_eq!(repos(&result), vec!["normal/repo"]); + } + + // ── filter_fork_only_actors ─────────────────────────────────────────────── + + #[test] + fn test_filter_fork_only_actors() { + let rows = vec![ + row("alice", "a/a", "PushEvent", 1), // alice has code activity → kept + row("alice", "a/a", "ForkEvent", 1), // kept: alice isn't fork-only + row("bob", "b/b", "ForkEvent", 3), // bob is fork-only → dropped + row("carol", "c/c", "WatchEvent", 2), // carol is watch-only → dropped + ]; + let result = filter_fork_only_actors(rows); + assert_eq!(actors(&result), vec!["alice", "alice"]); + } + + // ── filter_high_volume_issue_repos ──────────────────────────────────────── + + #[test] + fn test_filter_high_volume_issue_repos() { + let rows = vec![ + row("alice", "spam/repo", "IssuesEvent", 200), // total issues=200 > limit=100 + row("alice", "spam/repo", "PushEvent", 1), // push kept even for spam/repo + row("bob", "good/repo", "IssuesEvent", 50), // total issues=50, kept + ]; + let result = filter_high_volume_issue_repos(rows, 100); + assert_eq!( + result.iter().map(|r| (r.repo.as_str(), r.event_type.as_str())).collect::>(), + vec![("spam/repo", "PushEvent"), ("good/repo", "IssuesEvent")] + ); + } + + // ── filter_issue_only_actors ────────────────────────────────────────────── + + #[test] + fn test_filter_issue_only_actors() { + let rows = vec![ + row("alice", "a/a", "PushEvent", 1), // alice has code → issue row kept + row("alice", "a/a", "IssuesEvent", 2), + row("bob", "b/b", "IssuesEvent", 5), // bob has no code → issue row dropped + ]; + let result = filter_issue_only_actors(rows); + assert_eq!( + result.iter().map(|r| r.actor.as_str()).collect::>(), + vec!["alice", "alice"] + ); + } + + // ── end-to-end via run() ────────────────────────────────────────────────── + + #[test] + fn test_filter_archive_end_to_end() -> Result<()> { + let tmp = tempfile::tempdir()?; + let dir = tmp.path(); + + std::fs::write( + dir.join("archive-202401.csv"), + r#"actor,repo,event_type,action,language,count +alice,rust-lang/rust,PushEvent,,,5 +alice,rust-lang/rust,PullRequestEvent,opened,,3 +alice,rust-lang/rust,IssuesEvent,opened,,2 +bob,rust-lang/rust,PushEvent,,,2 +dependabot[bot],some/repo,PullRequestEvent,opened,,1 +carol,single-event/repo,PushEvent,,,1 +dave,golang/go,WatchEvent,,,10 +eve,spam/repo,IssuesEvent,opened,,20000 +eve,spam/repo,PushEvent,,,1 +"#, + )?; + + run(Args { + input: dir.join("archive-202401.csv"), + output: dir.join("archive-202401-filtered.csv"), + actor_event_limit: 1_000, + repo_issue_limit: 10_000, + })?; + + assert_eq!( + std::fs::read_to_string(dir.join("archive-202401-filtered.csv"))?, + r#"actor,repo,event_type,action,language,count +alice,rust-lang/rust,PushEvent,,,5 +alice,rust-lang/rust,PullRequestEvent,opened,,3 +alice,rust-lang/rust,IssuesEvent,opened,,2 +bob,rust-lang/rust,PushEvent,,,2 +"# + ); + + Ok(()) + } +} + diff --git a/src/bin/produce_statistics.rs b/src/bin/produce_statistics.rs index ee40849..fb6a769 100644 --- a/src/bin/produce_statistics.rs +++ b/src/bin/produce_statistics.rs @@ -139,8 +139,10 @@ struct RepoCounts { // ── Main ───────────────────────────────────────────────────────────────────── fn main() -> Result<()> { - let args = Args::parse(); + run(Args::parse()) +} +fn run(args: Args) -> Result<()> { // Infer YYYY-MM from the archive filename. let year_month = infer_year_month(&args.archive)?; eprintln!("Inferred period: {year_month}"); @@ -316,7 +318,7 @@ fn open_writer(path: &PathBuf) -> Result> { /// Write a sorted-descending list of (language, rating) pairs as JSONL. /// Each record includes the rating and its percentage share of the total. -fn write_ratings(w: &mut BufWriter, ratings: &[(String, f64)]) -> Result<()> { +fn write_ratings(w: &mut impl Write, ratings: &[(String, f64)]) -> Result<()> { let total: f64 = ratings.iter().map(|(_, r)| r).sum(); for (language, rating) in ratings { let rating = (rating * 100.0).round() / 100.0; @@ -526,3 +528,109 @@ fn open(path: &PathBuf) -> Result> { .with_context(|| format!("cannot open {:?}", path)) .map(BufReader::new) } + +// ── Tests ───────────────────────────────────────────────────────────────────── + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_produce_statistics() -> Result<()> { + let tmp = tempfile::tempdir()?; + let dir = tmp.path(); + + // Two repos: rust-lang/rust (Rust-heavy) and golang/go (Go-only). + std::fs::write( + dir.join("archive-202401-filtered.csv"), + r#"actor,repo,event_type,action,language,count +alice,rust-lang/rust,PullRequestEvent,opened,,3 +bob,rust-lang/rust,PushEvent,,,5 +alice,rust-lang/rust,IssuesEvent,opened,,2 +carol,golang/go,PushEvent,,,4 +carol,golang/go,WatchEvent,,,10 +"#, + )?; + + // rust-lang/rust: 90% Rust, 10% C. golang/go: 100% Go. + std::fs::write( + dir.join("languages-2024-01.jsonl"), + r#"{"repo":"rust-lang/rust","total_size":1000,"languages":[{"language":"Rust","size":900},{"language":"C","size":100}]} +{"repo":"golang/go","total_size":500,"languages":[{"language":"Go","size":500}]} +"#, + )?; + + run(Args { + archive: dir.join("archive-202401-filtered.csv"), + languages: dir.join("languages-2024-01.jsonl"), + output_dir: dir.to_path_buf(), + primary_only: false, + })?; + + // ── pr-count ───────────────────────────────────────────────────────── + // Only rust-lang/rust had PRs (count=3). Rust gets 90%, C gets 10%. + assert_eq!( + std::fs::read_to_string(dir.join("language-ratings-2024-01-pr-count.jsonl"))?, + r#"{"language":"Rust","rating":2.7,"percentage":90.0} +{"language":"C","rating":0.3,"percentage":10.0} +"# + ); + + // ── push-count ─────────────────────────────────────────────────────── + // rust-lang/rust: 5 pushes → Rust 4.5, C 0.5. golang/go: 4 pushes → Go 4.0. + // Total = 9.0 → Rust 50.0%, Go 44.44%, C 5.56% + assert_eq!( + std::fs::read_to_string(dir.join("language-ratings-2024-01-push-count.jsonl"))?, + r#"{"language":"Rust","rating":4.5,"percentage":50.0} +{"language":"Go","rating":4.0,"percentage":44.44} +{"language":"C","rating":0.5,"percentage":5.56} +"# + ); + + // ── issue-count ────────────────────────────────────────────────────── + // Only rust-lang/rust had issues (count=2). Rust 90%, C 10%. + assert_eq!( + std::fs::read_to_string(dir.join("language-ratings-2024-01-issue-count.jsonl"))?, + r#"{"language":"Rust","rating":1.8,"percentage":90.0} +{"language":"C","rating":0.2,"percentage":10.0} +"# + ); + + // ── developer-activity ─────────────────────────────────────────────── + // rust-lang/rust: alice (PR) + bob (push) = 2 distinct devs → Rust 1.8, C 0.2. + // golang/go: carol (push) = 1 dev → Go 1.0. + // Total = 3.0 → Rust 60.0%, Go 33.33%, C 6.67% + assert_eq!( + std::fs::read_to_string( + dir.join("language-ratings-2024-01-developer-activity.jsonl") + )?, + r#"{"language":"Rust","rating":1.8,"percentage":60.0} +{"language":"Go","rating":1.0,"percentage":33.33} +{"language":"C","rating":0.2,"percentage":6.67} +"# + ); + + // ── active-repos ───────────────────────────────────────────────────── + // Both repos are active (each counts as 1). + // rust-lang/rust → Rust 0.9, C 0.1. golang/go → Go 1.0. + // Total = 2.0 → Go 50.0%, Rust 45.0%, C 5.0% + assert_eq!( + std::fs::read_to_string(dir.join("language-ratings-2024-01-active-repos.jsonl"))?, + r#"{"language":"Go","rating":1.0,"percentage":50.0} +{"language":"Rust","rating":0.9,"percentage":45.0} +{"language":"C","rating":0.1,"percentage":5.0} +"# + ); + + // ── star-count ─────────────────────────────────────────────────────── + // golang/go had 10 WatchEvents → Go 10.0. + assert_eq!( + std::fs::read_to_string(dir.join("language-ratings-2024-01-star-count.jsonl"))?, + r#"{"language":"Go","rating":10.0,"percentage":100.0} +"# + ); + + Ok(()) + } +} + From ebd3c39e061ae9a3ef132fc40d39976a24fac3cc Mon Sep 17 00:00:00 2001 From: guenhter Date: Sun, 19 Jul 2026 07:47:37 +0200 Subject: [PATCH 4/6] ci: add test workflow --- .github/workflows/test.yml | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 .github/workflows/test.yml diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..70d9fc9 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,29 @@ +name: Test + +on: + push: + branches: ["**"] + +jobs: + test: + name: cargo test + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@stable + + - name: Cache cargo registry & build artifacts + uses: actions/cache@v4 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + target + key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} + restore-keys: ${{ runner.os }}-cargo- + + - name: Run tests + run: cargo test From ea9a718b3371c09365c4913807127475e7d64497 Mon Sep 17 00:00:00 2001 From: guenhter Date: Sun, 19 Jul 2026 08:01:50 +0200 Subject: [PATCH 5/6] refactor: replace hand-rolled CSV I/O with the csv crate - Add csv = "1" to Cargo.toml - github_archive_loader: replace manual writeln!/csv_field() with csv::WriterBuilder - filter_archive: replace split_csv_line() + csv_field() with csv::Reader + csv::WriterBuilder - produce_statistics: replace splitn/trim_matches with csv::Reader Deletes ~80 lines of hand-rolled RFC 4180 logic (quoting helper, character-level parser state machine, naive splitn parser). All 16 tests pass; fmt and clippy clean. --- Cargo.lock | 28 ++++++ Cargo.toml | 1 + src/bin/filter_archive.rs | 144 +++++++++++-------------------- src/bin/github_archive_loader.rs | 40 ++++----- src/bin/produce_statistics.rs | 47 +++++----- 5 files changed, 116 insertions(+), 144 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2510381..3e33b67 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -316,6 +316,27 @@ version = "0.8.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" +[[package]] +name = "csv" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52cd9d68cf7efc6ddfaaee42e7288d3a99d613d4b50f76ce9827ae0c6e14f938" +dependencies = [ + "csv-core", + "itoa", + "ryu", + "serde_core", +] + +[[package]] +name = "csv-core" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "704a3c26996a80471189265814dbc2c257598b96b8a7feae2d31ace646bb9782" +dependencies = [ + "memchr", +] + [[package]] name = "deranged" version = "0.5.8" @@ -403,6 +424,7 @@ dependencies = [ "bytes", "chrono", "clap", + "csv", "flate2", "humanize-duration", "reqwest", @@ -1425,6 +1447,12 @@ version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + [[package]] name = "same-file" version = "1.0.6" diff --git a/Cargo.toml b/Cargo.toml index bc03fe9..4cd1273 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,6 +11,7 @@ chrono = { version = "0.4", features = ["clock"] } serde = { version = "1", features = ["derive"] } serde_json = { version = "1", features = ["preserve_order"] } clap = { version = "4", features = ["derive", "env"] } +csv = "1" flate2 = "1" anyhow = "1" humanize-duration = "0.0.7" diff --git a/src/bin/filter_archive.rs b/src/bin/filter_archive.rs index 8e72d27..273e419 100644 --- a/src/bin/filter_archive.rs +++ b/src/bin/filter_archive.rs @@ -19,7 +19,6 @@ use anyhow::{Context, Result}; use clap::Parser; use std::collections::{HashMap, HashSet}; use std::fs::File; -use std::io::{BufRead, BufReader, BufWriter, Write}; use std::path::{Path, PathBuf}; // ── CLI ─────────────────────────────────────────────────────────────────────── @@ -420,29 +419,26 @@ fn log_filter(name: &str, before: usize, after: usize, note: &str) { /// Reads the CSV into a `Vec`, skipping the header and any malformed lines. fn read_csv(path: &Path) -> Result> { let file = File::open(path).with_context(|| format!("open {path:?}"))?; - let reader = BufReader::with_capacity(1 << 20, file); - let mut lines = reader.lines(); - - // Consume and validate header - let header = lines.next().with_context(|| "file is empty")??; - if !header.starts_with("actor,") { - eprintln!("WARN: unexpected header: {header}"); - } + let mut rdr = csv::ReaderBuilder::new() + .buffer_capacity(1 << 20) + .from_reader(file); let mut rows = Vec::new(); let mut bad = 0u64; - for line_result in lines { - let line = line_result.context("I/O error reading line")?; - if line.is_empty() { - continue; - } - let fields = split_csv_line(&line); - if fields.len() != 6 { + for result in rdr.records() { + let record = match result { + Ok(r) => r, + Err(_) => { + bad += 1; + continue; + } + }; + if record.len() != 6 { bad += 1; continue; } - let count: u64 = match fields[5].trim().parse() { + let count: u64 = match record[5].trim().parse() { Ok(n) => n, Err(_) => { bad += 1; @@ -450,11 +446,11 @@ fn read_csv(path: &Path) -> Result> { } }; rows.push(Row { - actor: fields[0].clone(), - repo: fields[1].clone(), - event_type: fields[2].clone(), - action: fields[3].clone(), - language: fields[4].clone(), + actor: record[0].to_string(), + repo: record[1].to_string(), + event_type: record[2].to_string(), + action: record[3].to_string(), + language: record[4].to_string(), count, }); } @@ -469,21 +465,21 @@ fn read_csv(path: &Path) -> Result> { /// Writes rows as RFC 4180 CSV. fn write_csv(rows: Vec, path: &Path) -> Result<()> { let file = File::create(path).with_context(|| format!("create {path:?}"))?; - let mut w = BufWriter::new(file); - w.write_all(b"actor,repo,event_type,action,language,count\n") + let mut w = csv::WriterBuilder::new() + .terminator(csv::Terminator::Any(b'\n')) + .from_writer(file); + w.write_record(["actor", "repo", "event_type", "action", "language", "count"]) .context("write header")?; for r in &rows { - let actor = csv_field(&r.actor); - let repo = csv_field(&r.repo); - let event_type = csv_field(&r.event_type); - let action = csv_field(&r.action); - let language = csv_field(&r.language); - writeln!( - w, - "{actor},{repo},{event_type},{action},{language},{}", - r.count - ) + w.write_record([ + &r.actor, + &r.repo, + &r.event_type, + &r.action, + &r.language, + &r.count.to_string(), + ]) .context("write row")?; } @@ -495,46 +491,6 @@ fn write_csv(rows: Vec, path: &Path) -> Result<()> { Ok(()) } -// ── CSV helpers ─────────────────────────────────────────────────────────────── - -/// Quotes a CSV field if it contains a comma, double-quote, or newline. -fn csv_field(s: &str) -> std::borrow::Cow<'_, str> { - if s.contains([',', '"', '\n', '\r']) { - std::borrow::Cow::Owned(format!("\"{}\"", s.replace('"', "\"\""))) - } else { - std::borrow::Cow::Borrowed(s) - } -} - -/// Minimal RFC 4180 CSV line splitter. -fn split_csv_line(line: &str) -> Vec { - let mut fields = Vec::with_capacity(6); - let mut field = String::new(); - let mut in_quotes = false; - let mut chars = line.chars().peekable(); - - while let Some(c) = chars.next() { - match c { - '"' if in_quotes => { - if chars.peek() == Some(&'"') { - chars.next(); - field.push('"'); - } else { - in_quotes = false; - } - } - '"' => in_quotes = true, - ',' if !in_quotes => { - fields.push(field.clone()); - field.clear(); - } - other => field.push(other), - } - } - fields.push(field); - fields -} - // ── Tests ───────────────────────────────────────────────────────────────────── #[cfg(test)] @@ -583,7 +539,7 @@ mod tests { let rows = vec![ row("alice", "a/a", "PushEvent", 1), row("github-actions[bot]", "a/a", "PushEvent", 1), - row("Renovate-App", "a/a", "PullRequestEvent", 1), // case-insensitive + row("Renovate-App", "a/a", "PullRequestEvent", 1), // case-insensitive row("snyk-io", "a/a", "PushEvent", 1), row("netlify[bot]", "a/a", "PushEvent", 1), ]; @@ -597,8 +553,8 @@ mod tests { fn test_filter_single_event_repos() { let rows = vec![ row("alice", "active/repo", "PushEvent", 3), - row("bob", "active/repo", "PushEvent", 1), // total=4, kept - row("carol", "quiet/repo", "PushEvent", 1), // total=1, dropped + row("bob", "active/repo", "PushEvent", 1), // total=4, kept + row("carol", "quiet/repo", "PushEvent", 1), // total=1, dropped ]; let result = filter_single_event_repos(rows); assert_eq!(repos(&result), vec!["active/repo", "active/repo"]); @@ -610,9 +566,9 @@ mod tests { fn test_filter_empty_repos() { let rows = vec![ row("alice", "code/repo", "PushEvent", 2), - row("alice", "code/repo", "IssuesEvent", 1), // kept: repo has pushes - row("bob", "watch/repo", "WatchEvent", 5), // dropped: no push/PR - row("bob", "watch/repo", "IssuesEvent", 1), // dropped: same repo + row("alice", "code/repo", "IssuesEvent", 1), // kept: repo has pushes + row("bob", "watch/repo", "WatchEvent", 5), // dropped: no push/PR + row("bob", "watch/repo", "IssuesEvent", 1), // dropped: same repo ]; let result = filter_empty_repos(rows); assert_eq!(repos(&result), vec!["code/repo", "code/repo"]); @@ -624,8 +580,8 @@ mod tests { fn test_filter_high_volume_actors() { let rows = vec![ row("alice", "a/a", "PushEvent", 5), - row("alice", "a/b", "PushEvent", 5), // alice total=10, kept (limit=10) - row("bob", "b/a", "PushEvent", 11), // bob total=11, dropped + row("alice", "a/b", "PushEvent", 5), // alice total=10, kept (limit=10) + row("bob", "b/a", "PushEvent", 11), // bob total=11, dropped ]; let result = filter_high_volume_actors(rows, 10); assert_eq!(actors(&result), vec!["alice", "alice"]); @@ -653,10 +609,10 @@ mod tests { #[test] fn test_filter_fork_only_actors() { let rows = vec![ - row("alice", "a/a", "PushEvent", 1), // alice has code activity → kept - row("alice", "a/a", "ForkEvent", 1), // kept: alice isn't fork-only - row("bob", "b/b", "ForkEvent", 3), // bob is fork-only → dropped - row("carol", "c/c", "WatchEvent", 2), // carol is watch-only → dropped + row("alice", "a/a", "PushEvent", 1), // alice has code activity → kept + row("alice", "a/a", "ForkEvent", 1), // kept: alice isn't fork-only + row("bob", "b/b", "ForkEvent", 3), // bob is fork-only → dropped + row("carol", "c/c", "WatchEvent", 2), // carol is watch-only → dropped ]; let result = filter_fork_only_actors(rows); assert_eq!(actors(&result), vec!["alice", "alice"]); @@ -667,13 +623,16 @@ mod tests { #[test] fn test_filter_high_volume_issue_repos() { let rows = vec![ - row("alice", "spam/repo", "IssuesEvent", 200), // total issues=200 > limit=100 - row("alice", "spam/repo", "PushEvent", 1), // push kept even for spam/repo - row("bob", "good/repo", "IssuesEvent", 50), // total issues=50, kept + row("alice", "spam/repo", "IssuesEvent", 200), // total issues=200 > limit=100 + row("alice", "spam/repo", "PushEvent", 1), // push kept even for spam/repo + row("bob", "good/repo", "IssuesEvent", 50), // total issues=50, kept ]; let result = filter_high_volume_issue_repos(rows, 100); assert_eq!( - result.iter().map(|r| (r.repo.as_str(), r.event_type.as_str())).collect::>(), + result + .iter() + .map(|r| (r.repo.as_str(), r.event_type.as_str())) + .collect::>(), vec![("spam/repo", "PushEvent"), ("good/repo", "IssuesEvent")] ); } @@ -683,9 +642,9 @@ mod tests { #[test] fn test_filter_issue_only_actors() { let rows = vec![ - row("alice", "a/a", "PushEvent", 1), // alice has code → issue row kept + row("alice", "a/a", "PushEvent", 1), // alice has code → issue row kept row("alice", "a/a", "IssuesEvent", 2), - row("bob", "b/b", "IssuesEvent", 5), // bob has no code → issue row dropped + row("bob", "b/b", "IssuesEvent", 5), // bob has no code → issue row dropped ]; let result = filter_issue_only_actors(rows); assert_eq!( @@ -736,4 +695,3 @@ bob,rust-lang/rust,PushEvent,,,2 Ok(()) } } - diff --git a/src/bin/github_archive_loader.rs b/src/bin/github_archive_loader.rs index 0c456d7..be68ea8 100644 --- a/src/bin/github_archive_loader.rs +++ b/src/bin/github_archive_loader.rs @@ -21,7 +21,8 @@ //! `Vec` keyed by //! (actor, repo, event_type, action). //! -//! write_csv — writes `Vec` as RFC 4180 CSV. +//! write_csv — writes `Vec` as RFC 4180 CSV +//! using the `csv` crate. //! //! Usage: //! github-archive-loader --year 2026 --month 1 --parallelism 10 --output events.csv @@ -67,7 +68,7 @@ use flate2::read::GzDecoder; use serde_json::Value; use std::collections::HashMap; use std::fs::File; -use std::io::{BufRead, BufReader, BufWriter, Write}; +use std::io::{BufRead, BufReader}; use std::path::PathBuf; use std::time::{Duration, Instant}; use tokio::sync::mpsc; @@ -429,21 +430,21 @@ async fn aggregate_events(mut rx: mpsc::Receiver) -> Result, path: &PathBuf) -> Result<()> { let file = File::create(path).with_context(|| format!("create {path:?}"))?; - let mut w = BufWriter::new(file); - w.write_all(b"actor,repo,event_type,action,language,count\n") + let mut w = csv::WriterBuilder::new() + .terminator(csv::Terminator::Any(b'\n')) + .from_writer(file); + w.write_record(["actor", "repo", "event_type", "action", "language", "count"]) .context("write CSV header")?; for row in &rows { - let actor = csv_field(&row.actor); - let repo = csv_field(&row.repo); - let etype = csv_field(&row.event_type); - let action = csv_field(&row.action); - let language = csv_field(&row.language); - writeln!( - w, - "{actor},{repo},{etype},{action},{language},{}", - row.count - ) + w.write_record([ + &row.actor, + &row.repo, + &row.event_type, + &row.action, + &row.language, + &row.count.to_string(), + ]) .context("write CSV row")?; } @@ -576,14 +577,3 @@ fn extract_event(value: &Value) -> Option { language, }) } - -// ── CSV helpers ─────────────────────────────────────────────────────────────── - -/// Quotes a CSV field if it contains a comma, double-quote, or newline. -fn csv_field(s: &str) -> std::borrow::Cow<'_, str> { - if s.contains([',', '"', '\n', '\r']) { - std::borrow::Cow::Owned(format!("\"{}\"", s.replace('"', "\"\""))) - } else { - std::borrow::Cow::Borrowed(s) - } -} diff --git a/src/bin/produce_statistics.rs b/src/bin/produce_statistics.rs index fb6a769..2fb17db 100644 --- a/src/bin/produce_statistics.rs +++ b/src/bin/produce_statistics.rs @@ -376,7 +376,8 @@ fn load_languages(path: &PathBuf) -> Result { /// CSV format (first row is header): /// actor,repo,event_type,action,language,count fn collect_counts(path: &PathBuf) -> Result { - let reader = open(path)?; + let file = File::open(path).with_context(|| format!("cannot open {path:?}"))?; + let mut rdr = csv::Reader::from_reader(file); let mut pr_counts: HashMap = HashMap::new(); let mut dev_actor_sets: HashMap> = HashMap::new(); @@ -386,36 +387,33 @@ fn collect_counts(path: &PathBuf) -> Result { let mut star_counts: HashMap = HashMap::new(); let mut parse_errors = 0u64; - for (i, line) in reader.lines().enumerate() { - let line = line.context("read error")?; - let line = line.trim(); - if line.is_empty() { - continue; - } - // Skip header row. - if i == 0 && line.starts_with("actor,") { - continue; - } - - let fields: Vec<&str> = line.splitn(6, ',').collect(); - if fields.len() < 6 { + for (i, result) in rdr.records().enumerate() { + let record = match result { + Ok(r) => r, + Err(e) => { + eprintln!(" [skip] CSV record {}: {e}", i + 2); + parse_errors += 1; + continue; + } + }; + if record.len() < 6 { eprintln!( - " [skip] CSV line {}: expected 6 fields, got {}", - i + 1, - fields.len() + " [skip] CSV record {}: expected 6 fields, got {}", + i + 2, + record.len() ); parse_errors += 1; continue; } - let actor = fields[0].trim_matches('"'); - let repo = fields[1].trim_matches('"'); - let event_type = fields[2].trim_matches('"'); - let count_str = fields[5].trim_matches('"'); + let actor = &record[0]; + let repo = &record[1]; + let event_type = &record[2]; + let count_str = &record[5]; let count: u64 = match count_str.parse() { Ok(v) => v, Err(_) => { - eprintln!(" [skip] non-numeric count on CSV line {}", i + 1); + eprintln!(" [skip] non-numeric count on CSV record {}", i + 2); parse_errors += 1; continue; } @@ -601,9 +599,7 @@ carol,golang/go,WatchEvent,,,10 // golang/go: carol (push) = 1 dev → Go 1.0. // Total = 3.0 → Rust 60.0%, Go 33.33%, C 6.67% assert_eq!( - std::fs::read_to_string( - dir.join("language-ratings-2024-01-developer-activity.jsonl") - )?, + std::fs::read_to_string(dir.join("language-ratings-2024-01-developer-activity.jsonl"))?, r#"{"language":"Rust","rating":1.8,"percentage":60.0} {"language":"Go","rating":1.0,"percentage":33.33} {"language":"C","rating":0.2,"percentage":6.67} @@ -633,4 +629,3 @@ carol,golang/go,WatchEvent,,,10 Ok(()) } } - From ba4d657d27ff95869fdbe096571c03cd824590f8 Mon Sep 17 00:00:00 2001 From: guenhter Date: Sun, 19 Jul 2026 08:03:28 +0200 Subject: [PATCH 6/6] refactor: remove --primary-only from produce_statistics The experimental flag was unused in production. Removes the CLI arg, the branching in compute_ratings, the output_path conditional, and the related module-doc section. --- src/bin/produce_statistics.rs | 135 +++++++--------------------------- 1 file changed, 27 insertions(+), 108 deletions(-) diff --git a/src/bin/produce_statistics.rs b/src/bin/produce_statistics.rs index 2fb17db..28252ad 100644 --- a/src/bin/produce_statistics.rs +++ b/src/bin/produce_statistics.rs @@ -12,34 +12,17 @@ //! language-ratings-YYYY-MM-active-repos.jsonl //! language-ratings-YYYY-MM-star-count.jsonl //! -//! With --primary-only the filenames gain a "-primary" suffix: -//! language-ratings-YYYY-MM-pr-count-primary.jsonl (etc.) -//! //! Rating formula (all metric types): -//! -//! Default (proportional): -//! For each repo, distribute the event count across all its languages -//! weighted by byte share. -//! pr-count: rating[L] += pr_count × (size_L / total_size) -//! issue-count: rating[L] += issue_count × (size_L / total_size) -//! push-count: rating[L] += push_count × (size_L / total_size) -//! developer-activity: rating[L] += distinct_contributors × (size_L / total_size) -//! (distinct actors across PullRequestEvent + PushEvent) -//! active-repos: rating[L] += 1 × (size_L / total_size) -//! (once per repo that had any PushEvent or PullRequestEvent) -//! star-count: rating[L] += star_count × (size_L / total_size) -//! -//! --primary-only (experimental): -//! All credit goes to the single dominant (largest-by-bytes) language; -//! secondary languages are ignored. Score multiplier is always 1, not the -//! fractional byte share. -//! pr-count: rating[primary] += pr_count -//! issue-count: rating[primary] += issue_count -//! push-count: rating[primary] += push_count -//! developer-activity: rating[primary] += distinct_contributors -//! (distinct actors across PullRequestEvent + PushEvent) -//! active-repos: rating[primary] += 1 -//! star-count: rating[primary] += star_count +//! For each repo, distribute the event count across all its languages +//! weighted by byte share. +//! pr-count: rating[L] += pr_count × (size_L / total_size) +//! issue-count: rating[L] += issue_count × (size_L / total_size) +//! push-count: rating[L] += push_count × (size_L / total_size) +//! developer-activity: rating[L] += distinct_contributors × (size_L / total_size) +//! (distinct actors across PullRequestEvent + PushEvent) +//! active-repos: rating[L] += 1 × (size_L / total_size) +//! (once per repo that had any PushEvent or PullRequestEvent) +//! star-count: rating[L] += star_count × (size_L / total_size) //! //! Event types read from the archive CSV: //! PullRequestEvent → pr-count, developer-activity, and active-repos @@ -96,13 +79,6 @@ struct Args { /// Files are named: language-ratings-YYYY-MM-.jsonl #[arg(long)] output_dir: PathBuf, - - /// Experimental: attribute all of a repo's score to its primary (largest) - /// language only, with a weight of 1 instead of the fractional byte share. - /// Secondary languages are ignored entirely. - /// Output filenames gain a "-primary" suffix. - #[arg(long, default_value_t = false)] - primary_only: bool, } // ── Data types ──────────────────────────────────────────────────────────────── @@ -167,49 +143,34 @@ fn run(args: Args) -> Result<()> { ); // ── pr-count ───────────────────────────────────────────────────────────── - let out = output_path(&args.output_dir, &year_month, "pr-count", args.primary_only); + let out = output_path(&args.output_dir, &year_month, "pr-count"); eprintln!("Writing {out:?} …"); { let mut w = open_writer(&out)?; - let ratings = compute_ratings(&counts.pr_counts, &lang_map, "PR", args.primary_only); + let ratings = compute_ratings(&counts.pr_counts, &lang_map, "PR"); write_ratings(&mut w, &ratings)?; } // ── issue-count ────────────────────────────────────────────────────────── - let out = output_path( - &args.output_dir, - &year_month, - "issue-count", - args.primary_only, - ); + let out = output_path(&args.output_dir, &year_month, "issue-count"); eprintln!("Writing {out:?} …"); { let mut w = open_writer(&out)?; - let ratings = compute_ratings(&counts.issue_counts, &lang_map, "issue", args.primary_only); + let ratings = compute_ratings(&counts.issue_counts, &lang_map, "issue"); write_ratings(&mut w, &ratings)?; } // ── push-count ─────────────────────────────────────────────────────────── - let out = output_path( - &args.output_dir, - &year_month, - "push-count", - args.primary_only, - ); + let out = output_path(&args.output_dir, &year_month, "push-count"); eprintln!("Writing {out:?} …"); { let mut w = open_writer(&out)?; - let ratings = compute_ratings(&counts.push_counts, &lang_map, "push", args.primary_only); + let ratings = compute_ratings(&counts.push_counts, &lang_map, "push"); write_ratings(&mut w, &ratings)?; } // ── developer-activity ─────────────────────────────────────────────────── - let out = output_path( - &args.output_dir, - &year_month, - "developer-activity", - args.primary_only, - ); + let out = output_path(&args.output_dir, &year_month, "developer-activity"); eprintln!("Writing {out:?} …"); { let mut w = open_writer(&out)?; @@ -219,50 +180,25 @@ fn run(args: Args) -> Result<()> { .iter() .map(|(repo, n)| (repo.clone(), *n as u64)) .collect(); - let ratings = compute_ratings( - &dev_counts, - &lang_map, - "developer-activity", - args.primary_only, - ); + let ratings = compute_ratings(&dev_counts, &lang_map, "developer-activity"); write_ratings(&mut w, &ratings)?; } // ── active-repos ───────────────────────────────────────────────────────── - let out = output_path( - &args.output_dir, - &year_month, - "active-repos", - args.primary_only, - ); + let out = output_path(&args.output_dir, &year_month, "active-repos"); eprintln!("Writing {out:?} …"); { let mut w = open_writer(&out)?; - let ratings = compute_ratings( - &counts.active_repos, - &lang_map, - "active-repos", - args.primary_only, - ); + let ratings = compute_ratings(&counts.active_repos, &lang_map, "active-repos"); write_ratings(&mut w, &ratings)?; } // ── star-count ─────────────────────────────────────────────────────────── - let out = output_path( - &args.output_dir, - &year_month, - "star-count", - args.primary_only, - ); + let out = output_path(&args.output_dir, &year_month, "star-count"); eprintln!("Writing {out:?} …"); { let mut w = open_writer(&out)?; - let ratings = compute_ratings( - &counts.star_counts, - &lang_map, - "star-count", - args.primary_only, - ); + let ratings = compute_ratings(&counts.star_counts, &lang_map, "star-count"); write_ratings(&mut w, &ratings)?; } @@ -298,15 +234,8 @@ fn infer_year_month(path: &PathBuf) -> Result { } /// Build the output file path for a given type. -/// With `primary_only` the filename gains a "-primary" suffix before ".jsonl". -fn output_path(dir: &Path, year_month: &str, kind: &str, primary_only: bool) -> PathBuf { - if primary_only { - dir.join(format!( - "language-ratings-{year_month}-{kind}-primary.jsonl" - )) - } else { - dir.join(format!("language-ratings-{year_month}-{kind}.jsonl")) - } +fn output_path(dir: &Path, year_month: &str, kind: &str) -> PathBuf { + dir.join(format!("language-ratings-{year_month}-{kind}.jsonl")) } /// Open a file for writing, wrapped in a BufWriter. @@ -473,16 +402,12 @@ fn collect_counts(path: &PathBuf) -> Result { /// Compute language ratings from a map of per-repo event counts. /// -/// Default (proportional): distribute each repo's count across all its -/// languages weighted by byte share. -/// -/// Primary-only: attribute the full count to the single dominant language; -/// secondary languages are ignored. +/// For each repo, distributes the event count across all its languages +/// weighted by byte share (proportional attribution). fn compute_ratings( event_counts: &HashMap, lang_map: &LangMap, label: &str, - primary_only: bool, ) -> Vec<(String, f64)> { let mut ratings: HashMap = HashMap::new(); let mut matched = 0u64; @@ -490,12 +415,7 @@ fn compute_ratings( for (repo, count) in event_counts { if let Some((total_size, langs)) = lang_map.get(repo.as_str()) { - if primary_only { - // All credit to the primary (first = largest) language, weight = 1. - if let Some((lang, _)) = langs.first() { - *ratings.entry(lang.clone()).or_insert(0.0) += *count as f64; - } - } else if *total_size > 0 { + if *total_size > 0 { for (lang, size) in langs { let share = *size as f64 / *total_size as f64; *ratings.entry(lang.clone()).or_insert(0.0) += *count as f64 * share; @@ -562,7 +482,6 @@ carol,golang/go,WatchEvent,,,10 archive: dir.join("archive-202401-filtered.csv"), languages: dir.join("languages-2024-01.jsonl"), output_dir: dir.to_path_buf(), - primary_only: false, })?; // ── pr-count ─────────────────────────────────────────────────────────