Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
61 changes: 61 additions & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 3 additions & 1 deletion Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -42,11 +42,13 @@ flate2 = { version = "1", optional = true }
aes = { version = "0.9", optional = true }
base64 = "0.22"
pdf-extract = { version = "0.12", optional = true }
owo-colors = { version = "4", optional = true }
indicatif = { version = "0.17", optional = true }

[features]
default = ["full"]
retrieval = []
full = ["retrieval", "dep:parquet", "dep:axum", "dep:tower", "dep:pdf-extract", "dep:tar", "dep:flate2", "dep:aes", "dep:zip"]
full = ["retrieval", "dep:parquet", "dep:axum", "dep:tower", "dep:pdf-extract", "dep:tar", "dep:flate2", "dep:aes", "dep:zip", "dep:owo-colors", "dep:indicatif"]

[dev-dependencies]
tokio = { version = "1", features = ["macros", "rt-multi-thread"] }
Expand Down
65 changes: 48 additions & 17 deletions src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@ use kibble::{
fetch, index, ingest, init, mcp, pack, retrieve, serve, soul, train, tune,
};

mod ui;

#[derive(Parser)]
#[command(
name = "kibble",
Expand All @@ -12,8 +14,19 @@ use kibble::{
before_help = " U・ᴥ・U 🦴 kibble 🦴 — fetch · chew · digest"
)]
struct Cli {
/// When to colorize output.
#[arg(long, value_enum, global = true, default_value_t = ColorArg::Auto)]
color: ColorArg,
#[command(subcommand)]
command: Command,
command: Option<Command>,
}

#[derive(clap::ValueEnum, Clone, Copy)]
enum ColorArg { Auto, Always, Never }
impl From<ColorArg> for ui::ColorWhen {
fn from(c: ColorArg) -> Self {
match c { ColorArg::Auto => ui::ColorWhen::Auto, ColorArg::Always => ui::ColorWhen::Always, ColorArg::Never => ui::ColorWhen::Never }
}
}

#[derive(Subcommand)]
Expand Down Expand Up @@ -84,7 +97,7 @@ enum Command {
},
/// Run an MCP (Model Context Protocol) server over stdio, exposing KIBBLE's tools.
Mcp,
/// Crawl a website (BFS, same-host, markdown) into the ingest tree for `build`.
/// Crawl a website (BFS, same-host, markdown) into the ingest tree for `build` — polite by default.
Crawl {
/// Seed URL to crawl.
url: String,
Expand All @@ -98,18 +111,18 @@ enum Command {
#[arg(long = "all-hosts")]
all_hosts: bool,
},
/// Scaffold a ready-to-run kibble project (kibble.toml + sample corpus) in the current dir.
/// Scaffold a ready-to-run kibble project (kibble.toml + sample corpus) in the current dir — the easiest way to get started.
Init {
/// Overwrite an existing kibble.toml.
#[arg(long)]
force: bool,
},
/// Build the retrieval index over the corpus (or an explicit path).
/// Build the retrieval index over your corpus (chunks + BM25, semantic if an embed backend is set).
Index {
/// Optional dir/file to index (default: [index].sources).
path: Option<String>,
},
/// Search the retrieval index (hybrid semantic + BM25).
/// Search the index (hybrid semantic + BM25) and get back the good stuff.
Search {
/// The query text.
query: String,
Expand Down Expand Up @@ -228,7 +241,7 @@ trait OrExit<T> {
impl<T, E: std::fmt::Display> OrExit<T> for Result<T, E> {
fn or_exit(self) -> T {
self.unwrap_or_else(|e| {
eprintln!("kibble: {e}");
ui::err(&format!("kibble: {e}"));
std::process::exit(1);
})
}
Expand All @@ -240,18 +253,25 @@ async fn main() {
// Real env vars always win; this only fills in what isn't already set.
dotenv::load();
let cli = Cli::parse();
match cli.command {
ui::init_color(cli.color.into());
let Some(command) = cli.command else {
ui::greet();
return;
};
match command {
Command::Clean => {
use std::io::Read;
let mut input = String::new();
std::io::stdin().read_to_string(&mut input).or_exit();
print!("{}", clean::clean_text(&input));
}
Command::Build => {
let pb = ui::spinner("building dataset…");
let stats = build::run_build(std::path::Path::new(".")).await.or_exit();
ui::finish(pb);
if stats.total_documents == 0 {
eprintln!("kibble: no sources found — add [[source]] to kibble.toml or drop files in data/raw/.");
eprintln!(" try: kibble init (scaffolds a sample project you can build)");
ui::err("no sources found — add [[source]] to kibble.toml or drop files in data/raw/");
ui::hint("kibble init (scaffolds a sample project you can build)");
return;
}
println!(
Expand All @@ -262,7 +282,7 @@ async fn main() {
stats.dropped_topic_rebalanced
);
for (name, t, v, te) in &stats.sources {
println!(" source {name}: train {t} / valid {v} / test {te}");
ui::note(&format!(" source {name}: train {t} / valid {v} / test {te}"));
}
}
Command::Pack => {
Expand All @@ -281,15 +301,19 @@ async fn main() {
serve::run_serve(std::path::Path::new(".")).await.or_exit();
}
Command::Fetch { url, name, map, max_rows } => {
let pb = ui::spinner("fetching…");
fetch::run_fetch(std::path::Path::new("."), &url, name.as_deref(), map.as_deref(), max_rows)
.await
.or_exit();
ui::finish(pb);
}
Command::Extract { path, out, skip_unsupported } => {
let pb = ui::spinner("extracting…");
let n = extract::run_extract(std::path::Path::new("."), std::path::Path::new(&path), out.as_deref(), skip_unsupported)
.await
.or_exit();
println!("Extracted {n} artifact(s) -> data/extracted (or --out)");
ui::finish(pb);
ui::ok(&format!("extracted {n} artifacts → {}", out.as_deref().unwrap_or("data/extracted")));
}
Command::Caps { action } => {
let repo_root = std::path::Path::new(".");
Expand Down Expand Up @@ -344,24 +368,26 @@ async fn main() {
Path::new("data/raw/local/twitter"),
)
.or_exit();
println!("Wrote {n} tweet files to data/raw/local/twitter");
ui::ok(&format!("wrote {n} tweet files data/raw/local/twitter"));
}
IngestSource::Textfiles => {
let n = ingest::ingest_textfiles(
Path::new("textfiles"),
Path::new("data/raw/local/textfiles"),
)
.or_exit();
println!("Wrote {n} textfiles to data/raw/local/textfiles");
ui::ok(&format!("wrote {n} textfiles data/raw/local/textfiles"));
}
}
}
Command::Mcp => {
mcp::run_mcp(std::path::Path::new(".")).await.or_exit();
}
Command::Crawl { url, depth, max_pages, out, all_hosts } => {
let pb = ui::spinner("crawling…");
let st = crawl::run_crawl(std::path::Path::new("."), &url, depth, max_pages, out, all_hosts).await.or_exit();
println!("Crawled {} → {} pages, {} skipped → {}", st.seed, st.pages, st.skipped, st.out_dir);
ui::finish(pb);
ui::ok(&format!("crawled {} pages ({} skipped) → {}", st.pages, st.skipped, st.out_dir));
}
Command::Init { force } => {
let r = init::run_init(std::path::Path::new("."), force).or_exit();
Expand All @@ -375,8 +401,12 @@ async fn main() {
Command::Index { path } => {
let root = std::path::Path::new(".");
let explicit = path.as_ref().map(std::path::PathBuf::from);
let pb = ui::spinner("indexing…");
let n = index::build_index(root, explicit.as_deref()).await.or_exit();
println!("Indexed {n} chunk(s) -> data/index (or [index].dir)");
ui::finish(pb);
let dir = config::load_config(&root.join(crate::config::CONFIG_FILE)).index.dir;
ui::ok(&format!("indexed {n} chunks → {dir}"));
ui::hint("kibble search \"<query>\"");
}
Command::Search { query, k, json } => {
let hits = retrieve::search(std::path::Path::new("."), &query, k).await.or_exit();
Expand All @@ -387,7 +417,8 @@ async fn main() {
})).collect();
println!("{}", serde_json::to_string_pretty(&arr).unwrap());
} else if hits.is_empty() {
println!("No results.");
ui::warn(&format!("nothing matched {query:?}"));
ui::hint("broaden the query, or index more: kibble index <path>");
} else {
for (i, h) in hits.iter().enumerate() {
let snippet: String = h.chunk.text.chars().take(120).collect();
Expand Down Expand Up @@ -424,7 +455,7 @@ async fn main() {
}
Command::Train { dry_run, extra } => {
if let Err(e) = train::run_train(std::path::Path::new("."), dry_run, &extra) {
eprintln!("kibble: {e}");
ui::err(&format!("kibble: {e}"));
std::process::exit(1);
}
}
Expand Down
Loading
Loading