A Rust crate for training code: save, resume, fork and list checkpoints on a
local directory, a raw s3://, gs:// or az:// bucket, or o41://. It
speaks checkpoints and knows nothing about what their files mean. SexpGPU
checkpoints through it from its next release.
use o41_checkpoint::{open, step_name, Config, Name};// The location picks the backend: a directory, s3:// gs:// az://, or o41://.let checkpoints = open("o41://experiments/hoonshot/1234", Config::from_env()?).await?;// Before the first step: the latest, or None on an empty location, and the writer.let (resumed, mut writer) = checkpoints.resume().await?;if let Some(at) = &resumed { let state = at.file("state.json")?.bytes().await?; let header = at.file("params.safetensors")?.range(0..8).await?; // lazy, ranged}writer.save(&step_name(510), files, meta).await?;// A fresh start, refused on a location that has checkpoints.let writer = checkpoints.start().await?;// A fork into this empty location, from any checkpoint by its full path.let (source, writer) = checkpoints.fork("o41://experiments/hoonshot/1234/step-00000300").await?;// Readers.let latest = checkpoints.load(Name::Latest).await?;let listed = checkpoints.list().await?;let note = checkpoints.note("evaluated.json").await?;
Configuration
Config::from_env() reads:
Variable
Is
O41_ARTIFACTS_API_KEY
An organization API key; an o41:// location is refused without it