Sitelet https://github.com/starkware-libs/cairo/commit/329a5a8d2107c3241a5013074004659264c1e86a
Skip to content

Commit 329a5a8

Browse files
eytan-starkwareeytan-starkware
committed
(feat): cache external (plugin-generated) file content in the defs cache
Serialize each external file's VirtualFile content into the crate cache blob as its own (first) section, keyed by a portable, parse-free identity (ExternalFileKey), and serve it from ext_as_virtual before falling back to module_sub_files. A loaded crate cache can then materialize plugin-generated files without re-running plugins. The lookup depends only on the blob (no module-data edge), so it never re-enters cache loading.
1 parent 0f2cd03 commit 329a5a8

5 files changed

Lines changed: 353 additions & 13 deletions

File tree

‎crates/cairo-lang-defs/src/cache/mod.rs‎

Lines changed: 119 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ use cairo_lang_syntax::node::ids::{GreenId, SyntaxStablePtrId};
2121
use cairo_lang_syntax::node::kind::SyntaxKind;
2222
use cairo_lang_syntax::node::{SyntaxNode, SyntaxNodeId, TypedSyntaxNode, ast};
2323
use cairo_lang_utils::Intern;
24-
use cairo_lang_utils::ordered_hash_map::OrderedHashMap;
24+
use cairo_lang_utils::ordered_hash_map::{Entry, OrderedHashMap};
2525
use salsa::Database;
2626
use serde::{Deserialize, Serialize};
2727
use thiserror::Error;
@@ -41,8 +41,11 @@ use crate::ids::{
4141
};
4242
use crate::plugin::{DynGeneratedFileAuxData, PluginDiagnostic};
4343

44+
/// The index of the `external` section in a crate cache blob.
45+
pub(crate) const EXTERNAL_CACHE_SECTION: u8 = 0;
46+
4447
/// The index of the `def` section in a crate cache blob.
45-
pub const DEF_CACHE_SECTION: u8 = 0;
48+
const DEF_CACHE_SECTION: u8 = 1;
4649

4750
/// Metadata for a cached crate.
4851
#[derive(Serialize, Deserialize)]
@@ -99,7 +102,7 @@ pub fn load_cached_crate_modules<'db>(
99102
};
100103

101104
let (metadata, module_data, defs_lookups): DefCache<'_> =
102-
CacheBlobReader::read_section(db, content, DEF_CACHE_SECTION, &crate_id);
105+
CacheBlobReader::read_section(db, content, DEF_CACHE_SECTION, crate_id);
103106

104107
validate_metadata(crate_id, &metadata, db);
105108

@@ -178,7 +181,7 @@ impl<'a> CacheBlobReader<'a> {
178181
db: &'db dyn Database,
179182
content: &'a [u8],
180183
section: u8,
181-
crate_id: &CrateId<'db>,
184+
crate_id: CrateId<'db>,
182185
) -> T {
183186
let mut reader = CacheBlobReader::new(content);
184187
for _ in 0..section {
@@ -400,6 +403,12 @@ impl<'db> DefCacheSavingContext<'db> {
400403
pub fn new(db: &'db dyn Database, self_crate_id: CrateId<'db>) -> Self {
401404
Self { db, data: DefCacheSavingData::default(), self_crate_id }
402405
}
406+
407+
/// Serializes the external-file content as its own blob-section payload (read back by the
408+
/// `external_file_contents` query).
409+
pub fn serialize_external_file_contents(&self) -> Result<Vec<u8>, postcard::Error> {
410+
postcard::to_allocvec(&self.data.external_file_contents)
411+
}
403412
}
404413

405414
/// Data for saving cache from the database.
@@ -426,6 +435,9 @@ pub struct DefCacheSavingData<'db> {
426435
syntax_nodes: OrderedHashMap<SyntaxNode<'db>, SyntaxNodeCached>,
427436
file_ids: OrderedHashMap<FileId<'db>, FileIdCached>,
428437

438+
/// External (plugin-generated) file content, written as its own blob section in cache.
439+
external_file_contents: OrderedHashMap<ExternalFileKey, ExternalFileContentCached>,
440+
429441
pub lookups: DefCacheLookups,
430442
}
431443

@@ -1941,14 +1953,116 @@ impl FileCached {
19411953
}
19421954
}
19431955

1956+
/// Portable, parse-free identity for an external file — the key into the external-file content
1957+
/// table. Computed during cache load by reading only already-materialized fields, so it never
1958+
/// parses or re-runs plugins (which would cycle back into the cache).
1959+
#[derive(Serialize, Deserialize, Clone, Eq, Hash, PartialEq, Debug)]
1960+
pub(crate) enum ExternalFileKey {
1961+
/// An on-disk file, identified by its path.
1962+
OnDisk(PathBuf),
1963+
/// A virtual file: parent (key + span), name, and a content hash (hashed, not stored, so a
1964+
/// shared ancestor isn't duplicated across keys).
1965+
Virtual { parent: Option<(Box<ExternalFileKey>, TextSpan)>, name: String, content_hash: u64 },
1966+
/// A plugin-generated file: the parent file's key, the stable-ptr offset within it, and name.
1967+
Generated { parent: Box<ExternalFileKey>, offset: u32, name: String },
1968+
}
1969+
1970+
impl ExternalFileKey {
1971+
/// Computes the key for `file_id`; see the type docs for why this is parse/plugin/mint-free.
1972+
pub(crate) fn new<'db>(db: &'db dyn Database, file_id: FileId<'db>) -> Self {
1973+
match file_id.long(db) {
1974+
FileLongId::OnDisk(path) => ExternalFileKey::OnDisk(path.clone()),
1975+
FileLongId::Virtual(vf) => ExternalFileKey::Virtual {
1976+
parent: vf.parent.map(|parent| {
1977+
(Box::new(ExternalFileKey::new(db, parent.file_id)), parent.span)
1978+
}),
1979+
name: vf.name.long(db).to_string(),
1980+
content_hash: {
1981+
let mut hasher = xxhash_rust::xxh3::Xxh3::default();
1982+
vf.content.long(db).as_str().hash(&mut hasher);
1983+
hasher.finish()
1984+
},
1985+
},
1986+
FileLongId::External(external_id) => {
1987+
let long_id = PluginGeneratedFileId::from_intern_id(*external_id).long(db);
1988+
ExternalFileKey::Generated {
1989+
parent: Box::new(ExternalFileKey::new(db, long_id.stable_ptr.file_id(db))),
1990+
offset: long_id.stable_ptr.0.offset(db).as_u32(),
1991+
name: long_id.name.clone(),
1992+
}
1993+
}
1994+
}
1995+
}
1996+
}
1997+
1998+
/// An external file's content, enough to rebuild its `VirtualFile` on load. `parent` and `name` are
1999+
/// not stored — they come from the runtime `PluginGeneratedFileLongId` at lookup.
2000+
#[derive(Serialize, Deserialize, Clone, PartialEq, Eq, Debug)]
2001+
pub(crate) struct ExternalFileContentCached {
2002+
content: String,
2003+
code_mappings: Vec<CodeMapping>,
2004+
kind: FileKind,
2005+
original_item_removed: bool,
2006+
}
2007+
2008+
impl ExternalFileContentCached {
2009+
fn new<'db>(virtual_file: &VirtualFile<'db>, db: &'db dyn Database) -> Self {
2010+
Self {
2011+
content: virtual_file.content.to_string(db),
2012+
code_mappings: virtual_file.code_mappings.to_vec(),
2013+
kind: virtual_file.kind,
2014+
original_item_removed: virtual_file.original_item_removed,
2015+
}
2016+
}
2017+
2018+
/// Rebuilds the `VirtualFile`; `parent` and `name` come from the runtime stable ptr / long id.
2019+
pub(crate) fn to_virtual_file<'db>(
2020+
&self,
2021+
db: &'db dyn Database,
2022+
name: &str,
2023+
parent: Option<SpanInFile<'db>>,
2024+
) -> VirtualFile<'db> {
2025+
VirtualFile {
2026+
parent,
2027+
name: SmolStrId::from(db, name),
2028+
content: SmolStrId::from(db, self.content.clone()),
2029+
code_mappings: self.code_mappings.clone().into(),
2030+
kind: self.kind,
2031+
original_item_removed: self.original_item_removed,
2032+
}
2033+
}
2034+
}
2035+
19442036
#[derive(Serialize, Deserialize, Clone, Copy, Eq, Hash, PartialEq, salsa::Update, Debug)]
19452037
pub struct FileIdCached(usize);
19462038
impl FileIdCached {
19472039
fn new<'db>(id: FileId<'db>, ctx: &mut DefCacheSavingContext<'db>) -> Self {
19482040
if let Some(cached_id) = ctx.file_ids.get(&id) {
19492041
return *cached_id;
19502042
}
1951-
let cached = FileCached::new(id.long(ctx.db), ctx);
2043+
let long_id = id.long(ctx.db);
2044+
// For external (plugin-generated) files, also cache their content so `ext_as_virtual` can
2045+
// serve it from the blob instead of re-running plugins.
2046+
if let FileLongId::External(external_id) = long_id {
2047+
// Distinct `FileId`s can map to the same `ExternalFileKey` (the key is coarser), so
2048+
// dedup against the keys already stored in `external_file_contents`.
2049+
let key = ExternalFileKey::new(ctx.db, id);
2050+
let virtual_file = cairo_lang_filesystem::db::ext_as_virtual(ctx.db, *external_id);
2051+
let content = ExternalFileContentCached::new(virtual_file, ctx.db);
2052+
match ctx.external_file_contents.entry(key) {
2053+
Entry::Vacant(entry) => {
2054+
entry.insert(content);
2055+
}
2056+
// Two distinct external files collapsing to one key must carry identical content,
2057+
// else the load would serve the wrong bytes. Holds today (generated files have
2058+
// distinct parent+offset); guard it so a future key change fails loud, not silent.
2059+
Entry::Occupied(entry) => debug_assert!(
2060+
*entry.get() == content,
2061+
"external file content key collision with differing content"
2062+
),
2063+
}
2064+
}
2065+
let cached = FileCached::new(long_id, ctx);
19522066
let cached_id = FileIdCached(ctx.file_ids_lookup.len());
19532067
ctx.file_ids_lookup.push(cached);
19542068
ctx.file_ids.insert(id, cached_id);

‎crates/cairo-lang-defs/src/db.rs‎

Lines changed: 57 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,8 @@ use cairo_lang_diagnostics::{
66
};
77
use cairo_lang_filesystem::db::{ExtAsVirtual, FilesGroup, files_group_input};
88
use cairo_lang_filesystem::ids::{
9-
CrateId, CrateInput, Directory, FileId, FileKind, FileLongId, SmolStrId, Tracked, VirtualFile,
9+
BlobId, CrateId, CrateInput, Directory, FileId, FileKind, FileLongId, SmolStrId, Tracked,
10+
VirtualFile,
1011
};
1112
use cairo_lang_parser::db::ParserGroup;
1213
use cairo_lang_syntax::attribute::consts::{
@@ -25,7 +26,10 @@ use cairo_lang_utils::ordered_hash_set::OrderedHashSet;
2526
use itertools::{Itertools, chain};
2627
use salsa::{Database, Setter};
2728

28-
use crate::cache::{DefCacheLoadingData, load_cached_crate_modules};
29+
use crate::cache::{
30+
CacheBlobReader, DefCacheLoadingData, EXTERNAL_CACHE_SECTION, ExternalFileContentCached,
31+
ExternalFileKey, load_cached_crate_modules,
32+
};
2933
use crate::ids::*;
3034
use crate::plugin::{DynGeneratedFileAuxData, MacroPlugin, MacroPluginMetadata, PluginDiagnostic};
3135
use crate::plugin_utils::try_extract_unnamed_arg;
@@ -1110,13 +1114,63 @@ pub fn init_external_files<T: DefsGroup>(db: &mut T) {
11101114
files_group_input(db).set_ext_as_virtual_obj(db).to(Some(ext_as_virtual_impl));
11111115
}
11121116

1117+
/// The external-file content table from a crate's cache blob, keyed only on the blob — so reading
1118+
/// it from `ext_as_virtual` adds no module-data edge, keeping cache loading cycle-free.
1119+
#[salsa::tracked(returns(ref))]
1120+
fn external_file_contents<'db>(
1121+
db: &'db dyn Database,
1122+
blob_id: BlobId<'db>,
1123+
crate_id: CrateId<'db>,
1124+
) -> OrderedHashMap<ExternalFileKey, ExternalFileContentCached> {
1125+
let Some(content) = db.blob_content(blob_id) else {
1126+
return Default::default();
1127+
};
1128+
1129+
CacheBlobReader::read_section(db, content, EXTERNAL_CACHE_SECTION, crate_id)
1130+
}
1131+
1132+
/// Rebuilds an external file's `VirtualFile` from the cache blob. `None` if the crate has no cache
1133+
/// blob or the blob lacks this file — `ext_as_virtual_impl` then falls back to `module_sub_files`.
1134+
#[salsa::tracked(returns(ref))]
1135+
fn cached_external_virtual_file<'db>(
1136+
db: &'db dyn Database,
1137+
external_file: FileId<'db>,
1138+
) -> Option<VirtualFile<'db>> {
1139+
let FileLongId::External(raw_external_id) = external_file.long(db) else {
1140+
return None;
1141+
};
1142+
let long_id = PluginGeneratedFileId::from_intern_id(*raw_external_id).long(db);
1143+
let crate_id = long_id.module_id.owning_crate(db);
1144+
let blob_id = db.crate_config(crate_id)?.cache_file?;
1145+
// Parse/plugin/mint-free, so it cannot re-enter the cache. See `ExternalFileKey::new`.
1146+
let key = ExternalFileKey::new(db, external_file);
1147+
let content = external_file_contents(db, blob_id, crate_id).get(&key)?;
1148+
Some(content.to_virtual_file(db, &long_id.name, Some(long_id.stable_ptr.span_in_file(db))))
1149+
}
1150+
11131151
/// Returns the `VirtualFile` matching the given external id.
11141152
pub fn ext_as_virtual_impl<'db>(
11151153
db: &'db dyn Database,
11161154
external_id: salsa::Id,
11171155
) -> &'db VirtualFile<'db> {
1118-
let long_id = PluginGeneratedFileId::from_intern_id(external_id).long(db);
11191156
let file_id = FileLongId::External(external_id).intern(db);
1157+
// Serve from the cache blob first (no module-data edge); see `external_file_contents`.
1158+
if let Some(virtual_file) = cached_external_virtual_file(db, file_id) {
1159+
return virtual_file;
1160+
}
1161+
let long_id = PluginGeneratedFileId::from_intern_id(external_id).long(db);
1162+
// A loaded blob collected every external file at save time, so a miss means a stale/corrupt
1163+
// cache; falling back to `module_sub_files` would re-enter the in-flight load (cycle), so fail
1164+
// loud. An absent/unreadable blob means the crate isn't cache-backed — the fallback is safe.
1165+
if let Some(blob_id) =
1166+
db.crate_config(long_id.module_id.owning_crate(db)).and_then(|config| config.cache_file)
1167+
{
1168+
assert!(
1169+
db.blob_content(blob_id).is_none(),
1170+
"external file missing from a loaded crate cache: {:?}",
1171+
file_id.long(db)
1172+
);
1173+
}
11201174
let data =
11211175
module_sub_files(db, long_id.module_id, long_id.stable_ptr.file_id(db)).as_ref().unwrap();
11221176
&data.files[&file_id]

‎crates/cairo-lang-lowering/src/cache/mod.rs‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,7 @@ use crate::{
5858

5959
type LookupCache = (CacheLookups, Vec<(DefsFunctionWithBodyIdCached, MultiLoweringCached)>);
6060

61-
const LOWERING_CACHE_SECTION: u8 = 2;
61+
const LOWERING_CACHE_SECTION: u8 = 3;
6262

6363
/// Load the cached lowering of a crate if it has a cache file configuration.
6464
pub fn load_cached_crate_functions<'db>(
@@ -73,7 +73,7 @@ pub fn load_cached_crate_functions<'db>(
7373
let semantic_loading_data = db.cached_crate_semantic_data(crate_id)?.loading_data;
7474

7575
let (lookups, lowerings): LookupCache =
76-
CacheBlobReader::read_section(db, content, LOWERING_CACHE_SECTION, &crate_id);
76+
CacheBlobReader::read_section(db, content, LOWERING_CACHE_SECTION, crate_id);
7777

7878
// TODO(tomer): Fail on version, cfg, and dependencies mismatch.
7979

@@ -152,6 +152,16 @@ pub fn generate_crate_cache<'db>(
152152

153153
let mut artifact = Vec::<u8>::new();
154154

155+
write_cache_section(
156+
&mut artifact,
157+
&(CachedCrateMetadata::new(crate_id, db), def_cache, &ctx.semantic_ctx.defs_ctx.lookups),
158+
)?;
159+
// External (plugin-generated) file content comes first: it's filesystem-layer content (served
160+
// by `ext_as_virtual`) that the phase sections below depend on, and putting it first lets the
161+
// `external_file_contents` query (in defs) read it without deserializing them.
162+
let external = ctx.semantic_ctx.defs_ctx.serialize_external_file_contents()?;
163+
write_cache_section(&mut artifact, &external);
164+
155165
write_cache_section(
156166
&mut artifact,
157167
&(CachedCrateMetadata::new(crate_id, db), def_cache, &ctx.semantic_ctx.defs_ctx.lookups),

0 commit comments

Comments
 (0)