Use the new self-describing format, remove serde_json wrapper

Author: bgw

Cargo.lock

@@ -92,7 +92,7 @@ pub struct NextConfig {
     cache_handler: Option<RcStr>,
     #[bincode(with_serde)]
     cache_handlers: Option<FxIndexMap<RcStr, RcStr>>,
-    #[bincode(with = "turbo_bincode::serde_json")]
+    #[bincode(with = "turbo_bincode::serde_self_describing")]
     env: FxIndexMap<String, JsonValue>,
     experimental: ExperimentalConfig,
     images: ImageConfig,
@@ -101,12 +101,12 @@ pub struct NextConfig {
     react_production_profiling: Option<bool>,
     react_strict_mode: Option<bool>,
     transpile_packages: Option<Vec<RcStr>>,
-    #[bincode(with = "turbo_bincode::serde_json")]
+    #[bincode(with = "turbo_bincode::serde_self_describing")]
     modularize_imports: Option<FxIndexMap<String, ModularizeImportPackageConfig>>,
     dist_dir: RcStr,
     dist_dir_root: RcStr,
     deployment_id: Option<RcStr>,
-    #[bincode(with = "turbo_bincode::serde_json")]
+    #[bincode(with = "turbo_bincode::serde_self_describing")]
     sass_options: Option<serde_json::Value>,
     trailing_slash: Option<bool>,
     asset_prefix: Option<RcStr>,
@@ -119,9 +119,9 @@ pub struct NextConfig {
     output: Option<OutputType>,
     turbopack: Option<TurbopackConfig>,
     production_browser_source_maps: bool,
-    #[bincode(with = "turbo_bincode::serde_json")]
+    #[bincode(with = "turbo_bincode::serde_self_describing")]
     output_file_tracing_includes: Option<serde_json::Value>,
-    #[bincode(with = "turbo_bincode::serde_json")]
+    #[bincode(with = "turbo_bincode::serde_self_describing")]
     output_file_tracing_excludes: Option<serde_json::Value>,
     // TODO: This option is not respected, it uses Turbopack's root instead.
     output_file_tracing_root: Option<RcStr>,
@@ -154,9 +154,9 @@ pub struct NextConfig {
     http_agent_options: HttpAgentConfig,
     on_demand_entries: OnDemandEntriesConfig,
     powered_by_header: bool,
-    #[bincode(with = "turbo_bincode::serde_json")]
+    #[bincode(with = "turbo_bincode::serde_self_describing")]
     public_runtime_config: FxIndexMap<String, serde_json::Value>,
-    #[bincode(with = "turbo_bincode::serde_json")]
+    #[bincode(with = "turbo_bincode::serde_self_describing")]
     server_runtime_config: FxIndexMap<String, serde_json::Value>,
     static_page_generation_timeout: f64,
     target: Option<String>,
@@ -686,9 +686,9 @@ pub enum RemotePatternProtocol {
 )]
 #[serde(rename_all = "camelCase")]
 pub struct TurbopackConfig {
-    #[bincode(with = "turbo_bincode::serde_json")]
+    #[bincode(with = "turbo_bincode::serde_self_describing")]
     pub rules: Option<FxIndexMap<RcStr, RuleConfigCollection>>,
-    #[bincode(with = "turbo_bincode::serde_json")]
+    #[bincode(with = "turbo_bincode::serde_self_describing")]
     pub resolve_alias: Option<FxIndexMap<RcStr, JsonValue>>,
     pub resolve_extensions: Option<Vec<RcStr>>,
     pub debug_ids: Option<bool>,
@@ -955,7 +955,7 @@ pub struct ExperimentalConfig {
     /// @see [api reference](https://nextjs.org/docs/app/api-reference/next-config-js/mdxRs)
     mdx_rs: Option<MdxRsOptions>,
     strict_next_head: Option<bool>,
-    #[bincode(with = "turbo_bincode::serde_json")]
+    #[bincode(with = "turbo_bincode::serde_self_describing")]
     swc_plugins: Option<Vec<(RcStr, serde_json::Value)>>,
     external_middleware_rewrites_resolve: Option<bool>,
     scroll_restoration: Option<bool>,
@@ -964,7 +964,7 @@ pub struct ExperimentalConfig {
     middleware_prefetch: Option<MiddlewarePrefetchType>,
     /// optimizeCss can be boolean or critters' option object
     /// Use Record<string, unknown> as critters doesn't export its Option type ([link](https://github.com/GoogleChromeLabs/critters/blob/a590c05f9197b656d2aeaae9369df2483c26b072/packages/critters/src/index.d.ts))
-    #[bincode(with = "turbo_bincode::serde_json")]
+    #[bincode(with = "turbo_bincode::serde_self_describing")]
     optimize_css: Option<serde_json::Value>,
     next_script_workers: Option<bool>,
     web_vitals_attribution: Option<Vec<RcStr>>,
@@ -982,15 +982,15 @@ pub struct ExperimentalConfig {
     adjust_font_fallbacks_with_size_adjust: Option<bool>,
     after: Option<bool>,
     app_document_preloading: Option<bool>,
-    #[bincode(with = "turbo_bincode::serde_json")]
+    #[bincode(with = "turbo_bincode::serde_self_describing")]
     cache_life: Option<FxIndexMap<String, CacheLifeProfile>>,
     case_sensitive_routes: Option<bool>,
     cpus: Option<f64>,
     cra_compat: Option<bool>,
     disable_optimized_loading: Option<bool>,
     disable_postcss_preset_env: Option<bool>,
     esm_externals: Option<EsmExternals>,
-    #[bincode(with = "turbo_bincode::serde_json")]
+    #[bincode(with = "turbo_bincode::serde_self_describing")]
     extension_alias: Option<serde_json::Value>,
     external_dir: Option<bool>,
     /// If set to `false`, webpack won't fall back to polyfill Node.js modules
@@ -1005,7 +1005,7 @@ pub struct ExperimentalConfig {
     instrumentation_hook: Option<bool>,
     client_trace_metadata: Option<Vec<String>>,
     large_page_data_bytes: Option<f64>,
-    #[bincode(with = "turbo_bincode::serde_json")]
+    #[bincode(with = "turbo_bincode::serde_self_describing")]
     logging: Option<serde_json::Value>,
     memory_based_workers_count: Option<bool>,
     /// Optimize React APIs for server builds.
@@ -1024,7 +1024,7 @@ pub struct ExperimentalConfig {
     /// @internal Used by the Next.js internals only.
     trust_host_header: Option<bool>,
 
-    #[bincode(with = "turbo_bincode::serde_json")]
+    #[bincode(with = "turbo_bincode::serde_self_describing")]
     url_imports: Option<serde_json::Value>,
     /// This option is to enable running the Webpack build in a worker thread
     /// (doesn't apply to Turbopack).
@@ -1395,7 +1395,7 @@ pub struct ResolveExtensions(Option<Vec<RcStr>>);
 
 #[turbo_tasks::value(transparent)]
 pub struct SwcPlugins(
-    #[bincode(with = "turbo_bincode::serde_json")] Vec<(RcStr, serde_json::Value)>,
+    #[bincode(with = "turbo_bincode::serde_self_describing")] Vec<(RcStr, serde_json::Value)>,
 );
 
 #[turbo_tasks::value(transparent)]
@@ -1413,7 +1413,7 @@ pub struct OptionServerActions(Option<ServerActions>);
 
 #[turbo_tasks::value(transparent)]
 pub struct OptionJsonValue(
-    #[bincode(with = "turbo_bincode::serde_json")] pub Option<serde_json::Value>,
+    #[bincode(with = "turbo_bincode::serde_self_describing")] pub Option<serde_json::Value>,
 );
 
 fn turbopack_config_documentation_link() -> RcStr {
@@ -1466,7 +1466,7 @@ impl Issue for InvalidLoaderRuleRenameAsIssue {
 
 #[turbo_tasks::value(shared)]
 struct InvalidLoaderRuleConditionIssue {
-    #[bincode(with = "turbo_bincode::serde_json")]
+    #[bincode(with = "turbo_bincode::serde_self_describing")]
     condition: ConfigConditionItem,
     config_file_path: FileSystemPath,
 }
@@ -2207,7 +2207,7 @@ impl NextConfig {
 #[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize, Encode, Decode)]
 #[serde(rename_all = "camelCase")]
 pub struct JsConfig {
-    #[bincode(with = "turbo_bincode::serde_json")]
+    #[bincode(with = "turbo_bincode::serde_self_describing")]
     compiler_options: Option<serde_json::Value>,
 }
 

crates/next-core/src/next_config.rs

@@ -17,6 +17,5 @@ indexmap = { workspace = true }
 mime = { workspace = true }
 ringmap = { workspace = true }
 serde = { workspace = true }
-serde_json = { workspace = true }
 smallvec = { workspace = true }
 unty = { workspace = true }

turbopack/crates/turbo-bincode/Cargo.toml

@@ -473,76 +473,6 @@ pub mod mime_option {
     }
 }
 
-/// Encode/decode as a serialized string encoded using `serde_json`.
-///
-/// This encodes less efficiently than `#[bincode(with_serde)]` would, but avoids [bincode's known
-/// compatibility issues][serde-issues]. Use this for infrequently-serialized types and when you're
-/// unsure if the underlying type may trigger a serde compatibility issue.
-///
-/// In the future this could be replaced with a more efficient serde-compatible self-describing
-/// format with a compact binary representation (e.g. pot or MessagePack), but `serde_json` is
-/// convenient because it avoids introducing additional dependencies.
-///
-/// [serde-issues]: https://docs.rs/bincode/latest/bincode/serde/index.html#known-issues
-pub mod serde_json {
-    use super::*;
-
-    pub fn encode<E: Encoder, T: serde::Serialize>(
-        value: &T,
-        encoder: &mut E,
-    ) -> Result<(), EncodeError> {
-        let json_str =
-            ::serde_json::to_string(value).map_err(|e| EncodeError::OtherString(e.to_string()))?;
-        Encode::encode(&json_str, encoder)
-    }
-
-    pub fn decode<Context, D: Decoder<Context = Context>, T: serde::de::DeserializeOwned>(
-        decoder: &mut D,
-    ) -> Result<T, DecodeError> {
-        let json_str: String = Decode::decode(decoder)?;
-        ::serde_json::from_str(&json_str).map_err(|e| DecodeError::OtherString(e.to_string()))
-    }
-
-    pub fn borrow_decode<
-        'de,
-        Context,
-        D: BorrowDecoder<'de, Context = Context>,
-        T: serde::de::Deserialize<'de>,
-    >(
-        decoder: &mut D,
-    ) -> Result<T, DecodeError> {
-        let json_str: &str = BorrowDecode::borrow_decode(decoder)?;
-        ::serde_json::from_str(json_str).map_err(|e| DecodeError::OtherString(e.to_string()))
-    }
-
-    #[cfg(test)]
-    mod tests {
-        use ::serde_json::{Value, json};
-        use bincode::{decode_from_slice, encode_to_vec};
-
-        use super::*;
-
-        #[test]
-        fn test_roundtrip() {
-            let cfg = bincode::config::standard();
-
-            #[derive(Encode, Decode)]
-            struct Wrapper(#[bincode(with = "crate::serde_json")] Value);
-
-            let value1 = Wrapper(json!({
-                "key1": [1, 2, 3],
-                "key2": [4, 5, 6]
-            }));
-
-            let value2: Wrapper = decode_from_slice(&encode_to_vec(&value1, cfg).unwrap(), cfg)
-                .unwrap()
-                .0;
-
-            assert_eq!(value1.0, value2.0);
-        }
-    }
-}
-
 pub mod either {
     use ::either::Either;
 

turbopack/crates/turbo-bincode/src/lib.rs

@@ -14,9 +14,9 @@ verbose_log = []
 
 [dependencies]
 anyhow = { workspace = true }
-either = { workspace = true }
-pot = "3.0.0"
+bincode = { workspace = true }
 byteorder = { workspace = true }
+either = { workspace = true }
 jiff = "0.2.10"
 lzzzz = "1.1.0"
 memmap2 = "0.9.5"
@@ -27,6 +27,7 @@ rustc-hash = { workspace = true }
 smallvec = { workspace = true }
 thread_local = { workspace = true }
 tracing = { workspace = true }
+turbo-bincode= { workspace = true }
 twox-hash = { workspace = true }
 zstd = { version = "0.13.2", features = ["zdict_builder"] }
 

turbopack/crates/turbo-persistence/Cargo.toml

@@ -8,11 +8,13 @@ use std::{
 };
 
 use anyhow::{Context, Result, bail};
+use bincode::{Decode, Encode};
 use byteorder::{BE, ReadBytesExt};
 use either::Either;
 use memmap2::{Mmap, MmapOptions};
 use quick_cache::sync::GuardResult;
 use rustc_hash::FxHasher;
+use turbo_bincode::turbo_bincode_decode;
 
 use crate::{
     QueryKey,
@@ -31,6 +33,14 @@ impl quick_cache::Weighter<u32, Arc<qfilter::Filter>> for AmqfWeighter {
 pub type AmqfCache =
     quick_cache::sync::Cache<u32, Arc<qfilter::Filter>, AmqfWeighter, BuildHasherDefault<FxHasher>>;
 
+/// A wrapper around [`qfilter::Filter`] that implements [`Encode`] and [`Decode`].
+#[derive(Encode, Decode)]
+pub struct AmqfBincodeWrapper(
+    // this annotation can be replaced with `#[bincode(serde)]` once
+    // <https://github.com/arthurprs/qfilter/issues/13> is resolved
+    #[bincode(with = "turbo_bincode::serde_self_describing")] pub qfilter::Filter,
+);
+
 pub struct MetaEntry {
     /// The metadata for the static sorted file.
     sst_data: StaticSortedFileMetaData,
@@ -76,13 +86,15 @@ impl MetaEntry {
 
     pub fn deserialize_amqf(&self, meta: &MetaFile) -> Result<qfilter::Filter> {
         let amqf = self.raw_amqf(meta.amqf_data());
-        pot::from_slice(amqf).with_context(|| {
-            format!(
-                "Failed to deserialize AMQF from {:08}.meta for {:08}.sst",
-                meta.sequence_number,
-                self.sequence_number()
-            )
-        })
+        Ok(turbo_bincode_decode::<AmqfBincodeWrapper>(amqf)
+            .with_context(|| {
+                format!(
+                    "Failed to deserialize AMQF from {:08}.meta for {:08}.sst",
+                    meta.sequence_number,
+                    self.sequence_number()
+                )
+            })?
+            .0)
     }
 
     pub fn amqf(

turbopack/crates/turbo-persistence/src/meta_file.rs

@@ -8,9 +8,11 @@ use std::{
 
 use anyhow::{Context, Result};
 use byteorder::{BE, ByteOrder, WriteBytesExt};
+use turbo_bincode::{TurboBincodeBuffer, turbo_bincode_encode};
 
 use crate::{
     compression::compress_into_buffer,
+    meta_file::AmqfBincodeWrapper,
     static_sorted_file::{
         BLOCK_TYPE_INDEX, BLOCK_TYPE_KEY, KEY_BLOCK_ENTRY_TYPE_BLOB, KEY_BLOCK_ENTRY_TYPE_DELETED,
         KEY_BLOCK_ENTRY_TYPE_MEDIUM, KEY_BLOCK_ENTRY_TYPE_SMALL,
@@ -137,7 +139,7 @@ pub fn write_static_stored_file<E: Entry>(
     let meta = StaticSortedFileBuilderMeta {
         min_hash,
         max_hash,
-        amqf: Cow::Owned(amqf),
+        amqf: Cow::Owned(amqf.into_vec()),
         key_compression_dictionary_length: key_dict.len().try_into().unwrap(),
         block_count,
         size: file.stream_position()?,
@@ -390,7 +392,7 @@ fn write_key_blocks_and_compute_amqf(
     key_compression_dictionary: &[u8],
     writer: &mut BlockWriter<'_>,
     buffer: &mut Vec<u8>,
-) -> Result<Vec<u8>> {
+) -> Result<TurboBincodeBuffer> {
     let mut filter = qfilter::Filter::new(entries.len() as u64, AMQF_FALSE_POSITIVE_RATE)
         // This won't fail as we limit the number of entries per SST file
         .expect("Filter can't be constructed");
@@ -497,7 +499,7 @@ fn write_key_blocks_and_compute_amqf(
     writer.write_index_block(buffer, key_compression_dictionary)?;
     buffer.clear();
 
-    Ok(pot::to_vec(&filter).expect("AMQF serialization failed"))
+    Ok(turbo_bincode_encode(&AmqfBincodeWrapper(filter)).expect("AMQF serialization failed"))
 }
 
 /// Builder for a single key block

turbopack/crates/turbo-persistence/src/static_sorted_file_builder.rs

@@ -56,7 +56,7 @@ impl ManualEncodeWrapper for JsonValueEncodeWrapper<'_> {
 
 impl Encode for JsonValueEncodeWrapper<'_> {
     fn encode<E: Encoder>(&self, encoder: &mut E) -> Result<(), EncodeError> {
-        turbo_bincode::serde_json::encode(self.0, encoder)
+        turbo_bincode::serde_self_describing::encode(self.0, encoder)
     }
 }
 
@@ -72,6 +72,6 @@ impl ManualDecodeWrapper for JsonValueDecodeWrapper {
 
 impl<Context> Decode<Context> for JsonValueDecodeWrapper {
     fn decode<D: Decoder<Context = Context>>(decoder: &mut D) -> Result<Self, DecodeError> {
-        Ok(Self(turbo_bincode::serde_json::decode(decoder)?))
+        Ok(Self(turbo_bincode::serde_self_describing::decode(decoder)?))
     }
 }