diff --git a/crates/rrt-cli/src/main.rs b/crates/rrt-cli/src/main.rs index f52fd1e..062aeac 100644 --- a/crates/rrt-cli/src/main.rs +++ b/crates/rrt-cli/src/main.rs @@ -27,7 +27,8 @@ use rrt_runtime::{ SmpRt3105PackedProfileBlock, SmpSaveLoadSummary, WinInspectionReport, execute_step_command, extract_pk4_entry_file, inspect_campaign_exe_file, inspect_cargo_economy_sources_with_bindings, inspect_cargo_skin_pk4, inspect_cargo_types_dir, inspect_pk4_file, - inspect_save_company_and_chairman_analysis_file, inspect_smp_file, inspect_win_file, + inspect_save_company_and_chairman_analysis_file, inspect_smp_file, + inspect_unclassified_save_collection_headers_file, inspect_win_file, load_runtime_snapshot_document, load_runtime_state_import, load_save_slice_file, project_save_slice_to_runtime_state_import, save_runtime_overlay_import_document, save_runtime_save_slice_document, save_runtime_snapshot_document, @@ -130,6 +131,9 @@ enum Command { RuntimeInspectSaveCompanyChairman { smp_path: PathBuf, }, + RuntimeInspectUnclassifiedSaveCollections { + smp_path: PathBuf, + }, RuntimeImportSaveState { smp_path: PathBuf, output_path: PathBuf, @@ -853,6 +857,9 @@ fn real_main() -> Result<(), Box> { Command::RuntimeInspectSaveCompanyChairman { smp_path } => { run_runtime_inspect_save_company_chairman(&smp_path)?; } + Command::RuntimeInspectUnclassifiedSaveCollections { smp_path } => { + run_runtime_inspect_unclassified_save_collections(&smp_path)?; + } Command::RuntimeImportSaveState { smp_path, output_path, @@ -1056,6 +1063,13 @@ fn parse_command() -> Result> { smp_path: PathBuf::from(path), }) } + [command, subcommand, path] + if command == "runtime" && subcommand == "inspect-unclassified-save-collections" => + { + Ok(Command::RuntimeInspectUnclassifiedSaveCollections { + smp_path: PathBuf::from(path), + }) + } [command, subcommand, smp_path, output_path] if command == "runtime" && subcommand == "import-save-state" => { @@ -1259,7 +1273,7 @@ fn parse_command() -> Result> { }) } _ => Err( - "usage: rrt-cli [validate [repo-root] | finance eval | finance diff | runtime validate-fixture | runtime summarize-fixture | runtime export-fixture-state | runtime diff-state | runtime summarize-state | runtime import-state | runtime inspect-smp | runtime summarize-save-load | runtime load-save-slice | runtime inspect-save-company-chairman | runtime import-save-state | runtime export-save-slice | runtime export-overlay-import | runtime inspect-pk4 | runtime inspect-cargo-types | runtime inspect-cargo-skins | runtime inspect-cargo-economy-sources | runtime inspect-cargo-production-selector | runtime inspect-cargo-price-selector | runtime inspect-win | runtime extract-pk4-entry | runtime inspect-campaign-exe | runtime compare-classic-profile [saveN.gms...] | runtime compare-105-profile [saveN.gms...] | runtime compare-candidate-table [fileN...] | runtime compare-recipe-book-lines [fileN...] | runtime compare-setup-payload-core [fileN...] | runtime compare-setup-launch-payload [fileN...] | runtime compare-post-special-conditions-scalars [fileN...] | runtime scan-candidate-table-headers | runtime scan-special-conditions | runtime scan-aligned-runtime-rule-band | runtime scan-post-special-conditions-scalars | runtime scan-post-special-conditions-tail | runtime scan-recipe-book-lines | runtime export-profile-block ]" + "usage: rrt-cli [validate [repo-root] | finance eval | finance diff | runtime validate-fixture | runtime summarize-fixture | runtime export-fixture-state | runtime diff-state | runtime summarize-state | runtime import-state | runtime inspect-smp | runtime summarize-save-load | runtime load-save-slice | runtime inspect-save-company-chairman | runtime inspect-unclassified-save-collections | runtime import-save-state | runtime export-save-slice | runtime export-overlay-import | runtime inspect-pk4 | runtime inspect-cargo-types | runtime inspect-cargo-skins | runtime inspect-cargo-economy-sources | runtime inspect-cargo-production-selector | runtime inspect-cargo-price-selector | runtime inspect-win | runtime extract-pk4-entry | runtime inspect-campaign-exe | runtime compare-classic-profile [saveN.gms...] | runtime compare-105-profile [saveN.gms...] | runtime compare-candidate-table [fileN...] | runtime compare-recipe-book-lines [fileN...] | runtime compare-setup-payload-core [fileN...] | runtime compare-setup-launch-payload [fileN...] | runtime compare-post-special-conditions-scalars [fileN...] | runtime scan-candidate-table-headers | runtime scan-special-conditions | runtime scan-aligned-runtime-rule-band | runtime scan-post-special-conditions-scalars | runtime scan-post-special-conditions-tail | runtime scan-recipe-book-lines | runtime export-profile-block ]" .into(), ), } @@ -1500,6 +1514,18 @@ fn run_runtime_inspect_save_company_chairman( Ok(()) } +fn run_runtime_inspect_unclassified_save_collections( + smp_path: &Path, +) -> Result<(), Box> { + println!( + "{}", + serde_json::to_string_pretty(&inspect_unclassified_save_collection_headers_file( + smp_path + )?)? + ); + Ok(()) +} + fn run_runtime_import_save_state( smp_path: &Path, output_path: &Path, diff --git a/crates/rrt-runtime/src/lib.rs b/crates/rrt-runtime/src/lib.rs index 1810045..ed1a3e5 100644 --- a/crates/rrt-runtime/src/lib.rs +++ b/crates/rrt-runtime/src/lib.rs @@ -118,6 +118,7 @@ pub use smp::{ SmpRuntimePostSpanHeaderCandidate, SmpRuntimePostSpanProbe, SmpRuntimeTrailerBlock, SmpSaveAnchorRunBlock, SmpSaveBootstrapBlock, SmpSaveChairmanRecordAnalysisEntry, SmpSaveCompanyChairmanAnalysisReport, SmpSaveCompanyRecordAnalysisEntry, SmpSaveDwordCandidate, + SmpSavePlacedStructureDynamicSideBufferProbe, SmpSaveLoadCandidateTableSummary, SmpSaveLoadSummary, SmpSaveScalarCandidate, SmpSaveTaggedCollectionHeaderProbe, SmpSaveWorldEconomicTuningProbe, SmpSaveWorldFinanceNeighborhoodProbe, SmpSaveWorldIssue37Probe, @@ -125,7 +126,8 @@ pub use smp::{ SmpSecondaryVariantProbe, SmpSharedHeader, SmpSpecialConditionEntry, SmpSpecialConditionsProbe, inspect_save_company_and_chairman_analysis_bytes, inspect_save_company_and_chairman_analysis_file, inspect_smp_bytes, inspect_smp_file, - load_save_slice_file, load_save_slice_from_report, + inspect_unclassified_save_collection_headers_file, load_save_slice_file, + load_save_slice_from_report, }; pub use step::{BoundaryEvent, ServiceEvent, StepCommand, StepResult, execute_step_command}; pub use summary::RuntimeSummary; diff --git a/crates/rrt-runtime/src/smp.rs b/crates/rrt-runtime/src/smp.rs index 9aac944..98adb0e 100644 --- a/crates/rrt-runtime/src/smp.rs +++ b/crates/rrt-runtime/src/smp.rs @@ -1773,6 +1773,36 @@ pub struct SmpSavePlacedStructureRecordTripletProbe { pub evidence: Vec, } +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct SmpSavePlacedStructureDynamicSideBufferProbe { + pub profile_family: String, + pub source_kind: String, + pub semantic_family: String, + pub metadata_tag_offset: usize, + pub records_tag_offset: usize, + pub close_tag_offset: usize, + pub records_span_len: usize, + pub direct_record_stride: u32, + pub direct_record_stride_hex: String, + pub live_id_bound: u32, + pub live_id_bound_hex: String, + pub live_record_count: u32, + pub live_record_count_hex: String, + pub prefix_leading_dword: u32, + pub prefix_leading_dword_hex: String, + pub prefix_trailing_word: u16, + pub prefix_trailing_word_hex: String, + pub prefix_separator_byte: u8, + pub prefix_separator_byte_hex: String, + pub first_embedded_name_tag_relative_offset: usize, + pub embedded_name_tag_count: usize, + #[serde(default)] + pub first_embedded_primary_name: Option, + #[serde(default)] + pub first_embedded_secondary_name: Option, + pub evidence: Vec, +} + #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct SmpRt3105SaveNameTableProbe { pub profile_family: String, @@ -2719,6 +2749,9 @@ pub struct SmpSaveCompanyChairmanAnalysisReport { #[serde(default)] pub placed_structure_record_triplets: Option, #[serde(default)] + pub placed_structure_dynamic_side_buffer: + Option, + #[serde(default)] pub unclassified_tagged_collection_headers: Vec, #[serde(default)] pub company_entries: Vec, @@ -2987,6 +3020,9 @@ pub struct SmpInspectionReport { pub save_placed_structure_record_triplet_probe: Option, #[serde(default)] + pub save_placed_structure_dynamic_side_buffer_probe: + Option, + #[serde(default)] pub save_unclassified_tagged_collection_header_probes: Vec, #[serde(default)] @@ -3023,6 +3059,73 @@ pub fn inspect_smp_file(path: &Path) -> Result Result, Box> { + let bytes = fs::read(path)?; + let file_extension_hint = path + .extension() + .and_then(|extension| extension.to_str()) + .map(|extension| extension.to_ascii_lowercase()); + let shared_header = parse_shared_header(&bytes); + let header_variant_probe = shared_header.as_ref().map(classify_header_variant_probe); + let first_ascii_run = find_first_ascii_run(&bytes); + let early_content_probe = first_ascii_run + .as_ref() + .and_then(|ascii_run| probe_early_content_layout(&bytes, ascii_run)); + let secondary_variant_probe = early_content_probe + .as_ref() + .and_then(classify_secondary_variant_probe); + let container_profile = classify_container_profile( + file_extension_hint.as_deref(), + header_variant_probe.as_ref(), + secondary_variant_probe.as_ref(), + ); + let save_company_collection_header_probe = parse_save_company_collection_header_probe( + &bytes, + file_extension_hint.as_deref(), + container_profile.as_ref(), + ); + let save_chairman_profile_collection_header_probe = + parse_save_chairman_profile_collection_header_probe( + &bytes, + file_extension_hint.as_deref(), + container_profile.as_ref(), + ); + let save_train_collection_header_probe = parse_save_train_collection_header_probe( + &bytes, + file_extension_hint.as_deref(), + container_profile.as_ref(), + ); + let save_region_collection_header_probe = parse_save_region_collection_header_probe( + &bytes, + file_extension_hint.as_deref(), + container_profile.as_ref(), + ); + let save_placed_structure_collection_header_probe = + parse_save_placed_structure_collection_header_probe( + &bytes, + file_extension_hint.as_deref(), + container_profile.as_ref(), + ); + let known_header_probes = [ + save_company_collection_header_probe.as_ref(), + save_chairman_profile_collection_header_probe.as_ref(), + save_train_collection_header_probe.as_ref(), + save_region_collection_header_probe.as_ref(), + save_placed_structure_collection_header_probe.as_ref(), + ]; + let probes = scan_save_unclassified_tagged_collection_header_probes( + &bytes, + file_extension_hint.as_deref(), + container_profile.as_ref(), + ); + Ok(filter_unclassified_tagged_collection_header_probes_outside_known_spans( + probes, + &known_header_probes, + )) +} + pub fn inspect_smp_bytes(bytes: &[u8]) -> SmpInspectionReport { inspect_bundle_bytes(bytes, None) } @@ -3338,6 +3441,18 @@ pub fn load_save_slice_from_report( probe.entries.first().map(|entry| entry.profile_status_kind.as_str()) )); } + if let Some(probe) = &report.save_placed_structure_dynamic_side_buffer_probe { + notes.push(format!( + "Raw save also exposes the separate placed-structure dynamic-side-buffer candidate 0x38a5/0x38a6/0x38a7: live_record_count={}, first compact prefix=({},{},{}), first embedded names={:?}/{:?}, embedded 0x55f1 row count={}.", + probe.live_record_count, + probe.prefix_leading_dword_hex, + probe.prefix_trailing_word_hex, + probe.prefix_separator_byte_hex, + probe.first_embedded_primary_name.as_deref(), + probe.first_embedded_secondary_name.as_deref(), + probe.embedded_name_tag_count + )); + } if let Some(roster) = &report.save_company_roster_probe { notes.push(format!( "Raw save inspection reconstructed {} company direct records from the tagged company collection.", @@ -3402,8 +3517,11 @@ pub fn inspect_save_company_and_chairman_analysis_bytes( let region_record_triplets = report.save_region_record_triplet_probe.clone(); let placed_structure_record_triplets = report.save_placed_structure_record_triplet_probe.clone(); - let unclassified_tagged_collection_headers = - report.save_unclassified_tagged_collection_header_probes.clone(); + let placed_structure_dynamic_side_buffer = + report.save_placed_structure_dynamic_side_buffer_probe.clone(); + let unclassified_tagged_collection_headers = report + .save_unclassified_tagged_collection_header_probes + .clone(); let company_header_probe = report.save_company_collection_header_probe.as_ref(); let chairman_header_probe = report .save_chairman_profile_collection_header_probe @@ -3822,6 +3940,7 @@ pub fn inspect_save_company_and_chairman_analysis_bytes( .save_placed_structure_collection_header_probe .clone(), placed_structure_record_triplets, + placed_structure_dynamic_side_buffer, unclassified_tagged_collection_headers, company_entries, chairman_entries, @@ -7848,12 +7967,28 @@ fn inspect_bundle_bytes(bytes: &[u8], file_extension_hint: Option) -> Sm bytes, save_placed_structure_collection_header_probe.as_ref(), ); - let save_unclassified_tagged_collection_header_probes = - scan_save_unclassified_tagged_collection_header_probes( + let save_placed_structure_dynamic_side_buffer_probe = + parse_save_placed_structure_dynamic_side_buffer_probe( bytes, file_extension_hint.as_deref(), container_profile.as_ref(), ); + let known_header_probes = [ + save_company_collection_header_probe.as_ref(), + save_chairman_profile_collection_header_probe.as_ref(), + save_train_collection_header_probe.as_ref(), + save_region_collection_header_probe.as_ref(), + save_placed_structure_collection_header_probe.as_ref(), + ]; + let save_unclassified_tagged_collection_header_probes = + filter_unclassified_tagged_collection_header_probes_outside_known_spans( + scan_save_unclassified_tagged_collection_header_probes( + bytes, + file_extension_hint.as_deref(), + container_profile.as_ref(), + ), + &known_header_probes, + ); let save_company_roster_probe = parse_save_company_roster_probe( bytes, save_company_collection_header_probe.as_ref(), @@ -8023,6 +8158,7 @@ fn inspect_bundle_bytes(bytes: &[u8], file_extension_hint: Option) -> Sm save_region_record_triplet_probe, save_placed_structure_collection_header_probe, save_placed_structure_record_triplet_probe, + save_placed_structure_dynamic_side_buffer_probe, save_unclassified_tagged_collection_header_probes, save_company_roster_probe, save_chairman_profile_table_probe, @@ -10411,6 +10547,159 @@ fn parse_save_placed_structure_collection_header_probe( ) } +fn parse_save_placed_structure_dynamic_side_buffer_probe( + bytes: &[u8], + file_extension_hint: Option<&str>, + container_profile: Option<&SmpContainerProfile>, +) -> Option { + if file_extension_hint != Some("gms") { + return None; + } + let profile = container_profile?; + if !matches!( + profile.profile_family.as_str(), + "rt3-classic-save-container-v1" + | "rt3-105-save-container-v1" + | "rt3-105-scenario-save-container-v1" + | "rt3-105-alt-save-container-v1" + ) { + return None; + } + + let metadata_offsets = find_u32_le_offsets(bytes, 0x000038a5); + let records_offsets = find_u32_le_offsets(bytes, 0x000038a6); + let close_offsets = find_u32_le_offsets(bytes, 0x000038a7); + for metadata_tag_offset in metadata_offsets { + let Some(records_tag_offset) = records_offsets + .iter() + .copied() + .find(|offset| *offset > metadata_tag_offset) else { + continue; + }; + let Some(close_tag_offset) = close_offsets + .iter() + .copied() + .find(|offset| *offset > records_tag_offset) else { + continue; + }; + let Some(payload) = bytes.get(metadata_tag_offset + 4..records_tag_offset) else { + continue; + }; + if payload.len() < INDEXED_COLLECTION_SERIALIZED_HEADER_LEN { + continue; + } + let Some(header_words) = (0..INDEXED_COLLECTION_SERIALIZED_HEADER_DWORD_COUNT) + .map(|index| read_u32_at(payload, index * 4)) + .collect::>>() else { + continue; + }; + let Some(header_words): Option<[u32; INDEXED_COLLECTION_SERIALIZED_HEADER_DWORD_COUNT]> = + header_words.try_into().ok() + else { + continue; + }; + let summary = IndexedCollectionHeaderSummary { + metadata_tag_offset, + records_tag_offset, + close_tag_offset, + direct_collection_flag: header_words[0], + direct_record_stride: header_words[1], + live_id_bound: header_words[4], + live_record_count: header_words[5], + header_words, + }; + if !(summary.direct_collection_flag == 0 + && summary.direct_record_stride == 0x06 + && summary.header_words.get(2) == Some(&1000) + && summary.header_words.get(3) == Some(&500) + && summary.header_words.get(6) == Some(&0) + && summary.header_words.get(7) == Some(&1) + && summary.live_id_bound >= 0x100 + && summary.live_id_bound <= 0x1000 + && summary.live_record_count >= 0x100 + && summary.live_record_count <= summary.live_id_bound) + { + continue; + } + let Some(records_payload) = bytes.get(records_tag_offset + 4..close_tag_offset) else { + continue; + }; + let embedded_name_tag_offsets = + find_u16_le_offsets(records_payload, SAVE_REGION_RECORD_NAME_TAG); + let Some(&first_embedded_name_tag_relative_offset) = embedded_name_tag_offsets.first() else { + continue; + }; + let Some(prefix_payload) = records_payload.get(..first_embedded_name_tag_relative_offset) else { + continue; + }; + if prefix_payload.len() < 7 { + continue; + } + let Some(prefix_leading_dword) = read_u32_at(prefix_payload, 0) else { + continue; + }; + let Some(prefix_trailing_word) = read_u16_at(prefix_payload, 4) else { + continue; + }; + let Some(prefix_separator_byte) = prefix_payload.get(6).copied() else { + continue; + }; + let mut parsed_embedded_names = None; + for relative_name_offset in [4usize, 6usize] { + let Some(name_payload) = records_payload + .get(first_embedded_name_tag_relative_offset + relative_name_offset..) else { + continue; + }; + if let Some(names) = parse_save_len_prefixed_ascii_name_pair(name_payload) { + parsed_embedded_names = Some(names); + break; + } + } + let Some((first_embedded_primary_name, first_embedded_secondary_name)) = + parsed_embedded_names + else { + continue; + }; + return Some(SmpSavePlacedStructureDynamicSideBufferProbe { + profile_family: profile.profile_family.clone(), + source_kind: "save-placed-structure-dynamic-side-buffer-records".to_string(), + semantic_family: "scenario-save-placed-structure-dynamic-side-buffer-records".to_string(), + metadata_tag_offset, + records_tag_offset, + close_tag_offset, + records_span_len: close_tag_offset.saturating_sub(records_tag_offset + 4), + direct_record_stride: summary.direct_record_stride, + direct_record_stride_hex: format!("0x{:08x}", summary.direct_record_stride), + live_id_bound: summary.live_id_bound, + live_id_bound_hex: format!("0x{:08x}", summary.live_id_bound), + live_record_count: summary.live_record_count, + live_record_count_hex: format!("0x{:08x}", summary.live_record_count), + prefix_leading_dword, + prefix_leading_dword_hex: format!("0x{prefix_leading_dword:08x}"), + prefix_trailing_word, + prefix_trailing_word_hex: format!("0x{prefix_trailing_word:04x}"), + prefix_separator_byte, + prefix_separator_byte_hex: format!("0x{prefix_separator_byte:02x}"), + first_embedded_name_tag_relative_offset, + embedded_name_tag_count: embedded_name_tag_offsets.len(), + first_embedded_primary_name: Some(first_embedded_primary_name.clone()), + first_embedded_secondary_name: Some(first_embedded_secondary_name.clone()), + evidence: vec![ + "exact little-endian u32 tag family 0x38a5/0x38a6/0x38a7 appears as a separate save-side tagged collection on grounded saves".to_string(), + "records payload begins with a compact 6-byte prefix plus one separator byte before the first embedded 0x55f1 name row".to_string(), + "first embedded 0x55f1 row decodes with placed-structure-style dual names, which makes this the strongest current candidate for the separate placed-structure dynamic side-buffer owner seam".to_string(), + format!( + "grounded first embedded names are {:?}/{:?} with {} embedded 0x55f1 name rows in the tagged records span", + Some(first_embedded_primary_name), + Some(first_embedded_secondary_name), + embedded_name_tag_offsets.len() + ), + ], + }); + } + None +} + #[derive(Clone, Copy)] struct IndexedCollectionHeaderSummary { metadata_tag_offset: usize, @@ -10552,111 +10841,121 @@ fn scan_save_unclassified_tagged_collection_header_probes( 0x000036b1, EVENT_RUNTIME_COLLECTION_METADATA_TAG as u32, ]); + let mut low_tag_offsets: BTreeMap> = BTreeMap::new(); + for offset in 0..bytes.len().saturating_sub(4) { + let Some(tag) = read_u32_at(bytes, offset) else { + continue; + }; + if (3..=0xffff).contains(&tag) { + low_tag_offsets.entry(tag).or_default().push(offset); + } + } let mut probes = Vec::new(); - for metadata_tag_offset in 0..bytes.len().saturating_sub(INDEXED_COLLECTION_SERIALIZED_HEADER_LEN + 4) - { - let Some(metadata_tag) = read_u32_at(bytes, metadata_tag_offset) else { + for (&metadata_tag, metadata_offsets) in &low_tag_offsets { + if known_metadata_tags.contains(&metadata_tag) { + continue; + } + let Some(records_offsets) = low_tag_offsets.get(&(metadata_tag + 1)) else { continue; }; - if metadata_tag > 0xffff || known_metadata_tags.contains(&metadata_tag) { + let Some(close_offsets) = low_tag_offsets.get(&(metadata_tag + 2)) else { continue; - } - let mut header_words = [0u32; INDEXED_COLLECTION_SERIALIZED_HEADER_DWORD_COUNT]; - let mut valid_header = true; - for (index, word) in header_words.iter_mut().enumerate() { - let Some(value) = read_u32_at(bytes, metadata_tag_offset + 4 + index * 4) else { - valid_header = false; - break; - }; - *word = value; - } - if !valid_header { - continue; - } - let summary = IndexedCollectionHeaderSummary { - metadata_tag_offset, - records_tag_offset: 0, - close_tag_offset: 0, - direct_collection_flag: header_words[0], - direct_record_stride: header_words[1], - live_id_bound: header_words[4], - live_record_count: header_words[5], - header_words, }; - if !matches!(summary.direct_collection_flag, 0 | 1) - || summary.direct_record_stride == 0 - || summary.direct_record_stride > 0x4000 - || summary.live_id_bound == 0 - || summary.live_record_count == 0 - || summary.live_record_count > summary.live_id_bound - || summary.live_id_bound > 0x100000 - { - continue; - } let records_tag = metadata_tag + 1; let close_tag = metadata_tag + 2; - let records_search_start = metadata_tag_offset + 4; - let Some(records_relative_offset) = - find_u32_le_offsets(&bytes[records_search_start..], records_tag) - .into_iter() - .next() - else { - continue; - }; - let records_tag_offset = records_search_start + records_relative_offset; - let close_search_start = records_tag_offset + 4; - let Some(close_relative_offset) = - find_u32_le_offsets(&bytes[close_search_start..], close_tag) - .into_iter() - .next() - else { - continue; - }; - let close_tag_offset = close_search_start + close_relative_offset; - let records_span_len = close_tag_offset.saturating_sub(records_tag_offset + 4); - if records_span_len == 0 { - continue; - } - if probes.iter().any(|probe: &SmpSaveUnclassifiedTaggedCollectionHeaderProbe| { - probe.metadata_tag_offset == metadata_tag_offset - && probe.records_tag_offset == records_tag_offset - && probe.close_tag_offset == close_tag_offset - }) { - continue; - } - probes.push(SmpSaveUnclassifiedTaggedCollectionHeaderProbe { - profile_family: profile.profile_family.clone(), - source_kind: "save-unclassified-tagged-header-counts".to_string(), - semantic_family: "scenario-save-unclassified-tagged-header-counts".to_string(), - metadata_tag, - metadata_tag_hex: format!("0x{metadata_tag:08x}"), - records_tag, - records_tag_hex: format!("0x{records_tag:08x}"), - close_tag, - close_tag_hex: format!("0x{close_tag:08x}"), - metadata_tag_offset, - records_tag_offset, - close_tag_offset, - records_span_len, - direct_collection_flag: summary.direct_collection_flag, - direct_collection_flag_hex: format!("0x{:08x}", summary.direct_collection_flag), - direct_record_stride: summary.direct_record_stride, - direct_record_stride_hex: format!("0x{:08x}", summary.direct_record_stride), - live_id_bound: summary.live_id_bound, - live_id_bound_hex: format!("0x{:08x}", summary.live_id_bound), - live_record_count: summary.live_record_count, - live_record_count_hex: format!("0x{:08x}", summary.live_record_count), - header_words: summary.header_words.to_vec(), - header_hex_words: summary - .header_words + for &metadata_tag_offset in metadata_offsets { + let mut header_words = [0u32; INDEXED_COLLECTION_SERIALIZED_HEADER_DWORD_COUNT]; + let mut valid_header = true; + for (index, word) in header_words.iter_mut().enumerate() { + let Some(value) = read_u32_at(bytes, metadata_tag_offset + 4 + index * 4) else { + valid_header = false; + break; + }; + *word = value; + } + if !valid_header { + continue; + } + let summary = IndexedCollectionHeaderSummary { + metadata_tag_offset, + records_tag_offset: 0, + close_tag_offset: 0, + direct_collection_flag: header_words[0], + direct_record_stride: header_words[1], + live_id_bound: header_words[4], + live_record_count: header_words[5], + header_words, + }; + if !matches!(summary.direct_collection_flag, 0 | 1) + || summary.direct_record_stride == 0 + || summary.direct_record_stride > 0x2000 + || summary.live_id_bound == 0 + || summary.live_record_count == 0 + || summary.live_record_count > summary.live_id_bound + || summary.live_id_bound > 0x1000 + || summary.live_record_count > 0x1000 + { + continue; + } + let records_search_start = metadata_tag_offset + 4; + let records_index = + records_offsets.partition_point(|offset| *offset < records_search_start); + let Some(&records_tag_offset) = records_offsets.get(records_index) else { + continue; + }; + let close_search_start = records_tag_offset + 4; + let close_index = close_offsets.partition_point(|offset| *offset < close_search_start); + let Some(&close_tag_offset) = close_offsets.get(close_index) else { + continue; + }; + let records_span_len = close_tag_offset.saturating_sub(records_tag_offset + 4); + if records_span_len == 0 || records_span_len < summary.live_record_count as usize { + continue; + } + if probes .iter() - .map(|word| format!("0x{word:08x}")) - .collect(), - evidence: vec![ - "generic save-side tagged collection scan over plausible low u32 metadata tags not yet claimed by the checked-in collection probes".to_string(), - "candidate uses adjacent metadata/records/close tags with a header that matches the grounded indexed-collection shape (flag, stride, live_id_bound, live_record_count)".to_string(), - ], - }); + .any(|probe: &SmpSaveUnclassifiedTaggedCollectionHeaderProbe| { + probe.metadata_tag_offset == metadata_tag_offset + && probe.records_tag_offset == records_tag_offset + && probe.close_tag_offset == close_tag_offset + }) + { + continue; + } + probes.push(SmpSaveUnclassifiedTaggedCollectionHeaderProbe { + profile_family: profile.profile_family.clone(), + source_kind: "save-unclassified-tagged-header-counts".to_string(), + semantic_family: "scenario-save-unclassified-tagged-header-counts".to_string(), + metadata_tag, + metadata_tag_hex: format!("0x{metadata_tag:08x}"), + records_tag, + records_tag_hex: format!("0x{records_tag:08x}"), + close_tag, + close_tag_hex: format!("0x{close_tag:08x}"), + metadata_tag_offset, + records_tag_offset, + close_tag_offset, + records_span_len, + direct_collection_flag: summary.direct_collection_flag, + direct_collection_flag_hex: format!("0x{:08x}", summary.direct_collection_flag), + direct_record_stride: summary.direct_record_stride, + direct_record_stride_hex: format!("0x{:08x}", summary.direct_record_stride), + live_id_bound: summary.live_id_bound, + live_id_bound_hex: format!("0x{:08x}", summary.live_id_bound), + live_record_count: summary.live_record_count, + live_record_count_hex: format!("0x{:08x}", summary.live_record_count), + header_words: summary.header_words.to_vec(), + header_hex_words: summary + .header_words + .iter() + .map(|word| format!("0x{word:08x}")) + .collect(), + evidence: vec![ + "generic save-side tagged collection scan over plausible low u32 metadata tags not yet claimed by the checked-in collection probes".to_string(), + "candidate uses adjacent metadata/records/close tags with a header that matches the grounded indexed-collection shape (flag, stride, live_id_bound, live_record_count)".to_string(), + ], + }); + } } probes.sort_by(|left, right| { right @@ -10669,6 +10968,24 @@ fn scan_save_unclassified_tagged_collection_header_probes( probes } +fn filter_unclassified_tagged_collection_header_probes_outside_known_spans( + probes: Vec, + known_header_probes: &[Option<&SmpSaveTaggedCollectionHeaderProbe>], +) -> Vec { + probes + .into_iter() + .filter(|probe| { + !known_header_probes + .iter() + .flatten() + .any(|known| { + probe.metadata_tag_offset >= known.metadata_tag_offset + && probe.close_tag_offset <= known.close_tag_offset + }) + }) + .collect() +} + fn parse_save_len_prefixed_ascii_name(bytes: &[u8]) -> Option { let len = *bytes.first()? as usize; let text_bytes = bytes.get(1..1 + len)?; @@ -18259,6 +18576,71 @@ mod tests { ); } + #[test] + fn parses_placed_structure_dynamic_side_buffer_probe_from_embedded_name_row() { + let mut bytes = vec![0u8; 0x400]; + let metadata_tag_offset = 0x40usize; + let records_tag_offset = 0x140usize; + let close_tag_offset = 0x220usize; + bytes[metadata_tag_offset..metadata_tag_offset + 4] + .copy_from_slice(&0x000038a5u32.to_le_bytes()); + bytes[records_tag_offset..records_tag_offset + 4] + .copy_from_slice(&0x000038a6u32.to_le_bytes()); + bytes[close_tag_offset..close_tag_offset + 4] + .copy_from_slice(&0x000038a7u32.to_le_bytes()); + let header_words = [ + 0u32, 0x06, 1000, 500, 1000, 388, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + ]; + for (index, word) in header_words.into_iter().enumerate() { + let offset = metadata_tag_offset + 4 + index * 4; + bytes[offset..offset + 4].copy_from_slice(&word.to_le_bytes()); + } + let payload_offset = records_tag_offset + 4; + bytes[payload_offset..payload_offset + 4].copy_from_slice(&0x0005d368u32.to_le_bytes()); + bytes[payload_offset + 4..payload_offset + 6].copy_from_slice(&0x0001u16.to_le_bytes()); + bytes[payload_offset + 6] = 0xff; + let name_tag_offset = payload_offset + 7; + bytes[name_tag_offset..name_tag_offset + 2] + .copy_from_slice(&SAVE_REGION_RECORD_NAME_TAG.to_le_bytes()); + let first_name = "TrackCapST_Cap.3dp"; + let second_name = "Infrastructure"; + bytes[name_tag_offset + 4] = first_name.len() as u8; + bytes[name_tag_offset + 5..name_tag_offset + 5 + first_name.len()] + .copy_from_slice(first_name.as_bytes()); + let second_len_offset = name_tag_offset + 5 + first_name.len(); + bytes[second_len_offset] = second_name.len() as u8; + bytes[second_len_offset + 1..second_len_offset + 1 + second_name.len()] + .copy_from_slice(second_name.as_bytes()); + + let probe = parse_save_placed_structure_dynamic_side_buffer_probe( + &bytes, + Some("gms"), + Some(&SmpContainerProfile { + profile_family: "rt3-105-save-container-v1".to_string(), + profile_evidence: vec![], + is_known_profile: true, + }), + ) + .expect("placed-structure dynamic side-buffer probe should parse"); + + assert_eq!(probe.direct_record_stride, 0x06); + assert_eq!(probe.live_id_bound, 1000); + assert_eq!(probe.live_record_count, 388); + assert_eq!(probe.prefix_leading_dword_hex, "0x0005d368"); + assert_eq!(probe.prefix_trailing_word_hex, "0x0001"); + assert_eq!(probe.prefix_separator_byte_hex, "0xff"); + assert_eq!(probe.first_embedded_name_tag_relative_offset, 7); + assert_eq!(probe.embedded_name_tag_count, 1); + assert_eq!( + probe.first_embedded_primary_name.as_deref(), + Some("TrackCapST_Cap.3dp") + ); + assert_eq!( + probe.first_embedded_secondary_name.as_deref(), + Some("Infrastructure") + ); + } + #[test] fn parses_placed_structure_tagged_collection_header_probe_from_exact_u32_tags() { let mut bytes = vec![0u8; 0x400]; @@ -18308,10 +18690,9 @@ mod tests { .copy_from_slice(&0x00007001u32.to_le_bytes()); bytes[records_tag_offset..records_tag_offset + 4] .copy_from_slice(&0x00007002u32.to_le_bytes()); - bytes[close_tag_offset..close_tag_offset + 4] - .copy_from_slice(&0x00007003u32.to_le_bytes()); + bytes[close_tag_offset..close_tag_offset + 4].copy_from_slice(&0x00007003u32.to_le_bytes()); let header_words = [ - 0u32, 0x12, 0x0a, 0x14, 0x900, 0x808, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0u32, 0x12, 0x0a, 0x14, 0x90, 0x78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ]; for (index, word) in header_words.into_iter().enumerate() { let offset = metadata_tag_offset + 4 + index * 4; @@ -18335,9 +18716,12 @@ mod tests { assert_eq!(probe.records_tag, 0x7002); assert_eq!(probe.close_tag, 0x7003); assert_eq!(probe.direct_record_stride, 0x12); - assert_eq!(probe.live_id_bound, 0x900); - assert_eq!(probe.live_record_count, 0x808); - assert_eq!(probe.records_span_len, close_tag_offset - (records_tag_offset + 4)); + assert_eq!(probe.live_id_bound, 0x90); + assert_eq!(probe.live_record_count, 0x78); + assert_eq!( + probe.records_span_len, + close_tag_offset - (records_tag_offset + 4) + ); } #[test] diff --git a/docs/rehost-queue.md b/docs/rehost-queue.md index de02b3e..d39502e 100644 --- a/docs/rehost-queue.md +++ b/docs/rehost-queue.md @@ -21,8 +21,10 @@ Working rule: `0x36b1/0x36b2/0x36b3` header seam so the blocked city-connection / linked-transit branch can stop depending on atlas-only placed-structure and local-runtime refresh notes, especially the semantics of the now-grounded compact `0x55f3` footer dword/status lane and the newly exposed - unclassified tagged-collection candidates that may correspond to the separate placed-structure - dynamic side-buffer lane. + separate tagged side-buffer seam candidates, especially the exact `0x38a5/0x38a6/0x38a7` + family whose compact `6`-byte header pattern and embedded placed-structure-style `0x55f1` + name rows now make it the strongest current candidate for the separate placed-structure dynamic + side-buffer owner. - Extend shellless clock advancement so more periodic-company service branches consume owned runtime time state directly instead of only the explicit periodic service command. - Keep widening selected-year world-owner state only when a full owning reader/rebuild family is @@ -78,8 +80,16 @@ Working rule: from “find the hidden tail inside this payload” to “find the separate owner seam that backs the runtime latches the city-connection branch still reads.” - Save inspection now also exports a generic low-tag unclassified collection scan over plausible - indexed-collection headers, so the next city-connection pass can compare real save candidates - against the atlas-owned placed-structure dynamic side-buffer lane instead of blind tag hunting. + indexed-collection headers, now through a lightweight CLI path that does not require full bundle + inspection and now filters out candidates nested inside already-grounded company/chairman/train/ + region/placed-structure spans. +- That lightweight scan now also narrows the real save frontier to a much smaller stable candidate + set across `p.gms`, `q.gms`, and `Autosave.gms`, with the exact `0x38a5/0x38a6/0x38a7` family + standing out as the strongest current placed-structure dynamic side-buffer candidate. +- The `0x38a5/0x38a6/0x38a7` family now also has a first dedicated parser scaffold in + `rrt-runtime`: its synthetic regression is grounded, its header shape is checked in, and the + parser now expects a compact 6-byte prefix plus separator byte before an embedded + placed-structure-style dual-name row rather than treating the family as anonymous residue. - The placed-structure tagged save stream now also exposes repeated `0x55f1/0x55f2/0x55f3` triplets with dual name stems, a fixed five-`f32` policy row, and a compact `0x5dc1...0x5dc2` footer carrying one raw `u32` payload lane plus one live `i32` status lane, so the remaining