diff --git a/crates/rrt-cli/src/main.rs b/crates/rrt-cli/src/main.rs index 0f8c1a9..f0e993f 100644 --- a/crates/rrt-cli/src/main.rs +++ b/crates/rrt-cli/src/main.rs @@ -24,11 +24,11 @@ use rrt_runtime::{ RuntimeSaveSliceDocument, RuntimeSaveSliceDocumentSource, RuntimeSnapshotDocument, RuntimeSnapshotSource, RuntimeSummary, SAVE_SLICE_DOCUMENT_FORMAT_VERSION, SNAPSHOT_FORMAT_VERSION, SmpClassicPackedProfileBlock, SmpInspectionReport, SmpLoadedSaveSlice, - SmpRt3105PackedProfileBlock, SmpSaveLoadSummary, WinInspectionReport, execute_step_command, - extract_pk4_entry_file, inspect_campaign_exe_file, inspect_cargo_economy_sources_with_bindings, - inspect_cargo_skin_pk4, inspect_cargo_types_dir, inspect_pk4_file, - inspect_save_company_and_chairman_analysis_file, inspect_save_infrastructure_asset_trace_file, - inspect_save_periodic_company_service_trace_file, + SmpRt3105PackedProfileBlock, SmpSaveLoadSummary, WinInspectionReport, + compare_save_region_fixed_row_run_candidates, execute_step_command, extract_pk4_entry_file, + inspect_campaign_exe_file, inspect_cargo_economy_sources_with_bindings, inspect_cargo_skin_pk4, + inspect_cargo_types_dir, inspect_pk4_file, inspect_save_company_and_chairman_analysis_file, + inspect_save_infrastructure_asset_trace_file, inspect_save_periodic_company_service_trace_file, inspect_save_placed_structure_dynamic_side_buffer_file, inspect_save_region_queued_notice_records_file, inspect_save_region_service_trace_file, inspect_smp_file, inspect_unclassified_save_collection_headers_file, inspect_win_file, @@ -134,6 +134,10 @@ enum Command { RuntimeInspectSaveCompanyChairman { smp_path: PathBuf, }, + RuntimeCompareRegionFixedRowRuns { + left_path: PathBuf, + right_path: PathBuf, + }, RuntimeInspectPeriodicCompanyServiceTrace { smp_path: PathBuf, }, @@ -304,6 +308,13 @@ struct RuntimeSaveCompanyChairmanAnalysisOutput { analysis: rrt_runtime::SmpSaveCompanyChairmanAnalysisReport, } +#[derive(Debug, Serialize)] +struct RuntimeRegionFixedRowRunComparisonOutput { + left_path: String, + right_path: String, + comparison: rrt_runtime::SmpSaveRegionFixedRowRunComparisonReport, +} + #[derive(Debug, Serialize)] struct RuntimePeriodicCompanyServiceTraceOutput { path: String, @@ -893,6 +904,12 @@ fn real_main() -> Result<(), Box> { Command::RuntimeInspectSaveCompanyChairman { smp_path } => { run_runtime_inspect_save_company_chairman(&smp_path)?; } + Command::RuntimeCompareRegionFixedRowRuns { + left_path, + right_path, + } => { + run_runtime_compare_region_fixed_row_runs(&left_path, &right_path)?; + } Command::RuntimeInspectPeriodicCompanyServiceTrace { smp_path } => { run_runtime_inspect_periodic_company_service_trace(&smp_path)?; } @@ -1114,6 +1131,14 @@ fn parse_command() -> Result> { smp_path: PathBuf::from(path), }) } + [command, subcommand, left_path, right_path] + if command == "runtime" && subcommand == "compare-region-fixed-row-runs" => + { + Ok(Command::RuntimeCompareRegionFixedRowRuns { + left_path: PathBuf::from(left_path), + right_path: PathBuf::from(right_path), + }) + } [command, subcommand, path] if command == "runtime" && subcommand == "inspect-periodic-company-service-trace" => @@ -1362,7 +1387,7 @@ fn parse_command() -> Result> { }) } _ => Err( - "usage: rrt-cli [validate [repo-root] | finance eval | finance diff | runtime validate-fixture | runtime summarize-fixture | runtime export-fixture-state | runtime diff-state | runtime summarize-state | runtime import-state | runtime inspect-smp | runtime summarize-save-load | runtime load-save-slice | runtime inspect-save-company-chairman | runtime inspect-periodic-company-service-trace | runtime inspect-region-service-trace | runtime inspect-infrastructure-asset-trace | runtime inspect-save-region-queued-notice-records | runtime inspect-placed-structure-dynamic-side-buffer | runtime inspect-unclassified-save-collections | runtime import-save-state | runtime export-save-slice | runtime export-overlay-import | runtime inspect-pk4 | runtime inspect-cargo-types | runtime inspect-cargo-skins | runtime inspect-cargo-economy-sources | runtime inspect-cargo-production-selector | runtime inspect-cargo-price-selector | runtime inspect-win | runtime extract-pk4-entry | runtime inspect-campaign-exe | runtime compare-classic-profile [saveN.gms...] | runtime compare-105-profile [saveN.gms...] | runtime compare-candidate-table [fileN...] | runtime compare-recipe-book-lines [fileN...] | runtime compare-setup-payload-core [fileN...] | runtime compare-setup-launch-payload [fileN...] | runtime compare-post-special-conditions-scalars [fileN...] | runtime scan-candidate-table-headers | runtime scan-special-conditions | runtime scan-aligned-runtime-rule-band | runtime scan-post-special-conditions-scalars | runtime scan-post-special-conditions-tail | runtime scan-recipe-book-lines | runtime export-profile-block ]" + "usage: rrt-cli [validate [repo-root] | finance eval | finance diff | runtime validate-fixture | runtime summarize-fixture | runtime export-fixture-state | runtime diff-state | runtime summarize-state | runtime import-state | runtime inspect-smp | runtime summarize-save-load | runtime load-save-slice | runtime inspect-save-company-chairman | runtime compare-region-fixed-row-runs | runtime inspect-periodic-company-service-trace | runtime inspect-region-service-trace | runtime inspect-infrastructure-asset-trace | runtime inspect-save-region-queued-notice-records | runtime inspect-placed-structure-dynamic-side-buffer | runtime inspect-unclassified-save-collections | runtime import-save-state | runtime export-save-slice | runtime export-overlay-import | runtime inspect-pk4 | runtime inspect-cargo-types | runtime inspect-cargo-skins | runtime inspect-cargo-economy-sources | runtime inspect-cargo-production-selector | runtime inspect-cargo-price-selector | runtime inspect-win | runtime extract-pk4-entry | runtime inspect-campaign-exe | runtime compare-classic-profile [saveN.gms...] | runtime compare-105-profile [saveN.gms...] | runtime compare-candidate-table [fileN...] | runtime compare-recipe-book-lines [fileN...] | runtime compare-setup-payload-core [fileN...] | runtime compare-setup-launch-payload [fileN...] | runtime compare-post-special-conditions-scalars [fileN...] | runtime scan-candidate-table-headers | runtime scan-special-conditions | runtime scan-aligned-runtime-rule-band | runtime scan-post-special-conditions-scalars | runtime scan-post-special-conditions-tail | runtime scan-recipe-book-lines | runtime export-profile-block ]" .into(), ), } @@ -1603,6 +1628,23 @@ fn run_runtime_inspect_save_company_chairman( Ok(()) } +fn run_runtime_compare_region_fixed_row_runs( + left_path: &Path, + right_path: &Path, +) -> Result<(), Box> { + let left = inspect_save_company_and_chairman_analysis_file(left_path)?; + let right = inspect_save_company_and_chairman_analysis_file(right_path)?; + let comparison = compare_save_region_fixed_row_run_candidates(&left, &right) + .ok_or("save inspection did not expose grounded region fixed-row candidate probes")?; + let report = RuntimeRegionFixedRowRunComparisonOutput { + left_path: left_path.display().to_string(), + right_path: right_path.display().to_string(), + comparison, + }; + println!("{}", serde_json::to_string_pretty(&report)?); + Ok(()) +} + fn run_runtime_inspect_periodic_company_service_trace( smp_path: &Path, ) -> Result<(), Box> { diff --git a/crates/rrt-runtime/src/lib.rs b/crates/rrt-runtime/src/lib.rs index fd29ea5..bf0b1ff 100644 --- a/crates/rrt-runtime/src/lib.rs +++ b/crates/rrt-runtime/src/lib.rs @@ -122,13 +122,14 @@ pub use smp::{ SmpSaveLoadSummary, SmpSavePlacedStructureDynamicSideBufferAlignmentProbe, SmpSavePlacedStructureDynamicSideBufferNamePairSummary, SmpSavePlacedStructureDynamicSideBufferPrefixPatternSummary, - SmpSavePlacedStructureDynamicSideBufferProbe, SmpSaveRegionQueuedNoticeRecordProbe, - SmpSaveScalarCandidate, SmpSaveTaggedCollectionHeaderProbe, SmpSaveWorldEconomicTuningProbe, + SmpSavePlacedStructureDynamicSideBufferProbe, SmpSaveRegionFixedRowRunComparisonReport, + SmpSaveRegionQueuedNoticeRecordProbe, SmpSaveScalarCandidate, + SmpSaveTaggedCollectionHeaderProbe, SmpSaveWorldEconomicTuningProbe, SmpSaveWorldFinanceNeighborhoodProbe, SmpSaveWorldIssue37Probe, SmpSaveWorldSelectionRoleAnalysis, SmpSaveWorldSelectionRoleAnalysisEntry, SmpSecondaryVariantProbe, SmpServiceTraceBranchStatus, SmpSharedHeader, SmpSpecialConditionEntry, SmpSpecialConditionsProbe, - inspect_save_company_and_chairman_analysis_bytes, + compare_save_region_fixed_row_run_candidates, inspect_save_company_and_chairman_analysis_bytes, inspect_save_company_and_chairman_analysis_file, inspect_save_infrastructure_asset_trace_file, inspect_save_periodic_company_service_trace_file, inspect_save_placed_structure_dynamic_side_buffer_file, diff --git a/crates/rrt-runtime/src/smp.rs b/crates/rrt-runtime/src/smp.rs index 05e28f4..ac982e5 100644 --- a/crates/rrt-runtime/src/smp.rs +++ b/crates/rrt-runtime/src/smp.rs @@ -1,3 +1,4 @@ +use std::cmp::Reverse; use std::collections::{BTreeMap, BTreeSet}; use std::fs; use std::path::Path; @@ -155,6 +156,9 @@ const SAVE_REGION_COLLECTION_DIRECTORY_ENTRY_DWORD_COUNT: usize = 3; const SAVE_REGION_RECORD_NAME_TAG: u16 = 0x55f1; const SAVE_REGION_RECORD_POLICY_TAG: u16 = 0x55f2; const SAVE_REGION_RECORD_PROFILE_TAG: u16 = 0x55f3; +const SAVE_REGION_FIXED_ROW_STRIDE: usize = 0x29; +const SAVE_REGION_FIXED_ROW_DWORD_LANE_COUNT: usize = SAVE_REGION_FIXED_ROW_STRIDE / 4; +const SAVE_REGION_FIXED_ROW_CANDIDATE_PROBE_LIMIT: usize = 24; const SAVE_REGION_QUEUED_NOTICE_PAYLOAD_SEED: u32 = 0x005c87a8; const SAVE_REGION_QUEUED_NOTICE_NODE_KIND: u32 = 7; const SAVE_REGION_QUEUED_NOTICE_NODE_LEN: usize = 0x20; @@ -1775,6 +1779,98 @@ pub struct SmpSaveRegionQueuedNoticeRecordProbe { pub evidence: Vec, } +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct SmpSaveFixedRowRunDwordLaneSummary { + pub relative_offset: usize, + pub relative_offset_hex: String, + pub zero_count: usize, + pub nonzero_count: usize, + pub distinct_value_count: usize, + pub probable_normal_f32_count: usize, + pub small_unsigned_count: usize, + #[serde(default)] + pub sample_values_hex: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct SmpSaveRegionFixedRowRunCandidate { + pub count_offset: usize, + pub count_offset_hex: String, + pub row_count: usize, + pub row_stride: usize, + pub row_stride_hex: String, + pub rows_offset: usize, + pub rows_offset_hex: String, + pub rows_end_offset: usize, + pub rows_end_offset_hex: String, + pub distance_to_region_metadata_tag: usize, + pub distance_to_region_metadata_tag_hex: String, + #[serde(default)] + pub dword_lane_summaries: Vec, + pub shape_signature: String, + pub shape_family_signature: String, + pub trailing_byte_zero_count: usize, + pub trailing_byte_nonzero_count: usize, + pub trailing_byte_distinct_value_count: usize, + #[serde(default)] + pub trailing_byte_sample_values_hex: Vec, + #[serde(default)] + pub best_probable_density_lane_relative_offset_hex: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct SmpSaveRegionFixedRowRunCandidateProbe { + pub profile_family: String, + pub source_kind: String, + pub semantic_family: String, + pub target_row_count: usize, + pub target_row_stride: usize, + pub target_row_stride_hex: String, + pub scan_start_offset: usize, + pub scan_start_offset_hex: String, + pub scan_end_offset: usize, + pub scan_end_offset_hex: String, + #[serde(default)] + pub candidates: Vec, + pub evidence: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct SmpSaveRegionFixedRowRunSharedShapeMatch { + pub shape_signature: String, + pub left_rank: usize, + pub left_rows_offset_hex: String, + pub left_best_probable_density_lane_relative_offset_hex: Option, + pub right_rank: usize, + pub right_rows_offset_hex: String, + pub right_best_probable_density_lane_relative_offset_hex: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct SmpSaveRegionFixedRowRunComparisonReport { + pub left_profile_family: String, + pub right_profile_family: String, + pub left_best_rows_offset_hex: Option, + pub right_best_rows_offset_hex: Option, + pub left_best_shape_signature: Option, + pub right_best_shape_signature: Option, + pub left_best_shape_family_signature: Option, + pub right_best_shape_family_signature: Option, + #[serde(default)] + pub shared_shape_matches: Vec, + #[serde(default)] + pub shared_shape_family_matches: Vec, + #[serde(default)] + pub left_only_shape_signatures: Vec, + #[serde(default)] + pub right_only_shape_signatures: Vec, + #[serde(default)] + pub left_only_shape_family_signatures: Vec, + #[serde(default)] + pub right_only_shape_family_signatures: Vec, + pub evidence: Vec, +} + #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct SmpSavePlacedStructureRecordTripletEntryProbe { pub record_index: usize, @@ -3411,6 +3507,8 @@ pub struct SmpSaveCompanyChairmanAnalysisReport { #[serde(default)] pub region_queued_notice_records: Option, #[serde(default)] + pub region_fixed_row_run_candidates: Option, + #[serde(default)] pub placed_structure_collection_header: Option, #[serde(default)] pub placed_structure_record_triplets: Option, @@ -3856,6 +3954,8 @@ pub struct SmpInspectionReport { pub save_region_record_triplet_probe: Option, #[serde(default)] pub save_region_queued_notice_record_probe: Option, + #[serde(default)] + pub save_region_fixed_row_run_candidate_probe: Option, pub save_placed_structure_collection_header_probe: Option, pub save_placed_structure_record_triplet_probe: Option, @@ -6302,6 +6402,17 @@ pub fn inspect_save_company_and_chairman_analysis_bytes( report.save_region_collection_header_probe.as_ref(), ) }); + let region_fixed_row_run_candidates = report + .save_region_fixed_row_run_candidate_probe + .clone() + .or_else(|| { + parse_save_region_fixed_row_run_candidate_probe( + bytes, + report.file_extension_hint.as_deref(), + report.container_profile.as_ref(), + report.save_region_collection_header_probe.as_ref(), + ) + }); let placed_structure_record_triplets = report.save_placed_structure_record_triplet_probe.clone(); let placed_structure_dynamic_side_buffer = report @@ -6673,6 +6784,22 @@ pub fn inspect_save_company_and_chairman_analysis_bytes( queue_probe.entries[0].trailing_sentinel_i32_1_hex )); } + if let Some(fixed_row_candidates) = region_fixed_row_run_candidates.as_ref() { + notes.push(format!( + "Region analysis now also exports {} fixed-row run candidates keyed to live_record_count={} and stride {} before the tagged region metadata; best candidate rows offset is {:?} with shape signature {:?}.", + fixed_row_candidates.candidates.len(), + fixed_row_candidates.target_row_count, + fixed_row_candidates.target_row_stride_hex, + fixed_row_candidates + .candidates + .first() + .map(|candidate| candidate.rows_offset_hex.as_str()), + fixed_row_candidates + .candidates + .first() + .map(|candidate| candidate.shape_signature.as_str()) + )); + } if let Some(header) = report .save_placed_structure_collection_header_probe .as_ref() @@ -6812,6 +6939,7 @@ pub fn inspect_save_company_and_chairman_analysis_bytes( region_collection_header: report.save_region_collection_header_probe.clone(), region_record_triplets, region_queued_notice_records, + region_fixed_row_run_candidates, placed_structure_collection_header: report .save_placed_structure_collection_header_probe .clone(), @@ -6825,6 +6953,143 @@ pub fn inspect_save_company_and_chairman_analysis_bytes( }) } +pub fn compare_save_region_fixed_row_run_candidates( + left: &SmpSaveCompanyChairmanAnalysisReport, + right: &SmpSaveCompanyChairmanAnalysisReport, +) -> Option { + let left_probe = left.region_fixed_row_run_candidates.as_ref()?; + let right_probe = right.region_fixed_row_run_candidates.as_ref()?; + + let left_by_shape = left_probe + .candidates + .iter() + .enumerate() + .map(|(index, candidate)| (candidate.shape_signature.clone(), (index, candidate))) + .collect::>(); + let right_by_shape = right_probe + .candidates + .iter() + .enumerate() + .map(|(index, candidate)| (candidate.shape_signature.clone(), (index, candidate))) + .collect::>(); + + let mut shared_shape_matches = Vec::new(); + let mut shared_shape_family_matches = Vec::new(); + let mut left_only_shape_signatures = Vec::new(); + let mut right_only_shape_signatures = Vec::new(); + let mut left_only_shape_family_signatures = Vec::new(); + let mut right_only_shape_family_signatures = Vec::new(); + let left_family_by_shape = left_probe + .candidates + .iter() + .enumerate() + .map(|(index, candidate)| (candidate.shape_family_signature.clone(), (index, candidate))) + .collect::>(); + let right_family_by_shape = right_probe + .candidates + .iter() + .enumerate() + .map(|(index, candidate)| (candidate.shape_family_signature.clone(), (index, candidate))) + .collect::>(); + + for (shape_signature, (left_index, left_candidate)) in &left_by_shape { + if let Some((right_index, right_candidate)) = right_by_shape.get(shape_signature) { + shared_shape_matches.push(SmpSaveRegionFixedRowRunSharedShapeMatch { + shape_signature: shape_signature.clone(), + left_rank: left_index + 1, + left_rows_offset_hex: left_candidate.rows_offset_hex.clone(), + left_best_probable_density_lane_relative_offset_hex: left_candidate + .best_probable_density_lane_relative_offset_hex + .clone(), + right_rank: right_index + 1, + right_rows_offset_hex: right_candidate.rows_offset_hex.clone(), + right_best_probable_density_lane_relative_offset_hex: right_candidate + .best_probable_density_lane_relative_offset_hex + .clone(), + }); + } else { + left_only_shape_signatures.push(shape_signature.clone()); + } + } + + for shape_signature in right_by_shape.keys() { + if !left_by_shape.contains_key(shape_signature) { + right_only_shape_signatures.push(shape_signature.clone()); + } + } + + for (shape_family_signature, (left_index, left_candidate)) in &left_family_by_shape { + if let Some((right_index, right_candidate)) = + right_family_by_shape.get(shape_family_signature) + { + shared_shape_family_matches.push(SmpSaveRegionFixedRowRunSharedShapeMatch { + shape_signature: shape_family_signature.clone(), + left_rank: left_index + 1, + left_rows_offset_hex: left_candidate.rows_offset_hex.clone(), + left_best_probable_density_lane_relative_offset_hex: left_candidate + .best_probable_density_lane_relative_offset_hex + .clone(), + right_rank: right_index + 1, + right_rows_offset_hex: right_candidate.rows_offset_hex.clone(), + right_best_probable_density_lane_relative_offset_hex: right_candidate + .best_probable_density_lane_relative_offset_hex + .clone(), + }); + } else { + left_only_shape_family_signatures.push(shape_family_signature.clone()); + } + } + + for shape_family_signature in right_family_by_shape.keys() { + if !left_family_by_shape.contains_key(shape_family_signature) { + right_only_shape_family_signatures.push(shape_family_signature.clone()); + } + } + + Some(SmpSaveRegionFixedRowRunComparisonReport { + left_profile_family: left.profile_family.clone(), + right_profile_family: right.profile_family.clone(), + left_best_rows_offset_hex: left_probe + .candidates + .first() + .map(|candidate| candidate.rows_offset_hex.clone()), + right_best_rows_offset_hex: right_probe + .candidates + .first() + .map(|candidate| candidate.rows_offset_hex.clone()), + left_best_shape_signature: left_probe + .candidates + .first() + .map(|candidate| candidate.shape_signature.clone()), + right_best_shape_signature: right_probe + .candidates + .first() + .map(|candidate| candidate.shape_signature.clone()), + left_best_shape_family_signature: left_probe + .candidates + .first() + .map(|candidate| candidate.shape_family_signature.clone()), + right_best_shape_family_signature: right_probe + .candidates + .first() + .map(|candidate| candidate.shape_family_signature.clone()), + shared_shape_matches, + shared_shape_family_matches, + left_only_shape_signatures, + right_only_shape_signatures, + left_only_shape_family_signatures, + right_only_shape_family_signatures, + evidence: vec![ + format!( + "comparison keys the pre-region-header fixed-row candidates by derived lane-shape fingerprint instead of raw offset, because current grounded saves do not keep the same top rows_offset across files ({:?} vs {:?})", + left_probe.candidates.first().map(|candidate| candidate.rows_offset_hex.as_str()), + right_probe.candidates.first().map(|candidate| candidate.rows_offset_hex.as_str()) + ), + "shared shape matches mean two saves surfaced at least one candidate with the same exact probable-f32/small-unsigned/partial-zero/trailing-byte profile, while shared shape-family matches allow mild count drift inside the same dense lane family".to_string(), + ], + }) +} + fn derive_locomotive_catalog_from_named_availability_table( table: &SmpLoadedNamedLocomotiveAvailabilityTable, ) -> Option { @@ -10839,6 +11104,12 @@ fn inspect_bundle_bytes(bytes: &[u8], file_extension_hint: Option) -> Sm container_profile.as_ref(), save_region_collection_header_probe.as_ref(), ); + let save_region_fixed_row_run_candidate_probe = parse_save_region_fixed_row_run_candidate_probe( + bytes, + file_extension_hint.as_deref(), + container_profile.as_ref(), + save_region_collection_header_probe.as_ref(), + ); let save_placed_structure_collection_header_probe = parse_save_placed_structure_collection_header_probe( bytes, @@ -11040,6 +11311,7 @@ fn inspect_bundle_bytes(bytes: &[u8], file_extension_hint: Option) -> Sm save_region_collection_header_probe, save_region_record_triplet_probe, save_region_queued_notice_record_probe, + save_region_fixed_row_run_candidate_probe, save_placed_structure_collection_header_probe, save_placed_structure_record_triplet_probe, save_placed_structure_dynamic_side_buffer_probe, @@ -13464,6 +13736,309 @@ fn parse_save_region_queued_notice_record_probe( }) } +fn parse_save_region_fixed_row_run_candidate_probe( + bytes: &[u8], + file_extension_hint: Option<&str>, + container_profile: Option<&SmpContainerProfile>, + region_header_probe: Option<&SmpSaveTaggedCollectionHeaderProbe>, +) -> Option { + if file_extension_hint != Some("gms") { + return None; + } + let profile = container_profile?; + let region_header_probe = region_header_probe?; + let target_row_count = region_header_probe.live_record_count as usize; + if target_row_count == 0 { + return None; + } + let scan_end_offset = region_header_probe.metadata_tag_offset; + let row_span_len = target_row_count.checked_mul(SAVE_REGION_FIXED_ROW_STRIDE)?; + let scan_bytes = bytes.get(..scan_end_offset)?; + let mut candidates = find_u32_le_offsets(scan_bytes, region_header_probe.live_record_count) + .into_iter() + .filter_map(|count_offset| { + let rows_offset = count_offset.checked_add(4)?; + let rows_end_offset = rows_offset.checked_add(row_span_len)?; + if rows_end_offset > scan_end_offset { + return None; + } + let rows_bytes = bytes.get(rows_offset..rows_end_offset)?; + let mut dword_lane_summaries = + Vec::with_capacity(SAVE_REGION_FIXED_ROW_DWORD_LANE_COUNT); + let mut best_probable_density_lane = None::<(usize, usize)>; + for lane_index in 0..SAVE_REGION_FIXED_ROW_DWORD_LANE_COUNT { + let relative_offset = lane_index * 4; + let mut zero_count = 0usize; + let mut nonzero_count = 0usize; + let mut probable_normal_f32_count = 0usize; + let mut small_unsigned_count = 0usize; + let mut distinct_values = BTreeSet::new(); + let mut sample_values_hex = Vec::new(); + for row_index in 0..target_row_count { + let row_offset = row_index * SAVE_REGION_FIXED_ROW_STRIDE + relative_offset; + let raw_u32 = read_u32_at(rows_bytes, row_offset)?; + if raw_u32 == 0 { + zero_count += 1; + } else { + nonzero_count += 1; + } + if probable_normal_f32_string(raw_u32).is_some() { + probable_normal_f32_count += 1; + } + if raw_u32 <= 1024 { + small_unsigned_count += 1; + } + if distinct_values.insert(raw_u32) && sample_values_hex.len() < 6 { + sample_values_hex.push(format!("0x{raw_u32:08x}")); + } + } + if best_probable_density_lane + .is_none_or(|(_, best_count)| probable_normal_f32_count > best_count) + { + best_probable_density_lane = Some((relative_offset, probable_normal_f32_count)); + } + dword_lane_summaries.push(SmpSaveFixedRowRunDwordLaneSummary { + relative_offset, + relative_offset_hex: format!("0x{relative_offset:x}"), + zero_count, + nonzero_count, + distinct_value_count: distinct_values.len(), + probable_normal_f32_count, + small_unsigned_count, + sample_values_hex, + }); + } + let mut trailing_values = BTreeSet::new(); + let mut trailing_byte_zero_count = 0usize; + let mut trailing_byte_nonzero_count = 0usize; + let mut trailing_byte_sample_values_hex = Vec::new(); + for row_index in 0..target_row_count { + let value = *rows_bytes.get(row_index * SAVE_REGION_FIXED_ROW_STRIDE + 0x28)?; + if value == 0 { + trailing_byte_zero_count += 1; + } else { + trailing_byte_nonzero_count += 1; + } + if trailing_values.insert(value) && trailing_byte_sample_values_hex.len() < 8 { + trailing_byte_sample_values_hex.push(format!("0x{value:02x}")); + } + } + let shape_signature = build_save_region_fixed_row_run_candidate_shape_signature( + &dword_lane_summaries, + trailing_byte_zero_count, + trailing_values.len(), + target_row_count, + ); + let shape_family_signature = + build_save_region_fixed_row_run_candidate_shape_family_signature( + &dword_lane_summaries, + trailing_byte_zero_count, + trailing_values.len(), + target_row_count, + ); + Some(SmpSaveRegionFixedRowRunCandidate { + count_offset, + count_offset_hex: format!("0x{count_offset:x}"), + row_count: target_row_count, + row_stride: SAVE_REGION_FIXED_ROW_STRIDE, + row_stride_hex: format!("0x{:x}", SAVE_REGION_FIXED_ROW_STRIDE), + rows_offset, + rows_offset_hex: format!("0x{rows_offset:x}"), + rows_end_offset, + rows_end_offset_hex: format!("0x{rows_end_offset:x}"), + distance_to_region_metadata_tag: scan_end_offset.saturating_sub(rows_end_offset), + distance_to_region_metadata_tag_hex: format!( + "0x{:x}", + scan_end_offset.saturating_sub(rows_end_offset) + ), + dword_lane_summaries, + shape_signature, + shape_family_signature, + trailing_byte_zero_count, + trailing_byte_nonzero_count, + trailing_byte_distinct_value_count: trailing_values.len(), + trailing_byte_sample_values_hex, + best_probable_density_lane_relative_offset_hex: best_probable_density_lane + .filter(|(_, count)| *count != 0) + .map(|(relative_offset, _)| format!("0x{relative_offset:x}")), + }) + }) + .collect::>(); + candidates.sort_by_key(|candidate| { + ( + Reverse( + candidate + .dword_lane_summaries + .iter() + .map(|summary| summary.probable_normal_f32_count) + .max() + .unwrap_or_default(), + ), + candidate.distance_to_region_metadata_tag, + candidate.count_offset, + ) + }); + candidates.truncate(SAVE_REGION_FIXED_ROW_CANDIDATE_PROBE_LIMIT); + let candidate_count = candidates.len(); + let best_candidate_offset_hex = candidates + .first() + .map(|candidate| candidate.rows_offset_hex.clone()); + Some(SmpSaveRegionFixedRowRunCandidateProbe { + profile_family: profile.profile_family.clone(), + source_kind: "save-region-fixed-row-run-candidates".to_string(), + semantic_family: "scenario-save-region-fixed-row-run-candidates".to_string(), + target_row_count, + target_row_stride: SAVE_REGION_FIXED_ROW_STRIDE, + target_row_stride_hex: format!("0x{:x}", SAVE_REGION_FIXED_ROW_STRIDE), + scan_start_offset: 0, + scan_start_offset_hex: "0x0".to_string(), + scan_end_offset, + scan_end_offset_hex: format!("0x{scan_end_offset:x}"), + candidates, + evidence: vec![ + format!( + "candidate scan looks for pre-region-header counted runs keyed to the grounded live region count {} with fixed row stride 0x{:x}", + target_row_count, SAVE_REGION_FIXED_ROW_STRIDE + ), + format!( + "current scan range ends at region metadata tag offset 0x{:x}, because the atlas restore order places the fixed rows before the tagged 0x5209/0x520a/0x520b region collection", + scan_end_offset + ), + format!( + "kept {} highest-signal candidates after sorting by probable-f32 lane density and proximity to the region metadata tag; best candidate rows offset is {:?}", + candidate_count, best_candidate_offset_hex + ), + ], + }) +} + +fn build_save_region_fixed_row_run_candidate_shape_signature( + dword_lane_summaries: &[SmpSaveFixedRowRunDwordLaneSummary], + trailing_byte_zero_count: usize, + trailing_byte_distinct_value_count: usize, + row_count: usize, +) -> String { + fn pick_lane_terms( + summaries: &[SmpSaveFixedRowRunDwordLaneSummary], + score: F, + include: P, + row_count: usize, + max_terms: usize, + ) -> Vec + where + F: Fn(&SmpSaveFixedRowRunDwordLaneSummary) -> usize, + P: Fn(&SmpSaveFixedRowRunDwordLaneSummary) -> bool, + { + let high_signal_threshold = row_count.saturating_mul(3) / 4; + let mut picked = summaries + .iter() + .filter(|summary| include(summary)) + .filter(|summary| score(summary) >= high_signal_threshold) + .map(|summary| { + ( + summary.relative_offset, + format!("{}:{}", summary.relative_offset_hex, score(summary)), + ) + }) + .collect::>(); + if picked.is_empty() { + picked = summaries + .iter() + .filter(|summary| include(summary)) + .filter_map(|summary| { + let value = score(summary); + (value != 0).then(|| { + ( + summary.relative_offset, + format!("{}:{}", summary.relative_offset_hex, value), + ) + }) + }) + .collect::>(); + picked.sort_by_key(|(relative_offset, term)| { + let value = term + .split(':') + .nth(1) + .and_then(|part| part.parse::().ok()) + .unwrap_or_default(); + (Reverse(value), *relative_offset) + }); + picked.truncate(max_terms); + } + picked.into_iter().map(|(_, term)| term).collect() + } + + let probable_terms = pick_lane_terms( + dword_lane_summaries, + |summary| summary.probable_normal_f32_count, + |_| true, + row_count, + 3, + ); + let small_terms = pick_lane_terms( + dword_lane_summaries, + |summary| summary.small_unsigned_count, + |summary| summary.nonzero_count != 0, + row_count, + 2, + ); + let zero_terms = pick_lane_terms( + dword_lane_summaries, + |summary| summary.zero_count, + |summary| summary.zero_count != row_count, + row_count, + 2, + ); + + format!( + "pf32=[{}]|small=[{}]|zero=[{}]|trail={}/{}", + probable_terms.join(","), + small_terms.join(","), + zero_terms.join(","), + trailing_byte_zero_count, + trailing_byte_distinct_value_count + ) +} + +fn build_save_region_fixed_row_run_candidate_shape_family_signature( + dword_lane_summaries: &[SmpSaveFixedRowRunDwordLaneSummary], + trailing_byte_zero_count: usize, + trailing_byte_distinct_value_count: usize, + row_count: usize, +) -> String { + let dense_pf32_offsets = dword_lane_summaries + .iter() + .filter(|summary| summary.probable_normal_f32_count >= row_count.saturating_mul(3) / 4) + .map(|summary| summary.relative_offset_hex.clone()) + .collect::>(); + let partial_zero_offsets = dword_lane_summaries + .iter() + .filter(|summary| { + summary.zero_count != 0 + && summary.zero_count != row_count + && summary.zero_count >= row_count.saturating_mul(5) / 100 + }) + .map(|summary| summary.relative_offset_hex.clone()) + .collect::>(); + let small_nonzero_offsets = dword_lane_summaries + .iter() + .filter(|summary| { + summary.nonzero_count != 0 + && summary.small_unsigned_count >= row_count.saturating_mul(8) / 100 + }) + .map(|summary| summary.relative_offset_hex.clone()) + .collect::>(); + + format!( + "dense_pf32=[{}]|small_nonzero=[{}]|partial_zero=[{}]|trail_bucket={}/{}", + dense_pf32_offsets.join(","), + small_nonzero_offsets.join(","), + partial_zero_offsets.join(","), + trailing_byte_zero_count / 8, + trailing_byte_distinct_value_count / 8 + ) +} + fn parse_save_placed_structure_record_triplet_probe( bytes: &[u8], header_probe: Option<&SmpSaveTaggedCollectionHeaderProbe>, @@ -24135,6 +24710,88 @@ mod tests { assert_eq!(probe.entries[0].trailing_sentinel_i32_1, -1); } + #[test] + fn parses_region_fixed_row_run_candidate_probe_from_seeded_rows() { + let mut bytes = vec![0u8; 0x200]; + let count_offset = 0x20usize; + let rows_offset = count_offset + 4; + let metadata_tag_offset = 0x120usize; + bytes[count_offset..count_offset + 4].copy_from_slice(&2u32.to_le_bytes()); + + let first_row = rows_offset; + bytes[first_row..first_row + 4].copy_from_slice(&11u32.to_le_bytes()); + bytes[first_row + 4..first_row + 8].copy_from_slice(&0.25f32.to_bits().to_le_bytes()); + bytes[first_row + 8..first_row + 12].copy_from_slice(&0x11223344u32.to_le_bytes()); + bytes[first_row + 0x28] = 0x07; + + let second_row = rows_offset + SAVE_REGION_FIXED_ROW_STRIDE; + bytes[second_row..second_row + 4].copy_from_slice(&12u32.to_le_bytes()); + bytes[second_row + 4..second_row + 8].copy_from_slice(&0.5f32.to_bits().to_le_bytes()); + bytes[second_row + 8..second_row + 12].copy_from_slice(&0x55667788u32.to_le_bytes()); + bytes[second_row + 0x28] = 0x08; + + let probe = parse_save_region_fixed_row_run_candidate_probe( + &bytes, + Some("gms"), + Some(&SmpContainerProfile { + profile_family: "rt3-105-save-container-v1".to_string(), + profile_evidence: vec![], + is_known_profile: true, + }), + Some(&SmpSaveTaggedCollectionHeaderProbe { + profile_family: "rt3-105-save-container-v1".to_string(), + source_kind: "save-region-tagged-header-counts".to_string(), + semantic_family: "scenario-save-region-header-counts".to_string(), + metadata_tag_offset, + records_tag_offset: 0, + close_tag_offset: 0, + direct_collection_flag: 0, + direct_collection_flag_hex: "0x00000000".to_string(), + direct_record_stride: 0x06, + direct_record_stride_hex: "0x00000006".to_string(), + live_id_bound: 0x96, + live_id_bound_hex: "0x00000096".to_string(), + live_record_count: 2, + live_record_count_hex: "0x00000002".to_string(), + header_words: vec![], + header_hex_words: vec![], + evidence: vec![], + }), + ) + .expect("region fixed-row run candidate probe should parse"); + + assert_eq!(probe.target_row_count, 2); + assert_eq!(probe.target_row_stride, SAVE_REGION_FIXED_ROW_STRIDE); + assert_eq!(probe.candidates.len(), 1); + assert_eq!(probe.candidates[0].count_offset, count_offset); + assert_eq!(probe.candidates[0].rows_offset, rows_offset); + assert_eq!( + probe.candidates[0].best_probable_density_lane_relative_offset_hex, + Some("0x4".to_string()) + ); + assert_eq!( + probe.candidates[0].dword_lane_summaries[0].small_unsigned_count, + 2 + ); + assert_eq!( + probe.candidates[0].dword_lane_summaries[1].probable_normal_f32_count, + 2 + ); + assert_eq!(probe.candidates[0].trailing_byte_nonzero_count, 2); + assert_eq!( + probe.candidates[0].trailing_byte_sample_values_hex, + vec!["0x07".to_string(), "0x08".to_string()] + ); + assert_eq!( + probe.candidates[0].shape_signature, + "pf32=[0x4:2,0x8:2]|small=[0x0:2]|zero=[]|trail=0/2" + ); + assert_eq!( + probe.candidates[0].shape_family_signature, + "dense_pf32=[0x4,0x8]|small_nonzero=[0x0,0x4,0x8]|partial_zero=[]|trail_bucket=0/0" + ); + } + #[test] fn parses_placed_structure_record_triplet_probe_from_dual_name_rows() { let mut bytes = vec![0u8; 0x400]; @@ -26121,6 +26778,7 @@ mod tests { region_collection_header: None, region_record_triplets: None, region_queued_notice_records: None, + region_fixed_row_run_candidates: None, placed_structure_collection_header: None, placed_structure_record_triplets: None, placed_structure_dynamic_side_buffer: None, @@ -26132,6 +26790,156 @@ mod tests { } } + #[test] + fn compares_region_fixed_row_run_candidates_by_shape_signature() { + let mut left = empty_analysis_report(); + left.region_fixed_row_run_candidates = Some(SmpSaveRegionFixedRowRunCandidateProbe { + profile_family: left.profile_family.clone(), + source_kind: "save-region-fixed-row-run-candidates".to_string(), + semantic_family: "scenario-save-region-fixed-row-run-candidates".to_string(), + target_row_count: 145, + target_row_stride: 0x29, + target_row_stride_hex: "0x29".to_string(), + scan_start_offset: 0, + scan_start_offset_hex: "0x0".to_string(), + scan_end_offset: 0x100, + scan_end_offset_hex: "0x100".to_string(), + candidates: vec![ + SmpSaveRegionFixedRowRunCandidate { + count_offset: 0x20, + count_offset_hex: "0x20".to_string(), + row_count: 145, + row_stride: 0x29, + row_stride_hex: "0x29".to_string(), + rows_offset: 0x24, + rows_offset_hex: "0x24".to_string(), + rows_end_offset: 0x39, + rows_end_offset_hex: "0x39".to_string(), + distance_to_region_metadata_tag: 0xc7, + distance_to_region_metadata_tag_hex: "0xc7".to_string(), + dword_lane_summaries: Vec::new(), + shape_signature: "pf32=[0x14:120]|small=[0x20:17]|zero=[0x20:11]|trail=28/63" + .to_string(), + shape_family_signature: + "dense_pf32=[0x14]|small_nonzero=[0x20]|partial_zero=[0x20]|trail_bucket=3/7" + .to_string(), + trailing_byte_zero_count: 28, + trailing_byte_nonzero_count: 117, + trailing_byte_distinct_value_count: 63, + trailing_byte_sample_values_hex: Vec::new(), + best_probable_density_lane_relative_offset_hex: Some("0x14".to_string()), + }, + SmpSaveRegionFixedRowRunCandidate { + count_offset: 0x40, + count_offset_hex: "0x40".to_string(), + row_count: 145, + row_stride: 0x29, + row_stride_hex: "0x29".to_string(), + rows_offset: 0x44, + rows_offset_hex: "0x44".to_string(), + rows_end_offset: 0x59, + rows_end_offset_hex: "0x59".to_string(), + distance_to_region_metadata_tag: 0xa7, + distance_to_region_metadata_tag_hex: "0xa7".to_string(), + dword_lane_summaries: Vec::new(), + shape_signature: "pf32=[0x18:107]|small=[0x10:17]|zero=[0x14:12]|trail=32/58" + .to_string(), + shape_family_signature: + "dense_pf32=[]|small_nonzero=[0x10]|partial_zero=[0x14]|trail_bucket=4/7" + .to_string(), + trailing_byte_zero_count: 32, + trailing_byte_nonzero_count: 113, + trailing_byte_distinct_value_count: 58, + trailing_byte_sample_values_hex: Vec::new(), + best_probable_density_lane_relative_offset_hex: Some("0x18".to_string()), + }, + ], + evidence: Vec::new(), + }); + + let mut right = empty_analysis_report(); + right.region_fixed_row_run_candidates = Some(SmpSaveRegionFixedRowRunCandidateProbe { + profile_family: right.profile_family.clone(), + source_kind: "save-region-fixed-row-run-candidates".to_string(), + semantic_family: "scenario-save-region-fixed-row-run-candidates".to_string(), + target_row_count: 145, + target_row_stride: 0x29, + target_row_stride_hex: "0x29".to_string(), + scan_start_offset: 0, + scan_start_offset_hex: "0x0".to_string(), + scan_end_offset: 0x100, + scan_end_offset_hex: "0x100".to_string(), + candidates: vec![ + SmpSaveRegionFixedRowRunCandidate { + count_offset: 0x80, + count_offset_hex: "0x80".to_string(), + row_count: 145, + row_stride: 0x29, + row_stride_hex: "0x29".to_string(), + rows_offset: 0x84, + rows_offset_hex: "0x84".to_string(), + rows_end_offset: 0x99, + rows_end_offset_hex: "0x99".to_string(), + distance_to_region_metadata_tag: 0x67, + distance_to_region_metadata_tag_hex: "0x67".to_string(), + dword_lane_summaries: Vec::new(), + shape_signature: "pf32=[0x18:107]|small=[0x10:17]|zero=[0x14:12]|trail=32/58" + .to_string(), + shape_family_signature: + "dense_pf32=[]|small_nonzero=[0x10]|partial_zero=[0x14]|trail_bucket=4/7" + .to_string(), + trailing_byte_zero_count: 32, + trailing_byte_nonzero_count: 113, + trailing_byte_distinct_value_count: 58, + trailing_byte_sample_values_hex: Vec::new(), + best_probable_density_lane_relative_offset_hex: Some("0x18".to_string()), + }, + SmpSaveRegionFixedRowRunCandidate { + count_offset: 0xa0, + count_offset_hex: "0xa0".to_string(), + row_count: 145, + row_stride: 0x29, + row_stride_hex: "0x29".to_string(), + rows_offset: 0xa4, + rows_offset_hex: "0xa4".to_string(), + rows_end_offset: 0xb9, + rows_end_offset_hex: "0xb9".to_string(), + distance_to_region_metadata_tag: 0x47, + distance_to_region_metadata_tag_hex: "0x47".to_string(), + dword_lane_summaries: Vec::new(), + shape_signature: "pf32=[0x24:100]|small=[0xc:16]|zero=[0xc:11]|trail=34/60" + .to_string(), + shape_family_signature: + "dense_pf32=[]|small_nonzero=[0xc]|partial_zero=[0xc]|trail_bucket=4/7" + .to_string(), + trailing_byte_zero_count: 34, + trailing_byte_nonzero_count: 111, + trailing_byte_distinct_value_count: 60, + trailing_byte_sample_values_hex: Vec::new(), + best_probable_density_lane_relative_offset_hex: Some("0x24".to_string()), + }, + ], + evidence: Vec::new(), + }); + + let comparison = compare_save_region_fixed_row_run_candidates(&left, &right) + .expect("comparison should build"); + assert_eq!(comparison.shared_shape_matches.len(), 1); + assert_eq!( + comparison.shared_shape_matches[0].shape_signature, + "pf32=[0x18:107]|small=[0x10:17]|zero=[0x14:12]|trail=32/58" + ); + assert_eq!(comparison.shared_shape_matches[0].left_rank, 2); + assert_eq!(comparison.shared_shape_matches[0].right_rank, 1); + assert_eq!(comparison.shared_shape_family_matches.len(), 1); + assert_eq!( + comparison.shared_shape_family_matches[0].shape_signature, + "dense_pf32=[]|small_nonzero=[0x10]|partial_zero=[0x14]|trail_bucket=4/7" + ); + assert_eq!(comparison.left_only_shape_signatures.len(), 1); + assert_eq!(comparison.right_only_shape_signatures.len(), 1); + } + #[test] fn builds_region_service_trace_report_with_explicit_latch_blockers() { let mut analysis = empty_analysis_report(); diff --git a/docs/rehost-queue.md b/docs/rehost-queue.md index d998b2e..5d9543a 100644 --- a/docs/rehost-queue.md +++ b/docs/rehost-queue.md @@ -557,6 +557,20 @@ Working rule: appear to live in either the pre-`0x55f1` prefix band or the fixed `0x55f2` reserved dword band on grounded ordinary saves. That shifts the next region payload-comparison pass onto later body seams, not back onto the prefix or fixed-policy chunk. +- The new fixed-row run candidate probe pushes that same payload search one seam later, but it is + not grounded yet: on both `p.gms` and `q.gms` it finds high-signal counted runs keyed to the + live region count `145` with fixed row stride `0x29` before the tagged `0x5209/0x520a/0x520b` + region collection, yet the top candidate offset is not stable (`p.gms = 0xd13239`, + `q.gms = 0xd2d7d7`). So the next region payload pass should compare candidate lane-shape + fingerprints across saves rather than promoting any one absolute pre-header offset as the fixed + restore seam. +- The new two-save `runtime compare-region-fixed-row-runs ` report now does + that comparison directly. Current result: `p.gms` vs `q.gms` has `0` exact shape overlaps, and + the only coarse family overlaps are lower-ranked fully mixed candidates where every dword lane is + still simultaneously small-nonzero and partially-zero. That means the fixed-row scan remains + useful negative evidence, but it is still not honest to promote as the missing region restore + seam; the next region pass should stay focused on later restore owners or a more selective row + family discriminator above this mixed pre-header corpus. - The rest of `0x00455fc0` is ruled down further now too: after the `+0x48` callback it only runs `0x0052ebd0`, which reads two one-byte generic flags through `0x531150` into base object bytes `[this+0x20]`, `[this+0x8d]`, `[this+0x5c..+0x61]`, `[this+0x1ee]`, `[this+0x1fa]`, and