Validate GDSII record sizes through one fallible path
Route every record read through size validation and report malformed payloads as parse errors instead of panicking. Apply metadata size constraints through the Record trait, hide Arrow reader internals, and remove unused wrappers and scratch code. Add coverage for direct, skipped, and payload reads, truncated buffers, and FFI error offsets.
This commit is contained in:
parent
a384bb2f89
commit
54aa757678
9 changed files with 168 additions and 150 deletions
|
|
@ -206,7 +206,6 @@ mod tests {
|
||||||
fn test_parse_bitarray() {
|
fn test_parse_bitarray() {
|
||||||
use crate::basic::parse_bitarray;
|
use crate::basic::parse_bitarray;
|
||||||
|
|
||||||
//assert!(parse_bitarray(b"59") == 13625);
|
|
||||||
assert_eq!(parse_bitarray(b"\x00\x00").unwrap().1, [false; 16]);
|
assert_eq!(parse_bitarray(b"\x00\x00").unwrap().1, [false; 16]);
|
||||||
assert_eq!(parse_bitarray(b"\xff\xff").unwrap().1, [true; 16]);
|
assert_eq!(parse_bitarray(b"\xff\xff").unwrap().1, [true; 16]);
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -4,7 +4,6 @@
|
||||||
///
|
///
|
||||||
|
|
||||||
use crate::records::{
|
use crate::records::{
|
||||||
//BOX, BOUNDARY, NODE, PATH, TEXT, SREF, AREF,
|
|
||||||
DATATYPE, PATHTYPE, BOXTYPE, NODETYPE, TEXTTYPE,
|
DATATYPE, PATHTYPE, BOXTYPE, NODETYPE, TEXTTYPE,
|
||||||
LAYER, XY, WIDTH, COLROW, PRESENTATION, STRING,
|
LAYER, XY, WIDTH, COLROW, PRESENTATION, STRING,
|
||||||
STRANS, MAG, ANGLE, PROPATTR, PROPVALUE,
|
STRANS, MAG, ANGLE, PROPATTR, PROPVALUE,
|
||||||
|
|
@ -13,11 +12,10 @@ use crate::records::{
|
||||||
|
|
||||||
use crate::records;
|
use crate::records;
|
||||||
use crate::record::{RecordHeader, Record};
|
use crate::record::{RecordHeader, Record};
|
||||||
use crate::basic::{IResult, fail, take_bytes}; //OResult
|
use crate::basic::{IResult, fail, take_bytes};
|
||||||
|
|
||||||
use std::string::String;
|
use std::string::String;
|
||||||
use std::collections::HashMap;
|
use std::collections::HashMap;
|
||||||
//use std::io::Write;
|
|
||||||
|
|
||||||
use arrow_array::builder::{
|
use arrow_array::builder::{
|
||||||
StructBuilder, FixedSizeListBuilder, ListBuilder, StringBuilder, ArrayBuilder, BooleanBuilder,
|
StructBuilder, FixedSizeListBuilder, ListBuilder, StringBuilder, ArrayBuilder, BooleanBuilder,
|
||||||
|
|
@ -25,8 +23,8 @@ use arrow_array::builder::{
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
||||||
pub type DListBuilder = ListBuilder<Box<dyn ArrayBuilder>>;
|
pub(crate) type DListBuilder = ListBuilder<Box<dyn ArrayBuilder>>;
|
||||||
pub type FListBuilder = FixedSizeListBuilder<Box<dyn ArrayBuilder>>;
|
pub(crate) type FListBuilder = FixedSizeListBuilder<Box<dyn ArrayBuilder>>;
|
||||||
|
|
||||||
struct PropertyRecord {
|
struct PropertyRecord {
|
||||||
key: i16,
|
key: i16,
|
||||||
|
|
@ -72,15 +70,15 @@ struct ARefRecord {
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Clone)]
|
#[derive(Clone)]
|
||||||
pub struct ScanRefRecord {
|
pub(crate) struct ScanRefRecord {
|
||||||
pub target_id: u32,
|
pub(crate) target_id: u32,
|
||||||
pub invert_y: bool,
|
pub(crate) invert_y: bool,
|
||||||
pub scale: f64,
|
pub(crate) scale: f64,
|
||||||
pub angle_rad: f64,
|
pub(crate) angle_rad: f64,
|
||||||
pub xy: u64,
|
pub(crate) xy: u64,
|
||||||
pub xy0: u64,
|
pub(crate) xy0: u64,
|
||||||
pub xy1: u64,
|
pub(crate) xy1: u64,
|
||||||
pub counts: u32,
|
pub(crate) counts: u32,
|
||||||
}
|
}
|
||||||
|
|
||||||
enum RefRecord {
|
enum RefRecord {
|
||||||
|
|
@ -117,7 +115,7 @@ fn insert_layer(
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
pub fn read_elements<'a>(
|
pub(crate) fn read_elements<'a>(
|
||||||
input: &'a [u8],
|
input: &'a [u8],
|
||||||
cell_builder: &mut StructBuilder,
|
cell_builder: &mut StructBuilder,
|
||||||
names: &mut HashMap<String, u32>,
|
names: &mut HashMap<String, u32>,
|
||||||
|
|
@ -293,7 +291,7 @@ pub fn read_elements<'a>(
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
pub fn scan_elements<'a>(
|
pub(crate) fn scan_elements<'a>(
|
||||||
input: &'a [u8],
|
input: &'a [u8],
|
||||||
names: &mut HashMap<String, u32>,
|
names: &mut HashMap<String, u32>,
|
||||||
) -> IResult<'a, Vec<ScanRefRecord>> {
|
) -> IResult<'a, Vec<ScanRefRecord>> {
|
||||||
|
|
@ -583,7 +581,7 @@ fn append_boundary_prop(
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
pub fn read_path<'a>(
|
fn read_path<'a>(
|
||||||
input: &'a [u8],
|
input: &'a [u8],
|
||||||
cell_builder: &mut StructBuilder,
|
cell_builder: &mut StructBuilder,
|
||||||
layers: &mut HashMap<u32, u32>,
|
layers: &mut HashMap<u32, u32>,
|
||||||
|
|
@ -654,7 +652,7 @@ pub fn read_path<'a>(
|
||||||
Ok((input, ()))
|
Ok((input, ()))
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn read_boxnode<'a>(
|
fn read_boxnode<'a>(
|
||||||
input: &'a [u8],
|
input: &'a [u8],
|
||||||
cell_builder: &mut StructBuilder,
|
cell_builder: &mut StructBuilder,
|
||||||
tag: u16,
|
tag: u16,
|
||||||
|
|
@ -698,7 +696,7 @@ pub fn read_boxnode<'a>(
|
||||||
Ok((input, ()))
|
Ok((input, ()))
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn read_text<'a>(
|
fn read_text<'a>(
|
||||||
input: &'a [u8],
|
input: &'a [u8],
|
||||||
cell_builder: &mut StructBuilder,
|
cell_builder: &mut StructBuilder,
|
||||||
layers: &mut HashMap<u32, u32>,
|
layers: &mut HashMap<u32, u32>,
|
||||||
|
|
@ -967,7 +965,7 @@ fn read_ref<'a>(
|
||||||
/// Preserves record order and duplicate property attributes.
|
/// Preserves record order and duplicate property attributes.
|
||||||
/// Stops reading after consuming ENDEL record.
|
/// Stops reading after consuming ENDEL record.
|
||||||
///
|
///
|
||||||
pub fn read_properties<'a>(input: &'a [u8], props_builder: &mut DListBuilder) -> IResult<'a, ()> {
|
fn read_properties<'a>(input: &'a [u8], props_builder: &mut DListBuilder) -> IResult<'a, ()> {
|
||||||
let (input, ()) = walk_properties(input, |property| append_property(props_builder, property))?;
|
let (input, ()) = walk_properties(input, |property| append_property(props_builder, property))?;
|
||||||
props_builder.append(true);
|
props_builder.append(true);
|
||||||
Ok((input, ()))
|
Ok((input, ()))
|
||||||
|
|
|
||||||
|
|
@ -1,7 +1,7 @@
|
||||||
pub mod basic;
|
pub mod basic;
|
||||||
pub mod record;
|
pub mod record;
|
||||||
pub mod records;
|
pub mod records;
|
||||||
pub mod elements;
|
mod elements;
|
||||||
pub mod library;
|
pub mod library;
|
||||||
pub mod review;
|
pub mod review;
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -3,14 +3,11 @@
|
||||||
///
|
///
|
||||||
|
|
||||||
use std::io::Write;
|
use std::io::Write;
|
||||||
//use std::collections::HashMap;
|
|
||||||
|
|
||||||
pub use crate::record;
|
use crate::record::{RecordHeader, Record};
|
||||||
pub use crate::record::{RecordHeader, Record};
|
use crate::records;
|
||||||
pub use crate::records;
|
use crate::elements::{read_elements, scan_elements, DListBuilder, FListBuilder, ScanRefRecord};
|
||||||
pub use crate::elements;
|
use crate::basic::{IResult, OResult, take_bytes, fail};
|
||||||
pub use crate::elements::{read_elements, scan_elements, DListBuilder, FListBuilder, ScanRefRecord};
|
|
||||||
pub use crate::basic::{IResult, OResult, take_bytes, fail};
|
|
||||||
|
|
||||||
use std::string::String;
|
use std::string::String;
|
||||||
use std::collections::HashMap;
|
use std::collections::HashMap;
|
||||||
|
|
|
||||||
39
src/misc.py
39
src/misc.py
|
|
@ -1,39 +0,0 @@
|
||||||
'''
|
|
||||||
https://github.com/apache/arrow/blob/main/python/pyarrow/tests/test_cffi.py
|
|
||||||
'''
|
|
||||||
# -*- coding: utf-8 -*-
|
|
||||||
# Licensed to the Apache Software Foundation (ASF) under one
|
|
||||||
# or more contributor license agreements. See the NOTICE file
|
|
||||||
# distributed with this work for additional information
|
|
||||||
# regarding copyright ownership. The ASF licenses this file
|
|
||||||
# to you under the Apache License, Version 2.0 (the
|
|
||||||
# "License"); you may not use this file except in compliance
|
|
||||||
# with the License. You may obtain a copy of the License at
|
|
||||||
#
|
|
||||||
# http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
#
|
|
||||||
# Unless required by applicable law or agreed to in writing,
|
|
||||||
# software distributed under the License is distributed on an
|
|
||||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
||||||
# KIND, either express or implied. See the License for the
|
|
||||||
# specific language governing permissions and limitations
|
|
||||||
# under the License.
|
|
||||||
|
|
||||||
import ctypes
|
|
||||||
import pyarrow as pa
|
|
||||||
from pyarrow.cffi import ffi
|
|
||||||
|
|
||||||
c_schema = ffi.new('struct ArrowSchema*')
|
|
||||||
ptr_schema = int(ffi.cast('uintptr_t', c_schema))
|
|
||||||
c_array = ffi.new('struct ArrowArray*')
|
|
||||||
ptr_array = int(ffi.cast('uintptr_t', c_array))
|
|
||||||
|
|
||||||
# pyarrow.Array._import_from_c(ptr_array, pa.list_(pa.int32()))
|
|
||||||
|
|
||||||
# import gc
|
|
||||||
# gc.collect() # Make sure no Arrow data dangles in a ref cycle
|
|
||||||
# pyarrow.Array._export_from_c(arr, ptr_array, ptr_schema)
|
|
||||||
|
|
||||||
arr_new = pyarrow.Array._import_from_c(ptr_array, ptr_schema)
|
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -4,16 +4,12 @@
|
||||||
use std::io::{self, Write};
|
use std::io::{self, Write};
|
||||||
|
|
||||||
|
|
||||||
use crate::basic::{pack_datetime, pack_bitarray, pack_ascii, pack_int2, pack_int4, pack_real8}; #[warn(unused_imports)]
|
use crate::basic::{pack_datetime, pack_bitarray, pack_ascii, pack_int2, pack_int4, pack_real8};
|
||||||
use crate::basic::{parse_datetime, parse_bitarray, parse_ascii, parse_int2, parse_int4, parse_real8}; #[warn(unused_imports)]
|
use crate::basic::{parse_datetime, parse_bitarray, parse_ascii, parse_int2, parse_int4, parse_real8};
|
||||||
use crate::basic::{OResult, IResult, fail, parse_u16, take_bytes}; //ErrType,
|
use crate::basic::{OResult, IResult, fail, parse_u16, take_bytes};
|
||||||
use crate::records;
|
use crate::records;
|
||||||
|
|
||||||
|
|
||||||
//#[no_mangle]
|
|
||||||
//pub extern "C" fn write_record_header(
|
|
||||||
|
|
||||||
|
|
||||||
#[repr(C)]
|
#[repr(C)]
|
||||||
#[derive(Debug, Clone)]
|
#[derive(Debug, Clone)]
|
||||||
pub struct RecordHeader {
|
pub struct RecordHeader {
|
||||||
|
|
@ -53,7 +49,6 @@ pub trait RecordData {
|
||||||
|
|
||||||
fn read(input: &[u8], size: u16) -> IResult<'_, Self::BareData<'_>>;
|
fn read(input: &[u8], size: u16) -> IResult<'_, Self::BareData<'_>>;
|
||||||
fn pack_into(buf: &mut [u8], data: &Self::InData);
|
fn pack_into(buf: &mut [u8], data: &Self::InData);
|
||||||
//fn size(data: &Self::BareData<'_>) -> u16;
|
|
||||||
fn pack(data: &Self::InData) -> Self::ByteData;
|
fn pack(data: &Self::InData) -> Self::ByteData;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -73,22 +68,17 @@ pub trait Record<RData: RecordData> {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn read_header(input: &[u8]) -> IResult<'_, RecordHeader> {
|
|
||||||
RecordHeader::read(input)
|
|
||||||
}
|
|
||||||
|
|
||||||
fn write_header<W: Write>(ww: &mut W, data_size: u16) -> OResult {
|
fn write_header<W: Write>(ww: &mut W, data_size: u16) -> OResult {
|
||||||
RecordHeader{tag: Self::tag(), data_size}.write(ww)
|
RecordHeader{tag: Self::tag(), data_size}.write(ww)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn read_data(input: &[u8], size: u16) -> IResult<'_, RData::BareData<'_>> {
|
fn read_data(input: &[u8], size: u16) -> IResult<'_, RData::BareData<'_>> {
|
||||||
|
if let Err(message) = Self::check_size(size) {
|
||||||
|
return fail(input, message);
|
||||||
|
}
|
||||||
RData::read(input, size)
|
RData::read(input, size)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn pack_data(buf: &mut [u8], data: &RData::InData) {
|
|
||||||
RData::pack_into(buf, data)
|
|
||||||
}
|
|
||||||
|
|
||||||
///
|
///
|
||||||
/// Skip to the end of the next occurence of this record.
|
/// Skip to the end of the next occurence of this record.
|
||||||
///
|
///
|
||||||
|
|
@ -116,8 +106,7 @@ pub trait Record<RData: RecordData> {
|
||||||
(input, _) = take_bytes(input, header.data_size)?;
|
(input, _) = take_bytes(input, header.data_size)?;
|
||||||
(input, header) = RecordHeader::read(input)?;
|
(input, header) = RecordHeader::read(input)?;
|
||||||
}
|
}
|
||||||
let (input, data) = Self::read_data(input, header.data_size)?;
|
Self::read_data(input, header.data_size)
|
||||||
Ok((input, data))
|
|
||||||
}
|
}
|
||||||
|
|
||||||
fn expect_header(input: &[u8]) -> IResult<'_, u16> {
|
fn expect_header(input: &[u8]) -> IResult<'_, u16> {
|
||||||
|
|
@ -131,9 +120,7 @@ pub trait Record<RData: RecordData> {
|
||||||
|
|
||||||
fn read(input: &[u8]) -> IResult<'_, RData::BareData<'_>> {
|
fn read(input: &[u8]) -> IResult<'_, RData::BareData<'_>> {
|
||||||
let (input, size) = Self::expect_header(input)?;
|
let (input, size) = Self::expect_header(input)?;
|
||||||
Self::check_size(size).unwrap();
|
Self::read_data(input, size)
|
||||||
let (input, data) = Self::read_data(input, size)?;
|
|
||||||
Ok((input, data))
|
|
||||||
}
|
}
|
||||||
|
|
||||||
fn write<W: Write>(ww: &mut W, data: &RData::InData) -> OResult {
|
fn write<W: Write>(ww: &mut W, data: &RData::InData) -> OResult {
|
||||||
|
|
@ -155,7 +142,9 @@ impl RecordData for BitArray {
|
||||||
type ByteData = [u8; 2];
|
type ByteData = [u8; 2];
|
||||||
|
|
||||||
fn read(input: &[u8], size: u16) -> IResult<'_, Self::BareData<'_>> {
|
fn read(input: &[u8], size: u16) -> IResult<'_, Self::BareData<'_>> {
|
||||||
assert!(size == 2);
|
if size != 2 {
|
||||||
|
return fail(input, format!("Expected record size {}, got {size}", 2));
|
||||||
|
}
|
||||||
parse_bitarray(input)
|
parse_bitarray(input)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -178,7 +167,9 @@ impl RecordData for Int2 {
|
||||||
type ByteData = [u8; 2];
|
type ByteData = [u8; 2];
|
||||||
|
|
||||||
fn read(input: &[u8], size: u16) -> IResult<'_, Self::BareData<'_>> {
|
fn read(input: &[u8], size: u16) -> IResult<'_, Self::BareData<'_>> {
|
||||||
assert!(size == 2);
|
if size != 2 {
|
||||||
|
return fail(input, format!("Expected record size {}, got {size}", 2));
|
||||||
|
}
|
||||||
parse_int2(input)
|
parse_int2(input)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -200,7 +191,9 @@ impl RecordData for Int4 {
|
||||||
type ByteData = [u8; 4];
|
type ByteData = [u8; 4];
|
||||||
|
|
||||||
fn read(input: &[u8], size: u16) -> IResult<'_, Self::BareData<'_>> {
|
fn read(input: &[u8], size: u16) -> IResult<'_, Self::BareData<'_>> {
|
||||||
assert!(size == 4);
|
if size != 4 {
|
||||||
|
return fail(input, format!("Expected record size {}, got {size}", 4));
|
||||||
|
}
|
||||||
parse_int4(input)
|
parse_int4(input)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -223,8 +216,9 @@ impl RecordData for Int2Array {
|
||||||
type ByteData = Vec<u8>;
|
type ByteData = Vec<u8>;
|
||||||
|
|
||||||
fn read(input: &[u8], size: u16) -> IResult<'_, Self::BareData<'_>> {
|
fn read(input: &[u8], size: u16) -> IResult<'_, Self::BareData<'_>> {
|
||||||
assert!(size.is_multiple_of(2), "Record must contain an integer quantity of integers");
|
if !size.is_multiple_of(2) {
|
||||||
//let mut input = input;
|
return fail(input, format!("Record size must be divisible by 2, got {size}"));
|
||||||
|
}
|
||||||
let (input, bytes) = take_bytes(input, size)?;
|
let (input, bytes) = take_bytes(input, size)?;
|
||||||
Ok((input, Int2ArrayReader{bytes}))
|
Ok((input, Int2ArrayReader{bytes}))
|
||||||
}
|
}
|
||||||
|
|
@ -267,8 +261,9 @@ impl RecordData for Int4Array {
|
||||||
type ByteData = Vec<u8>;
|
type ByteData = Vec<u8>;
|
||||||
|
|
||||||
fn read(input: &[u8], size: u16) -> IResult<'_, Self::BareData<'_>> {
|
fn read(input: &[u8], size: u16) -> IResult<'_, Self::BareData<'_>> {
|
||||||
assert!(size.is_multiple_of(4), "Record must contain an integer quantity of integers");
|
if !size.is_multiple_of(4) {
|
||||||
//let mut input = input;
|
return fail(input, format!("Record size must be divisible by 4, got {size}"));
|
||||||
|
}
|
||||||
let (input, bytes) = take_bytes(input, size)?;
|
let (input, bytes) = take_bytes(input, size)?;
|
||||||
Ok((input, Int4ArrayReader{bytes}))
|
Ok((input, Int4ArrayReader{bytes}))
|
||||||
}
|
}
|
||||||
|
|
@ -311,7 +306,9 @@ impl RecordData for Real8 {
|
||||||
type ByteData = [u8; 8];
|
type ByteData = [u8; 8];
|
||||||
|
|
||||||
fn read(input: &[u8], size: u16) -> IResult<'_, Self::BareData<'_>> {
|
fn read(input: &[u8], size: u16) -> IResult<'_, Self::BareData<'_>> {
|
||||||
assert!(size == 8);
|
if size != 8 {
|
||||||
|
return fail(input, format!("Expected record size {}, got {size}", 8));
|
||||||
|
}
|
||||||
parse_real8(input)
|
parse_real8(input)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -333,7 +330,9 @@ impl RecordData for Real8Pair {
|
||||||
type ByteData = [u8; 2 * 8];
|
type ByteData = [u8; 2 * 8];
|
||||||
|
|
||||||
fn read(input: &[u8], size: u16) -> IResult<'_, Self::BareData<'_>> {
|
fn read(input: &[u8], size: u16) -> IResult<'_, Self::BareData<'_>> {
|
||||||
assert!(size == 2 * 8);
|
if size != 2 * 8 {
|
||||||
|
return fail(input, format!("Expected record size {}, got {size}", 2 * 8));
|
||||||
|
}
|
||||||
let (input, data0) = parse_real8(input)?;
|
let (input, data0) = parse_real8(input)?;
|
||||||
let (input, data1) = parse_real8(input)?;
|
let (input, data1) = parse_real8(input)?;
|
||||||
Ok((input, (data0, data1)))
|
Ok((input, (data0, data1)))
|
||||||
|
|
@ -350,10 +349,6 @@ impl RecordData for Real8Pair {
|
||||||
buf
|
buf
|
||||||
}
|
}
|
||||||
|
|
||||||
//fn write<W: Write>(ww: &mut W, data: &Self::BareData) -> OResult {
|
|
||||||
// let mut buf = [u8; 2 * 6 * 2];
|
|
||||||
// Self::pack_into(&mut buf, data)
|
|
||||||
//}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
pub struct ASCII;
|
pub struct ASCII;
|
||||||
|
|
@ -385,7 +380,9 @@ impl RecordData for DateTimePair {
|
||||||
type ByteData = [u8; 2 * 6 * 2];
|
type ByteData = [u8; 2 * 6 * 2];
|
||||||
|
|
||||||
fn read(input: &[u8], size: u16) -> IResult<'_, Self::BareData<'_>> {
|
fn read(input: &[u8], size: u16) -> IResult<'_, Self::BareData<'_>> {
|
||||||
assert!(size == 2 * 6 * 2);
|
if size != 2 * 6 * 2 {
|
||||||
|
return fail(input, format!("Expected record size {}, got {size}", 2 * 6 * 2));
|
||||||
|
}
|
||||||
let (input, data0) = parse_datetime(input)?;
|
let (input, data0) = parse_datetime(input)?;
|
||||||
let (input, data1) = parse_datetime(input)?;
|
let (input, data1) = parse_datetime(input)?;
|
||||||
Ok((input, [data0, data1]))
|
Ok((input, [data0, data1]))
|
||||||
|
|
@ -402,10 +399,6 @@ impl RecordData for DateTimePair {
|
||||||
buf
|
buf
|
||||||
}
|
}
|
||||||
|
|
||||||
//fn write<W: Write>(ww: &mut W, data: &Self::BareData) -> OResult {
|
|
||||||
// let mut buf = [u8; 2 * 6 * 2];
|
|
||||||
// Self::pack_into(&mut buf, data)
|
|
||||||
//}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
pub struct Empty;
|
pub struct Empty;
|
||||||
|
|
@ -415,7 +408,9 @@ impl RecordData for Empty {
|
||||||
type ByteData = [u8; 0];
|
type ByteData = [u8; 0];
|
||||||
|
|
||||||
fn read(input: &[u8], size: u16) -> IResult<'_, Self::BareData<'_>> {
|
fn read(input: &[u8], size: u16) -> IResult<'_, Self::BareData<'_>> {
|
||||||
assert!(size == 0);
|
if size != 0 {
|
||||||
|
return fail(input, format!("Expected record size {}, got {size}", 0));
|
||||||
|
}
|
||||||
Ok((input, ()))
|
Ok((input, ()))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -425,6 +420,4 @@ impl RecordData for Empty {
|
||||||
fn pack(_data: &Self::InData) -> Self::ByteData {
|
fn pack(_data: &Self::InData) -> Self::ByteData {
|
||||||
[]
|
[]
|
||||||
}
|
}
|
||||||
//fn write<W: Write>(ww: &mut W, data: &Self::BareData) {
|
|
||||||
//}
|
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -4,8 +4,6 @@
|
||||||
|
|
||||||
use crate::record::{Record, Int2, Int4, Int2Array, Int4Array, Real8, Real8Pair, DateTimePair, BitArray, ASCII, Empty};
|
use crate::record::{Record, Int2, Int4, Int2Array, Int4Array, Real8, Real8Pair, DateTimePair, BitArray, ASCII, Empty};
|
||||||
|
|
||||||
//use std::io::Write;
|
|
||||||
|
|
||||||
// record tags
|
// record tags
|
||||||
pub const RTAG_HEADER: u16 = 0x0002;
|
pub const RTAG_HEADER: u16 = 0x0002;
|
||||||
pub const RTAG_BGNLIB: u16 = 0x0102;
|
pub const RTAG_BGNLIB: u16 = 0x0102;
|
||||||
|
|
@ -77,37 +75,13 @@ pub const RTAG_USERCONSTRAINT: u16 = 0x4300;
|
||||||
pub const RTAG_SPACERERROR: u16 = 0x4400;
|
pub const RTAG_SPACERERROR: u16 = 0x4400;
|
||||||
pub const RTAG_CONTACT: u16 = 0x4500;
|
pub const RTAG_CONTACT: u16 = 0x4500;
|
||||||
|
|
||||||
/*
|
|
||||||
// data types
|
|
||||||
pub const DATA_TYPE_NONE: u16 = 0x00;
|
|
||||||
pub const DATA_TYPE_BIT: u16 = 0x01;
|
|
||||||
pub const DATA_TYPE_INT16: u16 = 0x02;
|
|
||||||
pub const DATA_TYPE_INT32: u16 = 0x03;
|
|
||||||
pub const DATA_TYPE_REAL32: u16 = 0x04;
|
|
||||||
pub const DATA_TYPE_REAL64: u16 = 0x05;
|
|
||||||
pub const DATA_TYPE_STR: u16 = 0x06;
|
|
||||||
|
|
||||||
pub const MAX_DATA_SIZE: usize = 8;
|
|
||||||
|
|
||||||
/// Returns the size of the given data type in bytes.
|
|
||||||
pub fn data_size(t: u16) -> Option<u16> {
|
|
||||||
match t {
|
|
||||||
x if x == DATA_TYPE_NONE => 0,
|
|
||||||
x if x == DATA_TYPE_BIT => 2,
|
|
||||||
x if x == DATA_TYPE_INT16 => 2,
|
|
||||||
x if x == DATA_TYPE_INT32 => 4,
|
|
||||||
x if x == DATA_TYPE_REAL32 => 4,
|
|
||||||
x if x == DATA_TYPE_REAL64 => 8,
|
|
||||||
_ => 0
|
|
||||||
}
|
|
||||||
*/
|
|
||||||
|
|
||||||
pub struct HEADER;
|
pub struct HEADER;
|
||||||
impl Record<Int2> for HEADER {
|
impl Record<Int2> for HEADER {
|
||||||
fn tag() -> u16 { RTAG_HEADER }
|
fn tag() -> u16 { RTAG_HEADER }
|
||||||
fn expected_size() -> Option<u16> { Some(2) }
|
fn expected_size() -> Option<u16> { Some(2) }
|
||||||
}
|
}
|
||||||
//impl Record<Int2> for HEADER;
|
|
||||||
|
|
||||||
pub struct BGNLIB;
|
pub struct BGNLIB;
|
||||||
impl Record<DateTimePair> for BGNLIB {
|
impl Record<DateTimePair> for BGNLIB {
|
||||||
|
|
@ -288,9 +262,8 @@ pub struct REFLIBS;
|
||||||
impl Record<ASCII> for REFLIBS {
|
impl Record<ASCII> for REFLIBS {
|
||||||
fn tag() -> u16 { RTAG_REFLIBS }
|
fn tag() -> u16 { RTAG_REFLIBS }
|
||||||
fn expected_size() -> Option<u16> { None }
|
fn expected_size() -> Option<u16> { None }
|
||||||
}
|
|
||||||
impl REFLIBS {
|
fn check_size(actual_size: u16) -> Result<(), String> {
|
||||||
pub fn check_size(actual_size: usize) -> Result<(), String> {
|
|
||||||
if actual_size.is_multiple_of(44) {
|
if actual_size.is_multiple_of(44) {
|
||||||
Ok(())
|
Ok(())
|
||||||
} else {
|
} else {
|
||||||
|
|
@ -303,9 +276,8 @@ pub struct FONTS;
|
||||||
impl Record<ASCII> for FONTS {
|
impl Record<ASCII> for FONTS {
|
||||||
fn tag() -> u16 { RTAG_FONTS }
|
fn tag() -> u16 { RTAG_FONTS }
|
||||||
fn expected_size() -> Option<u16> { None }
|
fn expected_size() -> Option<u16> { None }
|
||||||
}
|
|
||||||
impl FONTS {
|
fn check_size(actual_size: u16) -> Result<(), String> {
|
||||||
pub fn check_size(actual_size: usize) -> Result<(), String> {
|
|
||||||
if actual_size.is_multiple_of(44) {
|
if actual_size.is_multiple_of(44) {
|
||||||
Ok(())
|
Ok(())
|
||||||
} else {
|
} else {
|
||||||
|
|
@ -330,9 +302,8 @@ pub struct ATTRTABLE;
|
||||||
impl Record<ASCII> for ATTRTABLE {
|
impl Record<ASCII> for ATTRTABLE {
|
||||||
fn tag() -> u16 { RTAG_ATTRTABLE }
|
fn tag() -> u16 { RTAG_ATTRTABLE }
|
||||||
fn expected_size() -> Option<u16> { None }
|
fn expected_size() -> Option<u16> { None }
|
||||||
}
|
|
||||||
impl ATTRTABLE {
|
fn check_size(actual_size: u16) -> Result<(), String> {
|
||||||
pub fn check_size(actual_size: usize) -> Result<(), String> {
|
|
||||||
if actual_size.is_multiple_of(44) {
|
if actual_size.is_multiple_of(44) {
|
||||||
Ok(())
|
Ok(())
|
||||||
} else {
|
} else {
|
||||||
|
|
|
||||||
|
|
@ -31,20 +31,20 @@ fn error_message() -> String {
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn path_errors_preserve_outputs_and_convert_parser_panics() {
|
fn path_errors_preserve_outputs_and_clear_after_success() {
|
||||||
let mut array = FFI_ArrowArray::empty();
|
let mut array = FFI_ArrowArray::empty();
|
||||||
let mut schema = FFI_ArrowSchema::empty();
|
let mut schema = FFI_ArrowSchema::empty();
|
||||||
assert_eq!(unsafe { read_path(ptr::null(), &mut array, &mut schema) }, -1);
|
assert_eq!(unsafe { read_path(ptr::null(), &mut array, &mut schema) }, -1);
|
||||||
assert!(error_message().contains("path pointer was null"));
|
assert!(error_message().contains("path pointer was null"));
|
||||||
assert!(array.is_released());
|
assert!(array.is_released());
|
||||||
|
|
||||||
// A mismatched timestamp size currently makes Record::read panic.
|
// A mismatched timestamp size returns a parse error with a byte offset.
|
||||||
let mut bytes = Vec::new();
|
let mut bytes = Vec::new();
|
||||||
common::record(&mut bytes, records::RTAG_HEADER, &600_i16.to_be_bytes());
|
common::record(&mut bytes, records::RTAG_HEADER, &600_i16.to_be_bytes());
|
||||||
common::record(&mut bytes, records::RTAG_BGNLIB, &[0; 12]);
|
common::record(&mut bytes, records::RTAG_BGNLIB, &[0; 12]);
|
||||||
let file = TestFile::new("invalid-timestamps", &bytes);
|
let file = TestFile::new("invalid-timestamps", &bytes);
|
||||||
assert_eq!(unsafe { read_path(file.cpath().as_ptr(), &mut array, &mut schema) }, -1);
|
assert_eq!(unsafe { read_path(file.cpath().as_ptr(), &mut array, &mut schema) }, -1);
|
||||||
assert!(error_message().contains("Expected record size"));
|
assert!(error_message().contains("read failed at byte 10: Expected record size"));
|
||||||
assert!(array.is_released());
|
assert!(array.is_released());
|
||||||
|
|
||||||
let mut bytes = common::library_header();
|
let mut bytes = common::library_header();
|
||||||
|
|
|
||||||
99
tests/record_validation.rs
Normal file
99
tests/record_validation.rs
Normal file
|
|
@ -0,0 +1,99 @@
|
||||||
|
mod common;
|
||||||
|
|
||||||
|
use pheon::{
|
||||||
|
basic::ErrType,
|
||||||
|
record::{
|
||||||
|
ASCII, BitArray, DateTimePair, Empty, Int2, Int2Array, Int4, Int4Array, Real8, Real8Pair,
|
||||||
|
Record, RecordData,
|
||||||
|
},
|
||||||
|
records,
|
||||||
|
};
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn fixed_record_sizes_are_checked_by_every_read_entry_point() {
|
||||||
|
let payload = [0; 4];
|
||||||
|
let mut bytes = Vec::new();
|
||||||
|
common::record(&mut bytes, records::RTAG_HEADER, &payload);
|
||||||
|
let mut prefixed = Vec::new();
|
||||||
|
common::record(&mut prefixed, records::RTAG_LIBNAME, b"skip");
|
||||||
|
prefixed.extend_from_slice(&bytes);
|
||||||
|
|
||||||
|
for result in [
|
||||||
|
records::HEADER::read(&bytes),
|
||||||
|
records::HEADER::skip_and_read(&prefixed),
|
||||||
|
records::HEADER::read_data(&payload, 4),
|
||||||
|
] {
|
||||||
|
let (remaining, reason) = result.unwrap_err();
|
||||||
|
assert_eq!(remaining, payload);
|
||||||
|
assert!(matches!(reason, ErrType::Failed(message)
|
||||||
|
if message == "Expected record size 2, got 4"));
|
||||||
|
}
|
||||||
|
|
||||||
|
// The underlying integer-array payload is aligned, but COLROW needs two values.
|
||||||
|
let payload = [0; 2];
|
||||||
|
let mut bytes = Vec::new();
|
||||||
|
common::record(&mut bytes, records::RTAG_COLROW, &payload);
|
||||||
|
assert!(records::COLROW::read(&bytes).is_err());
|
||||||
|
assert!(records::COLROW::skip_and_read(&bytes).is_err());
|
||||||
|
assert!(records::COLROW::read_data(&payload, 2).is_err());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn metadata_size_constraints_apply_to_every_read_entry_point() {
|
||||||
|
macro_rules! check {
|
||||||
|
($record:ident, $tag:ident) => {{
|
||||||
|
let payload = [b'x'; 44];
|
||||||
|
let mut bytes = Vec::new();
|
||||||
|
common::record(&mut bytes, records::$tag, &payload);
|
||||||
|
let mut prefixed = common::library_header();
|
||||||
|
prefixed.extend_from_slice(&bytes);
|
||||||
|
assert_eq!(records::$record::read(&bytes).unwrap().1, payload);
|
||||||
|
assert_eq!(
|
||||||
|
records::$record::skip_and_read(&prefixed).unwrap().1,
|
||||||
|
payload
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
records::$record::read_data(&payload, 44).unwrap().1,
|
||||||
|
payload
|
||||||
|
);
|
||||||
|
|
||||||
|
let payload = [b'x'; 46];
|
||||||
|
let mut bytes = Vec::new();
|
||||||
|
common::record(&mut bytes, records::$tag, &payload);
|
||||||
|
assert!(records::$record::read(&bytes).is_err());
|
||||||
|
assert!(records::$record::skip_and_read(&bytes).is_err());
|
||||||
|
assert!(records::$record::read_data(&payload, 46).is_err());
|
||||||
|
}};
|
||||||
|
}
|
||||||
|
|
||||||
|
check!(REFLIBS, RTAG_REFLIBS);
|
||||||
|
check!(FONTS, RTAG_FONTS);
|
||||||
|
check!(ATTRTABLE, RTAG_ATTRTABLE);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn malformed_payload_sizes_return_errors_without_panicking() {
|
||||||
|
let payload = [0; 32];
|
||||||
|
assert!(BitArray::read(&payload, 4).is_err());
|
||||||
|
assert!(Int2::read(&payload, 4).is_err());
|
||||||
|
assert!(Int4::read(&payload, 2).is_err());
|
||||||
|
assert!(Real8::read(&payload, 4).is_err());
|
||||||
|
assert!(Real8Pair::read(&payload, 8).is_err());
|
||||||
|
assert!(DateTimePair::read(&payload, 12).is_err());
|
||||||
|
assert!(Empty::read(&payload, 2).is_err());
|
||||||
|
assert!(Int2Array::read(&payload, 3).is_err());
|
||||||
|
assert!(Int4Array::read(&payload, 6).is_err());
|
||||||
|
|
||||||
|
let mut bytes = Vec::new();
|
||||||
|
common::record(&mut bytes, records::RTAG_XY, &payload[..6]);
|
||||||
|
assert!(records::XY::read(&bytes).is_err());
|
||||||
|
assert!(records::XY::skip_and_read(&bytes).is_err());
|
||||||
|
assert!(records::XY::read_data(&payload, 6).is_err());
|
||||||
|
|
||||||
|
// Valid sizes still reject a truncated buffer; an empty string remains valid.
|
||||||
|
assert!(Int4::read(&payload[..2], 4).is_err());
|
||||||
|
assert!(Real8Pair::read(&payload[..8], 16).is_err());
|
||||||
|
assert!(DateTimePair::read(&payload[..12], 24).is_err());
|
||||||
|
assert!(Int4Array::read(&payload[..4], 8).is_err());
|
||||||
|
assert_eq!(ASCII::read(&[], 0).unwrap(), (&[][..], Vec::<u8>::new()));
|
||||||
|
}
|
||||||
Loading…
Reference in a new issue