arrow attempt compiles, nothing tested yet

This commit is contained in:
jan 2025-04-10 01:07:11 -07:00
commit 4a7bc8090a

View file

@ -3,7 +3,8 @@
/// structure references) and associated properties.
///
use crate::records::{BOX, BOUNDARY, NODE, PATH, TEXT, SREF, AREF,
use crate::records::{
//BOX, BOUNDARY, NODE, PATH, TEXT, SREF, AREF,
DATATYPE, PATHTYPE, BOXTYPE, NODETYPE, TEXTTYPE,
LAYER, XY, WIDTH, COLROW, PRESENTATION, STRING,
STRANS, MAG, ANGLE, PROPATTR, PROPVALUE,
@ -12,10 +13,550 @@ use crate::records::{BOX, BOUNDARY, NODE, PATH, TEXT, SREF, AREF,
use crate::records;
use crate::record::{RecordHeader, Record};
use crate::basic::{OResult, IResult, fail};
use crate::basic::{IResult, fail, take_bytes}; //OResult
use std::string::String;
use std::collections::HashMap;
use std::io::Write;
//use std::io::Write;
use std::sync::Arc;
use arrow::datatypes::{DataType, Field, Fields};
use arrow::array::{
StructBuilder, ListBuilder, StringBuilder, ArrayBuilder, Float64Builder, BooleanBuilder,
Int32Builder, Int16Builder, UInt64Builder, UInt32Builder, UInt8Builder,
StructArray,
};
type DListBuilder = ListBuilder<Box<dyn ArrayBuilder>>;
pub fn read_library(input: &[u8]) -> IResult<StructArray> {
let input_size = input.len();
let property_t = DataType::Struct(Fields::from(vec![
Field::new("key", DataType::Int16, false),
Field::new("value", DataType::Utf8, false),
]));
let property_list_t = DataType::List(Arc::new(
Field::new_list_field(property_t, false)
));
let repetition_struct_t = DataType::Struct(Fields::from(vec![
Field::new("x0", DataType::Int32, false),
Field::new("y0", DataType::Int32, false),
Field::new("x1", DataType::Int32, false),
Field::new("y1", DataType::Int32, false),
Field::new("count0", DataType::Int16, false),
Field::new("count1", DataType::Int16, false),
]));
let ref_struct_t = DataType::Struct(Fields::from(vec![
Field::new("target", DataType::UInt32, false),
Field::new("invert_y", DataType::Boolean, true),
Field::new("mag", DataType::Float64, true),
Field::new("angle_deg", DataType::Float64, true),
Field::new("x", DataType::Int32, false),
Field::new("y", DataType::Int32, false),
Field::new("repetition", repetition_struct_t, true),
Field::new("properties", property_list_t.clone(), true),
]));
let text_struct_t = DataType::Struct(Fields::from(vec![
Field::new("layer", DataType::UInt16, false),
Field::new("dtype", DataType::UInt16, false),
Field::new("presentation_horiz", DataType::UInt8, true),
Field::new("presentation_vert", DataType::UInt8, true),
Field::new("presentation_font", DataType::UInt8, true),
Field::new("path_type", DataType::Int16, true),
Field::new("width", DataType::Int32, true),
Field::new("invert_y", DataType::Boolean, true),
Field::new("mag", DataType::Float64, true),
Field::new("angle_deg", DataType::Float64, true),
Field::new("x", DataType::Int32, false),
Field::new("y", DataType::Int32, false),
Field::new("string", DataType::Utf8, false),
Field::new("properties", property_list_t.clone(), true),
]));
let coords_t = DataType::List(Arc::new(
Field::new_list_field(DataType::Int32, false)
));
let boundary_struct_t = DataType::Struct(Fields::from(vec![
Field::new("layer", DataType::UInt16, false),
Field::new("dtype", DataType::UInt16, false),
Field::new("xy", coords_t.clone(), false),
Field::new("properties", property_list_t.clone(), true),
]));
let path_struct_t = DataType::Struct(Fields::from(vec![
Field::new("layer", DataType::UInt16, false),
Field::new("dtype", DataType::UInt16, false),
Field::new("path_type", DataType::Int16, false),
Field::new("extension_start", DataType::Int32, true),
Field::new("extension_end", DataType::Int32, true),
Field::new("width", DataType::Int32, false),
Field::new("xy", coords_t.clone(), false),
Field::new("properties", property_list_t.clone(), true),
]));
let boxnode_struct_t = DataType::Struct(Fields::from(vec![
Field::new("layer", DataType::UInt16, false),
Field::new("dtype", DataType::UInt16, false),
Field::new("xy", coords_t.clone(), false),
Field::new("properties", property_list_t.clone(), true),
]));
let ref_list_t = DataType::List(Arc::new(
Field::new_list_field(ref_struct_t, false)
));
let text_list_t = DataType::List(Arc::new(
Field::new_list_field(text_struct_t, false)
));
let boundary_list_t = DataType::List(Arc::new(
Field::new_list_field(boundary_struct_t, false)
));
let path_list_t = DataType::List(Arc::new(
Field::new_list_field(path_struct_t, false)
));
let boxnode_list_t = DataType::List(Arc::new(
Field::new_list_field(boxnode_struct_t, false)
));
let cell_struct_t = DataType::Struct(Fields::from(vec![
Field::new("id", DataType::UInt32, false),
Field::new("file_offset", DataType::UInt64, false),
Field::new("refs", ref_list_t, false),
Field::new("boundaries", boundary_list_t, false),
Field::new("paths", path_list_t, false),
Field::new("nodes", boxnode_list_t.clone(), true),
Field::new("boxes", boxnode_list_t.clone(), true),
Field::new("texts", text_list_t, false),
]));
let mut lib_builder = StructBuilder::from_fields(vec![
Field::new("cell_names", DataType::Utf8, false),
Field::new("cells", cell_struct_t, false),
],
0,
);
let cells_builder = lib_builder.field_builder::<DListBuilder>(0).unwrap();
let mut names = HashMap::<String, u32>::new();
let (mut input, mut header) = RecordHeader::read(input)?;
while header.tag != records::RTAG_ENDLIB {
(input, _) = take_bytes(input, header.data_size)?;
if header.tag == records::RTAG_BGNSTR {
let name_bytes;
(input, name_bytes) = records::STRNAME::read(input)?;
let name = String::from_utf8(name_bytes).unwrap();
let next_id = names.len();
let id = names.entry(name).or_insert(next_id.try_into().unwrap());
let position = input_size - input.len();
let cell_builder = cells_builder.values().as_any_mut().downcast_mut::<StructBuilder>().unwrap();
let id_builder = cell_builder.field_builder::<UInt32Builder>(0).unwrap();
id_builder.append_value(*id);
let offset_builder = cell_builder.field_builder::<UInt64Builder>(1).unwrap();
offset_builder.append_value(position.try_into().unwrap());
(input, _) = read_elements(input, cell_builder, &mut names)?;
cells_builder.append(true);
}
(input, header) = RecordHeader::read(input)?;
}
let mut ids: HashMap<u32, String> = names.into_iter().map(|(kk, vv)| (vv, kk)).collect();
let names_builder = lib_builder.field_builder::<StringBuilder>(1).unwrap();
for id in 0..ids.len() {
names_builder.append_value(ids.remove(&id.try_into().unwrap()).unwrap());
}
let lib = lib_builder.finish();
Ok((input, lib))
}
pub fn read_elements<'a>(input: &'a [u8], cell_builder: &mut StructBuilder, names: &mut HashMap<String, u32>) -> IResult<'a, ()> {
let (mut input, mut header) = RecordHeader::read(input)?;
while header.tag != records::RTAG_ENDSTR {
match header.tag {
records::RTAG_BOUNDARY => {(input, _) = read_boundary(input, cell_builder)?;},
records::RTAG_PATH => {read_path(input, cell_builder)?;},
records::RTAG_NODE => {read_boxnode(input, cell_builder, header.tag)?;},
records::RTAG_BOX => {read_boxnode(input, cell_builder, header.tag)?;},
records::RTAG_TEXT => {read_text(input, cell_builder)?;},
records::RTAG_SREF => {read_ref(input, cell_builder, header.tag, names)?;},
records::RTAG_AREF => {read_ref(input, cell_builder, header.tag, names)?;},
_ => {
// don't care, skip
(input, _) = take_bytes(input, header.data_size)?;
}
}
(input, header) = RecordHeader::read(input)?;
}
Ok((input, ()))
}
pub fn read_boundary<'a>(input: &'a [u8], cell_builder: &mut StructBuilder) -> IResult<'a, ()> {
let boundaries_builder = cell_builder.field_builder::<DListBuilder>(3).unwrap();
let boundary_builder = boundaries_builder.values().as_any_mut().downcast_mut::<StructBuilder>().unwrap();
let (input, _) = records::BOUNDARY::read(input)?;
let (input, layer) = LAYER::skip_and_read(input)?;
let layer_builder = boundary_builder.field_builder::<Int16Builder>(0).unwrap();
layer_builder.append_value(layer);
let (input, dtype) = DATATYPE::read(input)?;
let dtype_builder = boundary_builder.field_builder::<Int16Builder>(1).unwrap();
dtype_builder.append_value(dtype);
let xys_builder = boundary_builder.field_builder::<DListBuilder>(2).unwrap();
let xy_builder = xys_builder.values().as_any_mut().downcast_mut::<Int32Builder>().unwrap();
let (input, xy_iter) = XY::read(input)?;
for xy in xy_iter {
xy_builder.append_value(xy);
}
xys_builder.append(true);
let props_builder = boundary_builder.field_builder::<DListBuilder>(3).unwrap();
let (input, ()) = read_properties(input, props_builder)?;
boundary_builder.append(true);
boundaries_builder.append(true);
Ok((input, ()))
}
pub fn read_path<'a>(input: &'a [u8], cell_builder: &mut StructBuilder) -> IResult<'a, ()> {
let paths_builder = cell_builder.field_builder::<DListBuilder>(4).unwrap();
let path_builder = paths_builder.values().as_any_mut().downcast_mut::<StructBuilder>().unwrap();
let (input, _) = records::PATH::read(input)?;
let (input, layer) = LAYER::skip_and_read(input)?;
let (input, dtype) = DATATYPE::read(input)?;
let layer_builder = path_builder.field_builder::<Int16Builder>(0).unwrap();
layer_builder.append_value(layer);
let dtype_builder = path_builder.field_builder::<Int16Builder>(1).unwrap();
dtype_builder.append_value(dtype);
let mut path_type = None;
let mut width = None;
let mut bgn_ext = None;
let mut end_ext = None;
let (mut input, mut header) = RecordHeader::read(&input)?;
while header.tag != records::RTAG_XY {
match header.tag {
records::RTAG_PATHTYPE => {
let _path_type;
(input, _path_type) = PATHTYPE::read_data(input, header.data_size)?;
path_type = Some(_path_type);
},
records::RTAG_WIDTH => {
let _width;
(input, _width) = WIDTH::read_data(input, header.data_size)?;
width = Some(_width);
},
records::RTAG_BGNEXTN => {
let _bgn_ext;
(input, _bgn_ext) = BGNEXTN::read_data(input, header.data_size)?;
bgn_ext = Some(_bgn_ext);
},
records::RTAG_ENDEXTN => {
let _end_ext;
(input, _end_ext) = ENDEXTN::read_data(input, header.data_size)?;
end_ext = Some(_end_ext);
},
_ =>
return fail(input, format!("Unexpected tag {:04x}", header.tag)),
};
(input, header) = RecordHeader::read(&input)?;
}
let path_type_builder = path_builder.field_builder::<Int16Builder>(2).unwrap();
path_type_builder.append_option(path_type);
let ext0_builder = path_builder.field_builder::<Int32Builder>(3).unwrap();
ext0_builder.append_option(bgn_ext);
let ext1_builder = path_builder.field_builder::<Int32Builder>(4).unwrap();
ext1_builder.append_option(end_ext);
let width_builder = path_builder.field_builder::<Int32Builder>(5).unwrap();
width_builder.append_option(width);
let xys_builder = path_builder.field_builder::<DListBuilder>(6).unwrap();
let (input, xy_iter) = XY::read(input)?;
for xy in xy_iter {
let xy_builder = xys_builder.values().as_any_mut().downcast_mut::<Int32Builder>().unwrap();
xy_builder.append_value(xy);
}
xys_builder.append(true);
let props_builder = path_builder.field_builder::<DListBuilder>(7).unwrap();
let (input, ()) = read_properties(input, props_builder)?;
path_builder.append(true);
paths_builder.append(true);
Ok((input, ()))
}
pub fn read_boxnode<'a>(input: &'a [u8], cell_builder: &mut StructBuilder, tag: u16) -> IResult<'a, ()> {
let field_num = match tag {
records::RTAG_NODE => 5,
records::RTAG_BOX => 6,
_ => return fail(input, format!("Unexpected tag {:04x}", tag)),
};
let boxnodes_builder = cell_builder.field_builder::<DListBuilder>(field_num).unwrap();
let boxnode_builder = boxnodes_builder.values().as_any_mut().downcast_mut::<StructBuilder>().unwrap();
let (input, _) = match tag {
records::RTAG_NODE => records::NODE::read(input)?,
records::RTAG_BOX => records::BOX::read(input)?,
_ => return fail(input, format!("Unexpected tag {:04x}", tag)),
};
let layer_builder = boxnode_builder.field_builder::<Int16Builder>(0).unwrap();
let (input, layer) = LAYER::skip_and_read(input)?;
layer_builder.append_value(layer);
let (input, dtype) = match tag {
records::RTAG_NODE => NODETYPE::read(input)?,
records::RTAG_BOX => BOXTYPE::read(input)?,
_ => return fail(input, format!("Unexpected tag {:04x}", tag)),
};
let dtype_builder = boxnode_builder.field_builder::<Int16Builder>(1).unwrap();
dtype_builder.append_value(dtype);
let xys_builder = boxnode_builder.field_builder::<DListBuilder>(2).unwrap();
let xy_builder = xys_builder.values().as_any_mut().downcast_mut::<Int32Builder>().unwrap();
let (input, xy_iter) = XY::read(input)?;
for xy in xy_iter {
xy_builder.append_value(xy);
}
xys_builder.append(true);
let props_builder = boxnode_builder.field_builder::<DListBuilder>(3).unwrap();
let (input, ()) = read_properties(input, props_builder)?;
boxnode_builder.append(true);
boxnodes_builder.append(true);
Ok((input, ()))
}
pub fn read_text<'a>(input: &'a [u8], cell_builder: &mut StructBuilder) -> IResult<'a, ()> {
let texts_builder = cell_builder.field_builder::<DListBuilder>(7).unwrap();
let text_builder = texts_builder.values().as_any_mut().downcast_mut::<StructBuilder>().unwrap();
let mut path_type = None;
let mut pres_hori = None;
let mut pres_vert = None;
let mut pres_font = None;
let mut invert_y = None;
let mut width = None;
let mut mag = None;
let mut angle_deg = None;
let (input, layer) = LAYER::skip_and_read(input)?;
let layer_builder = text_builder.field_builder::<Int16Builder>(0).unwrap();
layer_builder.append_value(layer);
let (input, dtype) = TEXTTYPE::read(input)?;
let dtype_builder = text_builder.field_builder::<Int16Builder>(1).unwrap();
dtype_builder.append_value(dtype);
let (mut input, mut header) = RecordHeader::read(input)?;
while header.tag != records::RTAG_XY {
match header.tag {
// TODO warn if repeat tags?
records::RTAG_PRESENTATION => {
let _presentation;
(input, _presentation) = PRESENTATION::read_data(input, header.data_size)?;
pres_hori = Some(_presentation[14] as u8 * 2 + _presentation[15] as u8);
pres_vert = Some(_presentation[12] as u8 * 2 + _presentation[13] as u8);
pres_font = Some(_presentation[10] as u8 * 2 + _presentation[11] as u8);
},
records::RTAG_PATHTYPE => {
let _path_type;
(input, _path_type) = PATHTYPE::read_data(input, header.data_size)?;
path_type = Some(_path_type);
},
records::RTAG_WIDTH => {
let _width;
(input, _width) = WIDTH::read_data(input, header.data_size)?;
width = Some(_width);
},
records::RTAG_STRANS => {
let strans;
(input, strans) = STRANS::read_data(input, header.data_size)?;
invert_y = Some(strans[0]);
},
records::RTAG_MAG => {
let _mag;
(input, _mag) = MAG::read_data(input, header.data_size)?;
mag = Some(_mag);
},
records::RTAG_ANGLE => {
let _angle_deg;
(input, _angle_deg) = ANGLE::read_data(input, header.data_size)?;
angle_deg = Some(_angle_deg);
},
_ =>
return fail(input, format!("Unexpected tag {:04x}", header.tag)),
}
(input, header) = RecordHeader::read(input)?;
}
let pres_hori_builder = text_builder.field_builder::<UInt8Builder>(2).unwrap();
pres_hori_builder.append_option(pres_hori);
let pres_vert_builder = text_builder.field_builder::<UInt8Builder>(3).unwrap();
pres_vert_builder.append_option(pres_vert);
let pres_font_builder = text_builder.field_builder::<UInt8Builder>(4).unwrap();
pres_font_builder.append_option(pres_font);
let path_type_builder = text_builder.field_builder::<Int16Builder>(5).unwrap();
path_type_builder.append_option(path_type);
let width_builder = text_builder.field_builder::<Int32Builder>(6).unwrap();
width_builder.append_option(width);
let inv_builder = text_builder.field_builder::<BooleanBuilder>(7).unwrap();
inv_builder.append_option(invert_y);
let mag_builder = text_builder.field_builder::<Float64Builder>(8).unwrap();
mag_builder.append_option(mag);
let angle_builder = text_builder.field_builder::<Float64Builder>(9).unwrap();
angle_builder.append_option(angle_deg);
let (input, mut xy_iter) = XY::read(input)?;
let x_builder = text_builder.field_builder::<Int32Builder>(10).unwrap();
x_builder.append_value(xy_iter.next().unwrap());
let y_builder = text_builder.field_builder::<Int32Builder>(11).unwrap();
y_builder.append_value(xy_iter.next().unwrap());
let (input, string_bytes) = STRING::read(input)?;
let string = String::from_utf8(string_bytes).unwrap();
let string_builder = text_builder.field_builder::<StringBuilder>(12).unwrap();
string_builder.append_value(string);
let props_builder = text_builder.field_builder::<DListBuilder>(13).unwrap();
let (input, ()) = read_properties(input, props_builder)?;
text_builder.append(true);
texts_builder.append(true);
Ok((input, ()))
}
pub fn read_ref<'a>(input: &'a [u8], cell_builder: &mut StructBuilder, tag: u16, names: &mut HashMap<String, u32>) -> IResult<'a, ()> {
let is_aref = tag == records::RTAG_AREF;
let refs_builder = cell_builder.field_builder::<DListBuilder>(7).unwrap();
let ref_builder = refs_builder.values().as_any_mut().downcast_mut::<StructBuilder>().unwrap();
let mut invert_y = None;
let mut mag = None;
let mut angle_deg = None;
let mut colrow = None;
let (input, struct_name_bytes) = SNAME::skip_and_read(input)?;
let struct_name = String::from_utf8(struct_name_bytes).unwrap();
let next_id = names.len();
let id = names.entry(struct_name).or_insert(next_id.try_into().unwrap());
let target_builder = ref_builder.field_builder::<UInt32Builder>(0).unwrap();
target_builder.append_value(*id);
let (mut input, mut header) = RecordHeader::read(input)?;
while header.tag != records::RTAG_XY {
match header.tag {
records::RTAG_STRANS => {
let strans;
(input, strans) = STRANS::read_data(input, header.data_size)?;
invert_y = Some(strans[0]);
},
records::RTAG_MAG => {
let _mag;
(input, _mag) = MAG::read_data(input, header.data_size)?;
mag = Some(_mag);
},
records::RTAG_ANGLE => {
let _angle_deg;
(input, _angle_deg) = ANGLE::read_data(input, header.data_size)?;
angle_deg = Some(_angle_deg);
},
records::RTAG_COLROW => {
let mut _colrow;
(input, _colrow) = COLROW::read_data(input, header.data_size)?;
colrow = Some((_colrow.next().unwrap(), _colrow.next().unwrap()));
if !is_aref {
return fail(input, "Got a COLROW record inside an SREF".to_string());
}
},
_ =>
return fail(input, format!("Unexpected tag {:04x}", header.tag)),
};
(input, header) = RecordHeader::read(input)?;
}
let inv_builder = ref_builder.field_builder::<BooleanBuilder>(1).unwrap();
inv_builder.append_option(invert_y);
let mag_builder = ref_builder.field_builder::<Float64Builder>(2).unwrap();
mag_builder.append_option(mag);
let angle_builder = ref_builder.field_builder::<Float64Builder>(3).unwrap();
angle_builder.append_option(angle_deg);
let (input, mut xy_iter) = XY::read_data(input, header.data_size)?;
let x_builder = ref_builder.field_builder::<Int32Builder>(4).unwrap();
x_builder.append_value(xy_iter.next().unwrap());
let y_builder = ref_builder.field_builder::<Int32Builder>(5).unwrap();
y_builder.append_value(xy_iter.next().unwrap());
let rep_builder = ref_builder.field_builder::<StructBuilder>(6).unwrap();
if is_aref {
let x0_builder = rep_builder.field_builder::<Int32Builder>(0).unwrap();
x0_builder.append_value(xy_iter.next().unwrap());
let y0_builder = rep_builder.field_builder::<Int32Builder>(1).unwrap();
y0_builder.append_value(xy_iter.next().unwrap());
let x1_builder = rep_builder.field_builder::<Int32Builder>(2).unwrap();
x1_builder.append_value(xy_iter.next().unwrap());
let y1_builder = rep_builder.field_builder::<Int32Builder>(3).unwrap();
y1_builder.append_value(xy_iter.next().unwrap());
match colrow {
None => return fail(input, "AREF without COLROW before XY".to_string()),
Some((count0, count1)) => {
let count0_builder = rep_builder.field_builder::<Int16Builder>(4).unwrap();
count0_builder.append_value(count0);
let count1_builder = rep_builder.field_builder::<Int16Builder>(5).unwrap();
count1_builder.append_value(count1);
},
}
}
rep_builder.append(is_aref);
let props_builder = ref_builder.field_builder::<DListBuilder>(7).unwrap();
let (input, ()) = read_properties(input, props_builder)?;
ref_builder.append(true);
refs_builder.append(true);
Ok((input, ()))
}
///
@ -30,26 +571,35 @@ use std::io::Write;
/// Returns:
/// propattr: -> propvalue mapping
///
pub fn read_properties(input: &[u8]) -> IResult<HashMap::<i16, Vec<u8>>> {
let mut properties = HashMap::new();
pub fn read_properties<'a>(input: &'a [u8], props_builder: &mut DListBuilder) -> IResult<'a, ()> {
let prop_builder = props_builder.values().as_any_mut().downcast_mut::<StructBuilder>().unwrap();
let (mut input, mut header) = RecordHeader::read(input)?;
while header.tag != ENDEL::tag() {
if header.tag == PROPATTR::tag() {
let result = PROPATTR::read_data(input, header.data_size)?;
input = result.0;
let key = result.1;
let result = PROPVALUE::read(input)?;
input = result.0;
let value = result.1;
assert!(!properties.contains_key(&key), "Duplicate property key: {}", key);
properties.insert(key, value);
let key;
let value_bytes;
(input, key) = PROPATTR::read_data(input, header.data_size)?;
(input, value_bytes) = PROPVALUE::read(input)?;
let value = String::from_utf8(value_bytes).unwrap();
//assert!(!properties.contains_key(&key), "Duplicate property key: {}", key);
let key_builder = prop_builder.field_builder::<Int16Builder>(0).unwrap();
key_builder.append_value(key);
let val_builder = prop_builder.field_builder::<StringBuilder>(1).unwrap();
val_builder.append_value(value);
prop_builder.append(true);
}
(input, header) = RecordHeader::read(input)?;
}
Ok((input, properties))
props_builder.append(true);
Ok((input, ()))
}
/*
///
/// Write element properties.
@ -530,3 +1080,4 @@ impl Element for Text {
Ok(size)
}
}
*/