Various target-gen optimizations
diff --git a/.vscode/launch.json b/.vscode/launch.json index ee01283..659e71d 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json
@@ -121,6 +121,64 @@ "debug", ], "cwd": "${workspaceFolder}/debugger" + }, + { + "type": "lldb", + "request": "launch", + "name": "target-gen Update supported ARM target definitions", + "cwd": "${workspaceFolder}/target-gen", + "cargo": { + "args": [ + "build", + "--bin=target-gen", + "--package=target-gen" + ], + "filter": { + "name": "target-gen", + "kind": "bin" + } + }, + "args": [ + "arm", + "${env:TMPDIR}target-gen" + ], + }, + { + "type": "lldb", + "request": "launch", + "name": "target-gen Generate target yaml from single PDSC file", + "cwd": "${workspaceFolder}/target-gen", + "cargo": { + "args": [ + "build", + "--bin=target-gen", + "--package=target-gen" + ], + "filter": { + "name": "target-gen", + "kind": "bin" + } + }, + "args": [ + "pack", + "${input:packFile}", + "${input:yamlFile}" + ], + }, + ], + "inputs": [ + { + "id": "packFile", + "type": "promptString", + "description": "Please enter the path to Pack file or the unzipped Pack directory.", + "default": "${workspaceFolder}/../pack_files/Keil.STM32H7xx_DFP.2.7.0.pack" + // "default": "${workspaceFolder}/../pack_files/NXP.LPC55S69_DFP.15.0.0.pack" + }, + { + "id": "yamlFile", + "type": "promptString", + "description": "Please enter the path to the output directory for yaml files.", + "default": "${workspaceFolder}/probe-rs/targets" } ] } \ No newline at end of file
diff --git a/debugger/Cargo.toml b/debugger/Cargo.toml index e231675..7b3e2e3 100644 --- a/debugger/Cargo.toml +++ b/debugger/Cargo.toml
@@ -33,9 +33,9 @@ parse_int = "0.6.0" num-traits = "0.2.14" bitfield = "0.14.0" -serde_json = "^1.0" -serde = { version = "^1.0", features = ["derive"] } -schemafy = "^0.6" +serde_json = "1" +serde = { version = "1", features = ["derive"] } +schemafy = "0.6" chrono = { version = "0.4", features = ["serde"] } goblin = "0.5.1" base64 = "0.13"
diff --git a/probe-rs-cli-util/Cargo.toml b/probe-rs-cli-util/Cargo.toml index c75ea70..8c3f2d7 100644 --- a/probe-rs-cli-util/Cargo.toml +++ b/probe-rs-cli-util/Cargo.toml
@@ -30,7 +30,7 @@ once_cell = "1.7.2" colored = "2.0.0" cargo_toml = "0.13.0" -serde = { version = "1.0.115", features = ["derive"] } +serde = { version = "1", features = ["derive"] } cargo_metadata = "0.15.0" dunce = "1.0.1" sentry = { version = "0.27.0", features = ["anyhow"], optional = true }
diff --git a/probe-rs-target/Cargo.toml b/probe-rs-target/Cargo.toml index fb7edf6..517988a 100644 --- a/probe-rs-target/Cargo.toml +++ b/probe-rs-target/Cargo.toml
@@ -11,14 +11,9 @@ keywords = ["embedded"] license = "MIT OR Apache-2.0" -[features] -# serde(skip_serializing_if) breaks with `bincode`, because it doesn't encode metadata (field names). -# Enable this if you want to serialize to bincode. Do not enable this if you want to serialize to other formats (YAML, JSON...) -# https://github.com/serde-rs/serde/issues/1732 -# https://github.com/bincode-org/bincode/issues/326 -bincode = [] - [dependencies] jep106 = "0.2.6" -serde = { version = "1.0.104", features = ["derive"] } +serde = { version = "1", features = ["derive"] } base64 = "0.13.0" +num-integer = "0.1" +num-traits = "0.2"
diff --git a/probe-rs-target/src/chip.rs b/probe-rs-target/src/chip.rs index 27cda54..69f92db 100644 --- a/probe-rs-target/src/chip.rs +++ b/probe-rs-target/src/chip.rs
@@ -1,7 +1,6 @@ use super::memory::MemoryRegion; -use crate::CoreType; +use crate::{serialize::hex_option, CoreType}; use serde::{Deserialize, Serialize}; - /// A single chip variant. /// /// This describes an exact chip variant, including the cores, flash and memory size. For example, @@ -14,12 +13,9 @@ pub name: String, /// The `PART` register of the chip. /// This value can be determined via the `cli info` command. - #[cfg_attr( - not(feature = "bincode"), - serde(skip_serializing_if = "Option::is_none") - )] pub part: Option<u16>, /// The cores available on the chip. + #[serde(default)] pub cores: Vec<Core>, /// The memory regions available on the chip. pub memory_map: Vec<MemoryRegion>, @@ -29,6 +25,7 @@ /// [`ChipFamily::flash_algorithms`] field. /// /// [`ChipFamily::flash_algorithms`]: crate::ChipFamily::flash_algorithms + #[serde(default)] pub flash_algorithms: Vec<String>, } @@ -84,9 +81,11 @@ pub psel: u32, /// The base address of the debug registers for the core. /// Required for Cortex-A, optional for Cortex-M + #[serde(serialize_with = "hex_option")] pub debug_base: Option<u64>, /// The base address of the cross trigger interface (CTI) for the core. /// Required in ARMv8-A + #[serde(serialize_with = "hex_option")] pub cti_base: Option<u64>, }
diff --git a/probe-rs-target/src/chip_family.rs b/probe-rs-target/src/chip_family.rs index b040781..c485000 100644 --- a/probe-rs-target/src/chip_family.rs +++ b/probe-rs-target/src/chip_family.rs
@@ -118,16 +118,11 @@ /// E.g. `nRF52832`. pub name: String, /// The JEP106 code of the manufacturer. - #[cfg_attr( - not(feature = "bincode"), - serde(skip_serializing_if = "Option::is_none") - )] pub manufacturer: Option<JEP106Code>, /// This vector holds all the variants of the family. pub variants: Vec<Chip>, /// This vector holds all available algorithms. pub flash_algorithms: Vec<RawFlashAlgorithm>, - #[serde(skip, default = "default_source")] /// Source of the target description, used for diagnostics pub source: TargetDescriptionSource,
diff --git a/probe-rs-target/src/flash_algorithm.rs b/probe-rs-target/src/flash_algorithm.rs index 8f19812..fde87f8 100644 --- a/probe-rs-target/src/flash_algorithm.rs +++ b/probe-rs-target/src/flash_algorithm.rs
@@ -1,5 +1,5 @@ use super::flash_properties::FlashProperties; - +use crate::serialize::{hex_option, hex_u_int}; use serde::{Deserialize, Serialize}; /// The raw flash algorithm is the description of a flash algorithm, @@ -21,22 +21,30 @@ #[serde(serialize_with = "serialize")] pub instructions: Vec<u8>, /// Address to load algo into RAM. Optional. + #[serde(serialize_with = "hex_option")] pub load_address: Option<u64>, /// Address of the `Init()` entry point. Optional. + #[serde(serialize_with = "hex_option")] pub pc_init: Option<u64>, /// Address of the `UnInit()` entry point. Optional. + #[serde(serialize_with = "hex_option")] pub pc_uninit: Option<u64>, /// Address of the `ProgramPage()` entry point. + #[serde(serialize_with = "hex_u_int")] pub pc_program_page: u64, /// Address of the `EraseSector()` entry point. + #[serde(serialize_with = "hex_u_int")] pub pc_erase_sector: u64, /// Address of the `EraseAll()` entry point. Optional. + #[serde(serialize_with = "hex_option")] pub pc_erase_all: Option<u64>, /// The offset from the start of RAM to the data section. + #[serde(serialize_with = "hex_u_int")] pub data_section_offset: u64, /// The properties of the flash on the device. pub flash_properties: FlashProperties, /// List of cores that can use this algorithm + #[serde(default)] pub cores: Vec<String>, }
diff --git a/probe-rs-target/src/flash_properties.rs b/probe-rs-target/src/flash_properties.rs index e51a58d..18a8cd9 100644 --- a/probe-rs-target/src/flash_properties.rs +++ b/probe-rs-target/src/flash_properties.rs
@@ -1,4 +1,5 @@ use super::memory::SectorDescription; +use crate::serialize::{hex_range, hex_u_int}; use serde::{Deserialize, Serialize}; use std::ops::Range; @@ -10,16 +11,20 @@ #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Hash)] pub struct FlashProperties { /// The range of the device flash. + #[serde(serialize_with = "hex_range")] pub address_range: Range<u64>, /// The page size of the device flash. + #[serde(serialize_with = "hex_u_int")] pub page_size: u32, /// The value of a byte in flash that was just erased. + #[serde(serialize_with = "hex_u_int")] pub erased_byte_value: u8, /// The approximative time it takes to program a page. pub program_page_timeout: u32, /// The approximative time it takes to erase a sector. pub erase_sector_timeout: u32, /// The available sectors of the device flash. + #[serde(default)] pub sectors: Vec<SectorDescription>, }
diff --git a/probe-rs-target/src/lib.rs b/probe-rs-target/src/lib.rs index a8ffd30..86af550 100644 --- a/probe-rs-target/src/lib.rs +++ b/probe-rs-target/src/lib.rs
@@ -16,6 +16,7 @@ mod flash_algorithm; mod flash_properties; mod memory; +pub(crate) mod serialize; pub use chip::{ArmCoreAccessOptions, Chip, Core, CoreAccessOptions, RiscvCoreAccessOptions}; pub use chip_family::{ @@ -24,5 +25,6 @@ pub use flash_algorithm::RawFlashAlgorithm; pub use flash_properties::FlashProperties; pub use memory::{ - MemoryRange, MemoryRegion, NvmRegion, PageInfo, RamRegion, SectorDescription, SectorInfo, + GenericRegion, MemoryRange, MemoryRegion, NvmRegion, PageInfo, RamRegion, SectorDescription, + SectorInfo, };
diff --git a/probe-rs-target/src/memory.rs b/probe-rs-target/src/memory.rs index 1a74c2b..51104c9 100644 --- a/probe-rs-target/src/memory.rs +++ b/probe-rs-target/src/memory.rs
@@ -1,3 +1,4 @@ +use crate::serialize::{hex_range, hex_u_int}; use core::ops::Range; use serde::{Deserialize, Serialize}; @@ -7,6 +8,7 @@ /// A name to describe the region pub name: Option<String>, /// Address range of the region + #[serde(serialize_with = "hex_range")] pub range: Range<u64>, /// True if the chip boots from this memory pub is_boot_memory: bool, @@ -29,6 +31,7 @@ /// A name to describe the region pub name: Option<String>, /// Address range of the region + #[serde(serialize_with = "hex_range")] pub range: Range<u64>, /// True if the chip boots from this memory pub is_boot_memory: bool, @@ -42,6 +45,7 @@ /// A name to describe the region pub name: Option<String>, /// Address range of the region + #[serde(serialize_with = "hex_range")] pub range: Range<u64>, /// List of cores that can access this region pub cores: Vec<String>, @@ -70,9 +74,11 @@ #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] pub struct SectorDescription { /// Size of each individual flash sector + #[serde(serialize_with = "hex_u_int")] pub size: u64, /// Start address of the group of flash sectors, relative /// to the start address of the flash. + #[serde(serialize_with = "hex_u_int")] pub address: u64, }
diff --git a/probe-rs-target/src/serialize/hex_option.rs b/probe-rs-target/src/serialize/hex_option.rs new file mode 100644 index 0000000..3ea8827 --- /dev/null +++ b/probe-rs-target/src/serialize/hex_option.rs
@@ -0,0 +1,20 @@ +use super::serialize_u_int::SerializeUnsignedInt; +use serde::{self, ser::Serializer, Serialize}; + +pub fn serialize<T, S>(variant_value: &Option<T>, serializer: S) -> Result<S::Ok, S::Error> +where + S: Serializer, + T: Serialize + std::fmt::LowerHex + SerializeUnsignedInt, +{ + match variant_value { + Some(val) => { + let check_for_human_readable = serializer.is_human_readable(); + if check_for_human_readable { + serializer.serialize_some(format!("{:#x}", val).as_str()) + } else { + serializer.serialize_some(&val) + } + } + None => serializer.serialize_none(), + } +}
diff --git a/probe-rs-target/src/serialize/hex_range.rs b/probe-rs-target/src/serialize/hex_range.rs new file mode 100644 index 0000000..9b11f55 --- /dev/null +++ b/probe-rs-target/src/serialize/hex_range.rs
@@ -0,0 +1,19 @@ +use serde::{self, ser::SerializeStruct, Serializer}; +use std::ops::Range; + +pub fn serialize<S>(memory_range: &Range<u64>, serializer: S) -> Result<S::Ok, S::Error> +where + S: Serializer, +{ + // We serialize the range as hex strings when generating human-readable formats such as YAML, + let check_for_human_readable = serializer.is_human_readable(); + let mut state = serializer.serialize_struct("Range", 2)?; + if check_for_human_readable { + state.serialize_field("start", format!("{:#x}", memory_range.start).as_str())?; + state.serialize_field("end", format!("{:#x}", memory_range.end).as_str())?; + } else { + state.serialize_field("start", &memory_range.start)?; + state.serialize_field("end", &memory_range.end)?; + } + state.end() +}
diff --git a/probe-rs-target/src/serialize/hex_u_int.rs b/probe-rs-target/src/serialize/hex_u_int.rs new file mode 100644 index 0000000..b8f6d41 --- /dev/null +++ b/probe-rs-target/src/serialize/hex_u_int.rs
@@ -0,0 +1,16 @@ +use super::serialize_u_int::SerializeUnsignedInt; +use serde::{self, Serializer}; + +pub(crate) fn serialize<T, S>(memory_address: &T, serializer: S) -> Result<S::Ok, S::Error> +where + S: Serializer, + T: std::fmt::LowerHex + SerializeUnsignedInt, +{ + // We serialize the range as hex strings when generating human-readable formats such as YAML, + let check_for_human_readable = serializer.is_human_readable(); + if check_for_human_readable { + serializer.serialize_str(format!("{:#x}", memory_address).as_str()) + } else { + memory_address.serialize_int(serializer) + } +}
diff --git a/probe-rs-target/src/serialize/mod.rs b/probe-rs-target/src/serialize/mod.rs new file mode 100644 index 0000000..4101596 --- /dev/null +++ b/probe-rs-target/src/serialize/mod.rs
@@ -0,0 +1,8 @@ +mod hex_option; +mod hex_range; +mod hex_u_int; +mod serialize_u_int; + +pub(crate) use hex_option::serialize as hex_option; +pub(crate) use hex_range::serialize as hex_range; +pub(crate) use hex_u_int::serialize as hex_u_int;
diff --git a/probe-rs-target/src/serialize/serialize_u_int.rs b/probe-rs-target/src/serialize/serialize_u_int.rs new file mode 100644 index 0000000..07089d5 --- /dev/null +++ b/probe-rs-target/src/serialize/serialize_u_int.rs
@@ -0,0 +1,44 @@ +use serde::Serializer; + +/// This trait is used by `probe-rs-target` to constrain the serialization of numbers to hex strings, to be generic for unsigned integers. +pub trait SerializeUnsignedInt { + fn serialize_int<S>(&self, serializer: S) -> Result<S::Ok, S::Error> + where + S: Serializer; +} + +impl SerializeUnsignedInt for u8 { + fn serialize_int<S>(&self, serializer: S) -> Result<S::Ok, S::Error> + where + S: Serializer, + { + serializer.serialize_u8(*self) + } +} + +impl SerializeUnsignedInt for u16 { + fn serialize_int<S>(&self, serializer: S) -> Result<S::Ok, S::Error> + where + S: Serializer, + { + serializer.serialize_u16(*self) + } +} + +impl SerializeUnsignedInt for u32 { + fn serialize_int<S>(&self, serializer: S) -> Result<S::Ok, S::Error> + where + S: Serializer, + { + serializer.serialize_u32(*self) + } +} + +impl SerializeUnsignedInt for u64 { + fn serialize_int<S>(&self, serializer: S) -> Result<S::Ok, S::Error> + where + S: Serializer, + { + serializer.serialize_u64(*self) + } +}
diff --git a/probe-rs/Cargo.toml b/probe-rs/Cargo.toml index 5023574..9175f6f 100644 --- a/probe-rs/Cargo.toml +++ b/probe-rs/Cargo.toml
@@ -54,8 +54,8 @@ ] } rusb = "0.9.0" scroll = "0.11.0" -serde = { version = "1.0.104", features = ["derive"] } -serde_yaml = "0.9.4" +serde = { version = "1", features = ["derive"] } +serde_yaml = "0.9" static_assertions = "1.1.0" svg = "0.10.0" thiserror = "1.0.10" @@ -65,15 +65,11 @@ libftdi1-sys = { version = "1.1.2", optional = true } # path -probe-rs-target = { path = "../probe-rs-target", version = "0.13.0", features = [ - "bincode", -] } +probe-rs-target = { path = "../probe-rs-target", version = "0.13.0" } [build-dependencies] bincode = "1.3.2" -probe-rs-target = { path = "../probe-rs-target", version = "0.13.0", features = [ - "bincode", -] } +probe-rs-target = { path = "../probe-rs-target", version = "0.13.0" } serde_yaml = "0.9.4" [dev-dependencies] @@ -82,6 +78,6 @@ rand = "0.8.0" reqwest = { version = "0.11.0", features = ["blocking", "json"] } serde_json = "1.0.47" -serde = "1.0.118" +serde = "1" clap = { version = "4.0", features = ["derive"] } itm-decode = { version = "0.6.1", default-features = false }
diff --git a/probe-rs/build.rs b/probe-rs/build.rs index 2ccd081..389a56a 100644 --- a/probe-rs/build.rs +++ b/probe-rs/build.rs
@@ -40,7 +40,13 @@ let dest_path = Path::new(&out_dir).join("targets.bincode"); std::fs::write(dest_path, &families_bin).unwrap(); - let _: Vec<ChipFamily> = bincode::deserialize(&families_bin).unwrap(); + let _: Vec<ChipFamily> = match bincode::deserialize(&families_bin) { + Ok(chip_families) => chip_families, + Err(deserialize_error) => panic!( + "Failed to deserialize supported target definitions from bincode: {:?}", + deserialize_error + ), + }; } /// One possible implementation of walking a directory only visiting files.
diff --git a/probe-rs/src/config/mod.rs b/probe-rs/src/config/mod.rs index f7c7d51..6dfd010 100644 --- a/probe-rs/src/config/mod.rs +++ b/probe-rs/src/config/mod.rs
@@ -27,8 +27,8 @@ mod target; pub use probe_rs_target::{ - Chip, ChipFamily, Core, CoreType, FlashProperties, InstructionSet, MemoryRange, MemoryRegion, - NvmRegion, PageInfo, RamRegion, RawFlashAlgorithm, SectorDescription, SectorInfo, + Chip, ChipFamily, Core, CoreType, FlashProperties, GenericRegion, InstructionSet, MemoryRange, + MemoryRegion, NvmRegion, PageInfo, RamRegion, RawFlashAlgorithm, SectorDescription, SectorInfo, TargetDescriptionSource, };
diff --git a/probe-rs/src/config/registry.rs b/probe-rs/src/config/registry.rs index 19df8d9..dfe7a86 100644 --- a/probe-rs/src/config/registry.rs +++ b/probe-rs/src/config/registry.rs
@@ -114,8 +114,13 @@ fn from_builtin_families() -> Self { const BUILTIN_TARGETS: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/targets.bincode")); - let mut families: Vec<ChipFamily> = bincode::deserialize(BUILTIN_TARGETS) - .expect("Failed to deserialize builtin targets. This is a bug."); + let mut families: Vec<ChipFamily> = match bincode::deserialize(BUILTIN_TARGETS) { + Ok(families) => families, + Err(err) => panic!( + "Failed to deserialize builtin targets. This is a bug : {:?}", + err + ), + }; add_generic_targets(&mut families);
diff --git a/rtt/Cargo.toml b/rtt/Cargo.toml index 8744578..f0286f0 100644 --- a/rtt/Cargo.toml +++ b/rtt/Cargo.toml
@@ -13,5 +13,5 @@ log = "0.4.8" probe-rs = { version = "0.13.0", path = "../probe-rs" } scroll = "0.10.1" -serde = { version = "1.0", features = ["derive"] } +serde = { version = "1", features = ["derive"] } thiserror = "1.0.11"
diff --git a/smoke-tester/Cargo.toml b/smoke-tester/Cargo.toml index 654d955..e9f83d0 100644 --- a/smoke-tester/Cargo.toml +++ b/smoke-tester/Cargo.toml
@@ -12,7 +12,7 @@ probe-rs = { path = "../probe-rs" } toml = "0.5.8" anyhow = "1.0.40" -serde = { version = "1.0.125", features = ["derive"] } +serde = { version = "1", features = ["derive"] } pretty_env_logger = "0.4.0" log = "0.4.14" clap = "4.0"
diff --git a/target-gen/Cargo.toml b/target-gen/Cargo.toml index fd3398d..f8fff57 100644 --- a/target-gen/Cargo.toml +++ b/target-gen/Cargo.toml
@@ -15,16 +15,12 @@ # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -probe-rs = { path = "../probe-rs", version = "0.13.0", default-features = false } +probe-rs = { path = "../probe-rs", version = "0.13.0", default-features = true } probe-rs-target = { path = "../probe-rs-target", version = "0.13.0", default-features = false } -cmsis-pack = { version = "0.6.0" } - -# , path = "../cmsis-pack-manager/rust/cmsis-pack" -# , git = "https://github.com/probe-rs/cmsis-pack-manager.git" - +cmsis-pack = { version = "0.6", git = "https://github.com/pyocd/cmsis-pack-manager" } goblin = "0.5.1" scroll = "0.11.0" -serde_yaml = "0.9.0" +serde_yaml = "^0.9" log = "0.4.16" zip = "0.6.2" clap = { version = "4.0", features = ["derive"] } @@ -32,8 +28,7 @@ simplelog = "0.12.0" anyhow = "1.0.57" reqwest = { version = "0.11.10", features = ["json", "blocking"] } -serde = { version = "1.0.136", features = ["derive"] } -# chrono = { version = "0.4.0", features = ["serde"] } +serde = { version = "1", features = ["derive"] } futures = "0.3.21" tokio = { version = "1.18.0", features = ["macros", "rt", "rt-multi-thread"] }
diff --git a/target-gen/src/generate.rs b/target-gen/src/generate.rs index 7ae8a81..87e3ff6 100644 --- a/target-gen/src/generate.rs +++ b/target-gen/src/generate.rs
@@ -7,7 +7,8 @@ use cmsis_pack::{pack_index::PdscRef, utils::FromElem}; use futures::StreamExt; use probe_rs::config::{ - Chip, ChipFamily, Core as ProbeCore, MemoryRegion, NvmRegion, RamRegion, RawFlashAlgorithm, + Chip, ChipFamily, Core as ProbeCore, GenericRegion, MemoryRegion, NvmRegion, RamRegion, + RawFlashAlgorithm, }; use probe_rs::{Architecture, CoreType}; use probe_rs_target::{ArmCoreAccessOptions, CoreAccessOptions, RiscvCoreAccessOptions}; @@ -25,6 +26,7 @@ pdsc: Package, mut kind: Kind<T>, families: &mut Vec<ChipFamily>, + only_supported_familes: bool, ) -> Result<()> where T: std::io::Seek + std::io::Read, @@ -34,6 +36,24 @@ devices.sort_by(|a, b| a.0.cmp(&b.0)); for (device_name, device) in devices { + // Only process this, if this belongs to a supported family. + let currently_supported_chip_families = probe_rs::config::families().map_err(|e| { + anyhow!( + "Currently supported chip families could not be read: {:?}", + e + ) + })?; + + if only_supported_familes + && !currently_supported_chip_families + .iter() + .any(|supported_family| supported_family.name == device.family) + { + // We only want to continue if the chip family is already represented as supported probe_rs target chip family. + log::debug!("Unsupprted chip family {}. Skipping ...", device.family); + return Ok(()); + } + // Check if this device family is already known. let mut potential_family = families .iter_mut() @@ -53,9 +73,6 @@ families.last_mut().unwrap() }; - // Extract the RAM info from the .pdsc file. - let ram = get_ram(&device); - // Extract the flash algorithm, block & sector size and the erased byte value from the ELF binary. let variant_flash_algorithms = device .algorithms @@ -97,13 +114,6 @@ ) .collect::<Vec<_>>(); - // Extract the flash info from the .pdsc file. - let flash = get_flash(&device); - - if device.processors.len() > 1 { - println!("{:#?}", device.processors); - } - let flash_algorithm_names: Vec<_> = variant_flash_algorithms .iter() .map(|fa| fa.name.to_string()) @@ -118,14 +128,6 @@ .map(|(_, s)| s.clone()) .collect(); - let mut memory_map: Vec<MemoryRegion> = Vec::new(); - if let Some(mem) = ram { - memory_map.push(MemoryRegion::Ram(mem)); - } - if let Some(mem) = flash { - memory_map.push(MemoryRegion::Nvm(mem)); - } - let cores = device .processors .iter() @@ -136,7 +138,7 @@ name: device_name, part: None, cores, - memory_map, + memory_map: get_mem_map(&device), flash_algorithms: flash_algorithm_names, }); } @@ -197,6 +199,7 @@ Package::from_path(&entry.path())?, Kind::Directory(path), families, + false, ) .context(format!( "Failed to process .pdsc file {}.", @@ -213,7 +216,6 @@ log::info!("Trying to open pack file: {}.", path.display()); // If we get a file, try to unpack it. let file = fs::File::open(&path)?; - let mut archive = zip::ZipArchive::new(file)?; let mut pdsc_file = find_pdsc_in_archive(&mut archive)? @@ -233,11 +235,13 @@ drop(pdsc_file); - handle_package(package, Kind::Archive(&mut archive), families) + handle_package(package, Kind::Archive(&mut archive), families, false) } pub(crate) fn visit_arm_files(families: &mut Vec<ChipFamily>) -> Result<()> { let packs = crate::fetch::get_vidx()?; + + //TODO: The multi-threaded logging makes it very difficult to track which errors/warnings belong where - needs some rework. Builder::new_multi_thread() .enable_all() .build() @@ -246,6 +250,7 @@ let mut stream = futures::stream::iter(packs.pdsc_index.iter().enumerate().filter_map( |(i, pack)| { if pack.deprecated.is_none() { + // We only want to download the pack if it is not deprecated. log::info!("Working PACK {}/{} ...", i, packs.pdsc_index.len()); Some(visit_arm_file(pack)) } else { @@ -334,12 +339,16 @@ } }; + let pdsc_name = pdsc_file.name().to_owned(); + drop(pdsc_file); let mut families = vec![]; - match handle_package(package, Kind::Archive(&mut archive), &mut families) { - Ok(_) => {} + match handle_package(package, Kind::Archive(&mut archive), &mut families, true) { + Ok(_) => { + log::info!("Handled package {}", pdsc_name); + } Err(err) => log::error!("Something went wrong while handling pack {}: {}", url, err), }; @@ -383,79 +392,138 @@ } } -// Clippy complains about `region.range.start == cur.range.end`, but that is correct ;). -#[allow(clippy::suspicious_operation_groupings)] -pub(crate) fn get_ram(device: &Device) -> Option<RamRegion> { - let mut regions: Vec<RamRegion> = Vec::new(); - for memory in device.memories.0.values() { - if memory.default && memory.access.read && memory.access.write { - regions.push(RamRegion { - range: memory.start..memory.start + memory.size, - is_boot_memory: memory.startup, - cores: vec!["main".to_owned()], - name: None, - }); - } - } - if regions.len() > 1 { - // Sort by start address - regions.sort_by_key(|r| r.range.start); - let mut merged: Vec<RamRegion> = Vec::new(); - let mut cur = regions.first().cloned().unwrap(); - for region in regions.iter().skip(1) { - if region.is_boot_memory == cur.is_boot_memory && region.range.start == cur.range.end { - // Merge with previous region - cur.range.end = region.range.end; - } else { - merged.push(cur); - cur = region.clone(); - } - } - merged.push(cur); - regions = merged; - - // Sort by region size - regions.sort_by_key(|r| r.range.end - r.range.start) - } - - regions.last().cloned() +/// A flag to indicate what type of memory this is. +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +enum MemoryType { + /// A RAM memory. + Ram, + /// A Non Volatile memory. + Nvm, + /// Generic + Generic, } -pub(crate) fn get_flash(device: &Device) -> Option<NvmRegion> { - // Make a Vec of all memories which are flash-like - let mut regions = Vec::new(); - for memory in device.memories.0.values() { - if memory.default && memory.access.read && memory.access.execute && !memory.access.write { - regions.push(NvmRegion { - range: memory.start..memory.start + memory.size, - is_boot_memory: memory.startup, - cores: vec!["main".to_owned()], - name: None, - }); - } - } +/// A struct to combine essential information from [`cmsis_pack::pdsc::device::Device::Memories`]. +/// This is used to apply the necessary sorting and filtering in creating [`MemoryRegion`]s. +// The sequence of the fields is important for the sorting by derived natural order. +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +struct DeviceMemory { + memory_type: MemoryType, + p_name: Option<String>, + is_boot_memory: bool, + memory_start: u64, + memory_end: u64, + name: String, +} - if regions.len() > 1 { - // Sort by start address - regions.sort_by_key(|r| r.range.start); +// From PR's https://github.com/probe-rs/target-gen/pull/20 and https://github.com/probe-rs/target-gen/pull/25: +// TODO: What is the logic that justifies PR#20 selecting the largest memory? Shouldn't we match flash algo's with RAM based on load_address? +// Flash and RAM regions are not guaranteed to be sorted in the PDSC file, so we: +// - Sort them here. +// - Merge contiguous regions. +// Update: For multiple cores, we have to take processor access into account during this merge. +/// Sorts the memory regions in the package and merges contiguous regions with the same attributes. +pub(crate) fn get_mem_map(device: &Device) -> Vec<MemoryRegion> { + let mut device_memories: Vec<DeviceMemory> = device + .memories + .0 + .iter() + .map(|(name, memory)| DeviceMemory { + name: name.clone(), + p_name: memory.p_name.clone(), + memory_type: if memory.default && memory.access.read && memory.access.write { + MemoryType::Ram + } else if memory.default + && memory.access.read + && memory.access.execute + && !memory.access.write + { + MemoryType::Nvm + } else { + MemoryType::Generic + }, + memory_start: memory.start, + memory_end: memory.start + memory.size, + is_boot_memory: memory.startup, + }) + .collect(); - // Merge contiguous flash regions - let mut merged = Vec::new(); - let mut cur = regions.first().cloned().unwrap(); + // Merge memory regions with the same attributes. + if device_memories.len() > 1 { + // Sort by memory type, then by processor name, then by boot memory, then by start address. + device_memories.sort(); - for region in regions.iter().skip(1) { - if region.range.start == cur.range.end { - cur.range.end = region.range.end; + let mut merged: Vec<DeviceMemory> = Vec::new(); + let mut cur = device_memories.first().cloned().unwrap(); + for region in device_memories.iter().skip(1) { + if region.is_boot_memory == cur.is_boot_memory && region.memory_start == cur.memory_end + { + // Merge with previous region. + cur.memory_end = region.memory_end; + cur.name = format!("{} + {}", cur.name, region.name); } else { merged.push(cur); cur = region.clone(); } } - merged.push(cur); - regions = merged; + device_memories = merged; } - // Return lowest-addressed region - regions.first().cloned() + // Finally, sort so that the LARGEST contiguous region is first for each core. + device_memories.sort_by_cached_key(|region| { + ( + region.memory_type.clone(), + region.p_name.clone(), + (region.memory_start as i128 - region.memory_end as i128), + ) + }); + + // Convert DeviceMemory's to MemoryRegion's, and assign cores to shared reqions. + let mut mem_map = vec![]; + for region in &device_memories { + let current_core = region + .p_name + .as_ref() + .map(|s| s.to_ascii_lowercase()) + .unwrap_or_else(|| "main".to_string()); + match region.memory_type { + MemoryType::Ram => if let Some(MemoryRegion::Ram(existing_region)) = mem_map.iter_mut().find(|existing_region|{ + matches!(existing_region, MemoryRegion::Ram(ram_region) if ram_region.name == Some(region.name.clone()))}) + { + existing_region.cores.push(current_core); + } else { + mem_map.push(MemoryRegion::Ram(RamRegion { + name: Some(region.name.clone()), + range: region.memory_start..region.memory_end, + is_boot_memory: region.is_boot_memory, + cores: vec![current_core], + })); + }, + MemoryType::Nvm => if let Some(MemoryRegion::Nvm(existing_region)) = mem_map.iter_mut().find(|existing_region|{ + matches!(existing_region, MemoryRegion::Nvm(nvm_region) if nvm_region.name == Some(region.name.clone()))}) + { + existing_region.cores.push(current_core); + } else { + mem_map.push(MemoryRegion::Nvm(NvmRegion { + name: Some(region.name.clone()), + range: region.memory_start..region.memory_end, + is_boot_memory: region.is_boot_memory, + cores: vec![current_core], + })); + }, + MemoryType::Generic => if let Some(MemoryRegion::Generic(existing_region)) = mem_map.iter_mut().find(|existing_region|{ + matches!(existing_region, MemoryRegion::Generic(generic_region) if generic_region.name == Some(region.name.clone()))}) + { + existing_region.cores.push(current_core); + } else { + mem_map.push(MemoryRegion::Generic(GenericRegion { + name: Some(region.name.clone()), + range: region.memory_start..region.memory_end, + cores: vec![current_core], + })); + }, + }; + } + mem_map }
diff --git a/target-gen/src/main.rs b/target-gen/src/main.rs index 5965978..7c15b13 100644 --- a/target-gen/src/main.rs +++ b/target-gen/src/main.rs
@@ -6,7 +6,7 @@ use std::{ fs::{create_dir, File, OpenOptions}, - io::Write, + io::{BufRead, Write}, path::{Path, PathBuf}, }; @@ -42,6 +42,8 @@ output_dir: PathBuf, }, /// Generates all the target descriptions from the entries listed in the ARM root VIDX/PIDX at <https://www.keil.com/pack/Keil.pidx>. + /// This will only generate target descriptions for chip families that are already supported by probe-rs, to avoid generating a lot of unsupportable chip families. + /// Please use the `pack` subcommand to generate target descriptions for other chip families. Arm { #[clap( name = "OUTPUT", @@ -149,8 +151,7 @@ } let target_description = File::create(&target_description_file)?; - - serde_yaml::to_writer(&target_description, &family)?; + serialize_to_yaml_file(&family, &target_description)?; } else { // Create a complete target specification, with place holder values let algorithm_name = algorithm.name.clone(); @@ -192,12 +193,10 @@ source: BuiltIn, }; - let serialized = serde_yaml::to_string(&chip_family)?; - match output { Some(output) => { // Ensure we don't overwrite an existing file - let mut file = OpenOptions::new() + let file = OpenOptions::new() .write(true) .create_new(true) .open(&output) @@ -205,10 +204,9 @@ "Failed to create target file '{}'.", output.display() ))?; - - file.write_all(serialized.as_bytes())?; + serialize_to_yaml_file(&chip_family, &file)?; } - None => println!("{}", serialized), + None => println!("{}", serde_yaml::to_string(&chip_family)?), } } @@ -253,10 +251,11 @@ let mut generated_files = Vec::with_capacity(families.len()); for family in &families { - let path = out_dir.join(family.name.clone() + ".yaml"); + let path = out_dir.join(family.name.clone().replace(' ', "_") + ".yaml"); let file = std::fs::File::create(&path) .context(format!("Failed to create file '{}'.", path.display()))?; - serde_yaml::to_writer(file, &family)?; + + serialize_to_yaml_file(family, &file)?; generated_files.push(path); } @@ -270,6 +269,31 @@ Ok(()) } +/// Some optimizations to improve the readability of the `serde_yaml` output: +/// - If `Option<T>` is `None`, it is serialized as `null` ... we want to omit it. +/// - If `Vec<T>` is empty, it is serialized as `[]` ... we want to omit it. +/// - `serde_yaml` serializes hex formatted integers as single quoted strings, e.g. '0x1234' ... we need to remove the single quotes so that it round-trips properly. +fn serialize_to_yaml_file(family: &ChipFamily, file: &File) -> Result<(), anyhow::Error> { + let yaml_string = serde_yaml::to_string(&family)?; + let mut reader = std::io::BufReader::new(yaml_string.as_bytes()); + let mut reader_line = String::new(); + let mut writer = std::io::BufWriter::new(file); + Ok(while reader.read_line(&mut reader_line)? > 0 { + if reader_line.ends_with(": null\n") || reader_line.ends_with(": []\n") { + // Skip the line + } else if (reader_line.contains("'0x") || reader_line.contains("'0X")) + && reader_line.ends_with("'\n") + { + reader_line = reader_line.replace("'", ""); + writer.write(reader_line.as_bytes())?; + // Remove the single quotes + } else { + writer.write(reader_line.as_bytes())?; + } + reader_line.clear(); + }) +} + /// Handle the arm subcommand. /// Generated target descriptions will be placed in `out_dir`. fn cmd_arm(out_dir: &Path) -> Result<()> { @@ -287,10 +311,10 @@ let mut generated_files = Vec::with_capacity(families.len()); for family in &families { - let path = out_dir.join(family.name.clone() + ".yaml"); + let path = out_dir.join(family.name.clone().replace(' ', "_") + ".yaml"); let file = std::fs::File::create(&path) .context(format!("Failed to create file '{}'.", path.display()))?; - serde_yaml::to_writer(file, &family)?; + serialize_to_yaml_file(&family, &file)?; generated_files.push(path); }