Extract manifest target metadata
This parses target metadata from manifest files into an in-memory
representation, verifying its validity in the process.
Change-Id: I71a060126f92d20e641f7800c76da8361ad0b847
Reviewed-on: https://pigweed-review.googlesource.com/c/pigweed/qg/+/125130
Reviewed-by: Erik Gilling <konkers@google.com>
Commit-Queue: Alexei Frolov <frolv@google.com>
diff --git a/examples/simple_project/qg.toml b/examples/simple_project/qg.toml
index 3b41715..7a9d84c 100644
--- a/examples/simple_project/qg.toml
+++ b/examples/simple_project/qg.toml
@@ -6,6 +6,8 @@
namespace = "global"
url = "https://chrome-infra-packages.appspot.com/client?platform={platform}&version={version}"
url_parameters = { version = "git_revision:97c6e32ecb4f25a703bbe23aa459ea57886464c2" }
+format = "bin"
+bin_name = "cipd_client"
[[targets.cipd.variants]]
match = { os = "linux", arch = "x64" }
@@ -18,3 +20,17 @@
[[targets.cipd.variants]]
match = { os = "windows", arch = "x64" }
url_parameters = { platform = "windows-amd64" }
+
+# TODO(frolv): Support archive downloads.
+
+# [targets.arm-none-eabi-toolchain]
+# type = "download"
+# namespace = "global"
+# url = "https://developer.arm.com/-/media/Files/downloads/gnu/11.3.rel1/binrel/{file}?rev={rev}"
+# url_parameters = { rev = "0f93cc5b9df1473dabc1f39b06feb468&hash=7DF6BEF69DFDF7226B812B30BF45F552" }
+# format = "txz"
+# bin_dirs = ["bin"]
+
+# [[targets.arm-none-eabi-toolchain.variants]]
+# match = { os = "macos", arch = "x64" }
+# url_parameters = { file = "arm-gnu-toolchain-11.3.rel1-darwin-x86_64-arm-none-eabi.tar.xz" }
diff --git a/qg/Cargo.toml b/qg/Cargo.toml
index dd2a817..182b32f 100644
--- a/qg/Cargo.toml
+++ b/qg/Cargo.toml
@@ -10,6 +10,8 @@
[dependencies]
futures = "0.3.25"
num-traits = "0.2.15"
+once_cell = "1.16.0"
+regex = "1.7.0"
serde = { version = "1.0.147", features = ["derive"] }
thiserror = "1.0.37"
tokio = { version = "1.21.2", features = ["full"] }
diff --git a/qg/src/download.rs b/qg/src/download.rs
new file mode 100644
index 0000000..9f0d645
--- /dev/null
+++ b/qg/src/download.rs
@@ -0,0 +1,23 @@
+// Copyright 2022 The Pigweed Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not
+// use this file except in compliance with the License. You may obtain a copy of
+// the License at
+//
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations under
+// the License.
+
+use std::path::PathBuf;
+
+/// The file format of a download target.
+#[derive(Debug)]
+pub enum Format {
+ /// An uncompressed binary file, storing the relative path to which it
+ /// should be downloaded.
+ Binary(PathBuf),
+}
diff --git a/qg/src/lib.rs b/qg/src/lib.rs
index 8c7c509..ab1be63 100644
--- a/qg/src/lib.rs
+++ b/qg/src/lib.rs
@@ -18,10 +18,13 @@
use std::path::{Path, PathBuf};
+pub mod platform;
pub mod project;
pub mod registry;
pub mod target;
+mod download;
+
#[doc(inline)]
pub use target::Target;
@@ -54,6 +57,11 @@
#[error("invalid path")]
InvalidPath,
+
+ // TODO(frolv): This is a placeholder error returned by some functions until
+ // a real error handling model is figured out.
+ #[error("placeholder error, please bug the qg team")]
+ GenericErrorPlaceholder,
}
impl Error {
diff --git a/qg/src/platform.rs b/qg/src/platform.rs
new file mode 100644
index 0000000..6925311
--- /dev/null
+++ b/qg/src/platform.rs
@@ -0,0 +1,55 @@
+// Copyright 2022 The Pigweed Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not
+// use this file except in compliance with the License. You may obtain a copy of
+// the License at
+//
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations under
+// the License.
+
+use std::str::FromStr;
+
+use crate::Error;
+
+#[derive(Debug)]
+pub enum System {
+ Linux,
+ MacOs,
+ Windows,
+}
+
+impl FromStr for System {
+ type Err = Error;
+
+ fn from_str(s: &str) -> Result<Self, Self::Err> {
+ match s {
+ "linux" => Ok(Self::Linux),
+ "macos" => Ok(Self::MacOs),
+ "windows" => Ok(Self::Windows),
+ _ => Err(Error::GenericErrorPlaceholder),
+ }
+ }
+}
+
+#[derive(Debug)]
+pub enum Architecture {
+ X64,
+ Arm64,
+}
+
+impl FromStr for Architecture {
+ type Err = Error;
+
+ fn from_str(s: &str) -> Result<Self, Self::Err> {
+ match s {
+ "x64" => Ok(Self::X64),
+ "arm64" => Ok(Self::Arm64),
+ _ => Err(Error::GenericErrorPlaceholder),
+ }
+ }
+}
diff --git a/qg/src/project/manifest.rs b/qg/src/project/manifest.rs
index 0d55272..bc57b14 100644
--- a/qg/src/project/manifest.rs
+++ b/qg/src/project/manifest.rs
@@ -147,6 +147,15 @@
/// order of definition, with the first match being applied.
#[serde(default)]
pub variants: Vec<DownloadVariants>,
+
+ /// File format of the downloaded file. One of:
+ ///
+ /// - `bin`: An uncompressed executable file.
+ ///
+ pub format: Option<String>,
+
+ // For a `bin` format, what to rename the downloaded binary.
+ pub bin_name: Option<PathBuf>,
}
/// An override of the default `url_parameters` for a downloadable package.
diff --git a/qg/src/project/mod.rs b/qg/src/project/mod.rs
index 06ecfd1..430eee6 100644
--- a/qg/src/project/mod.rs
+++ b/qg/src/project/mod.rs
@@ -199,7 +199,7 @@
&root_manifest.project.name,
target.namespace.is_global(),
target,
- ));
+ )?);
}
for (provider, desc) in &root_manifest.providers {
@@ -225,7 +225,7 @@
provider,
global_provider || target.namespace.is_global(),
target,
- ));
+ )?);
}
}
}
diff --git a/qg/src/target.rs b/qg/src/target.rs
index a746cd3..af1b105 100644
--- a/qg/src/target.rs
+++ b/qg/src/target.rs
@@ -12,9 +12,14 @@
// License for the specific language governing permissions and limitations under
// the License.
+use std::collections::{HashMap, HashSet};
use std::path::{Path, PathBuf};
+use once_cell::sync::Lazy;
+use regex::Regex;
+
use crate::project::manifest;
+use crate::{download, platform, Error, Result};
/// A source of targets.
#[derive(Debug)]
@@ -41,7 +46,7 @@
}
}
-/// An buildable `qg` target.
+/// A buildable `qg` target.
#[derive(Debug)]
pub struct Target {
/// The target's name, without a provider namespace.
@@ -65,20 +70,161 @@
/// Additional information about how to build a target specific to a type
/// of provider.
#[derive(Debug)]
-enum Metadata {
+pub enum Metadata {
DepOnly,
-
- // TODO(frolv): These shouldn't store the manifest data directly but convert
- // it into a more usable in-memory format.
- Cipd(manifest::CipdPackage),
- Download(manifest::DownloadablePackage),
+ Cipd(Cipd),
+ Download(Download),
}
-impl From<manifest::TargetType> for Metadata {
- fn from(tt: manifest::TargetType) -> Self {
- match tt {
- manifest::TargetType::Cipd(data) => Self::Cipd(data),
- manifest::TargetType::Download(data) => Self::Download(data),
+#[derive(Debug)]
+pub struct Cipd {
+ pub path: String,
+ pub platforms: Vec<String>,
+ pub tags: Vec<String>,
+ pub subdir: Option<PathBuf>,
+}
+
+impl From<manifest::CipdPackage> for Cipd {
+ fn from(value: manifest::CipdPackage) -> Self {
+ // TODO(frolv): This should do some verification.
+ Self {
+ path: value.path,
+ platforms: value.platforms,
+ tags: value.tags,
+ subdir: value.subdir,
+ }
+ }
+}
+
+#[derive(Debug)]
+pub struct Download {
+ pub format: download::Format,
+ pub url: String,
+ pub url_parameters: HashMap<String, String>,
+ pub variants: Vec<DownloadVariant>,
+}
+
+#[derive(Debug)]
+pub struct DownloadVariant {
+ pub matches: VariantMatch,
+ url_parameters: HashMap<String, String>,
+}
+
+#[derive(Debug)]
+pub struct VariantMatch {
+ system: Option<platform::System>,
+ arch: Option<platform::Architecture>,
+}
+
+static URL_PARAMETER_NAME_REGEX: Lazy<Regex> =
+ Lazy::new(|| Regex::new(r"^[a-zA-Z_][a-zA-Z0-9_]*$").expect("regex is valid"));
+static URL_PARAMETER_SUB_REGEX: Lazy<Regex> =
+ Lazy::new(|| Regex::new(r"\{(.*?)\}").expect("regex is valid"));
+
+impl TryFrom<manifest::DownloadablePackage> for Download {
+ type Error = Error;
+
+ fn try_from(value: manifest::DownloadablePackage) -> std::result::Result<Self, Self::Error> {
+ let url_parameters = value.url_parameters;
+ let mut all_params: HashSet<&str> = url_parameters.keys().map(String::as_str).collect();
+
+ let variants: Vec<DownloadVariant> = value
+ .variants
+ .into_iter()
+ .map(|variant| {
+ let mut variant_match = VariantMatch {
+ system: None,
+ arch: None,
+ };
+
+ for (var, val) in &variant.match_vars {
+ match var.as_str() {
+ "os" => variant_match.system = Some(val.parse::<platform::System>()?),
+ "arch" => variant_match.arch = Some(val.parse::<platform::Architecture>()?),
+ _ => {
+ // TODO(frolv): Return an error with the invalid variable.
+ return Err(Error::GenericErrorPlaceholder);
+ }
+ }
+ }
+
+ Ok(DownloadVariant {
+ matches: variant_match,
+ url_parameters: variant.url_parameters,
+ })
+ })
+ .collect::<Result<Vec<_>>>()?;
+
+ for variant in &variants {
+ all_params.extend(variant.url_parameters.keys().map(String::as_str));
+ }
+
+ // Check if any of the parameters specified in the `url_parameters`
+ // mapping have invalid names.
+ let invalid_vars: Vec<_> = all_params
+ .iter()
+ .filter(|&¶m| !URL_PARAMETER_NAME_REGEX.is_match(param))
+ .collect();
+ if !invalid_vars.is_empty() {
+ // TODO(frolv): Return an error containing the invalid variables.
+ println!("invalid URL variable names {invalid_vars:?}");
+ return Err(Error::GenericErrorPlaceholder);
+ }
+
+ // Next, check the URL string itself. Each parameter substitution
+ // defined within braces should have a valid name and exist within a
+ // provided `url_parameters` mapping.
+ let mut invalid_vars = Vec::new();
+ let mut missing_vars = Vec::new();
+
+ for var in URL_PARAMETER_SUB_REGEX
+ .captures_iter(&value.url)
+ .filter_map(|cap| cap.get(1))
+ {
+ let var = var.as_str();
+ if !URL_PARAMETER_NAME_REGEX.is_match(var) {
+ invalid_vars.push(var);
+ }
+ if !all_params.contains(var) {
+ missing_vars.push(var);
+ }
+ }
+
+ if !invalid_vars.is_empty() {
+ // TODO(frolv): Return an error containing the invalid variables.
+ println!("invalid URL parameters {invalid_vars:?}");
+ return Err(Error::GenericErrorPlaceholder);
+ }
+
+ if !missing_vars.is_empty() {
+ // TODO(frolv): Return an error containing the missing variables.
+ println!("missing URL parameters {missing_vars:?}");
+ return Err(Error::GenericErrorPlaceholder);
+ }
+
+ let format = match value.format.ok_or(Error::GenericErrorPlaceholder)?.as_str() {
+ "bin" => {
+ download::Format::Binary(value.bin_name.ok_or(Error::GenericErrorPlaceholder)?)
+ }
+ _ => return Err(Error::GenericErrorPlaceholder),
+ };
+
+ Ok(Self {
+ format,
+ url: value.url,
+ url_parameters,
+ variants,
+ })
+ }
+}
+
+impl TryFrom<manifest::TargetType> for Metadata {
+ type Error = Error;
+
+ fn try_from(value: manifest::TargetType) -> std::result::Result<Self, Self::Error> {
+ match value {
+ manifest::TargetType::Cipd(data) => Ok(Self::Cipd(data.into())),
+ manifest::TargetType::Download(data) => data.try_into().map(Self::Download),
}
}
}
@@ -89,14 +235,18 @@
provider: &str,
global: bool,
target: manifest::Target,
- ) -> Self {
- Self {
+ ) -> Result<Self> {
+ let target = Self {
name: name.to_owned(),
provider: provider.to_owned(),
global,
dependencies: target.deps,
- metadata: target.desc.map_or(Metadata::DepOnly, Metadata::from),
- }
+ metadata: target
+ .desc
+ .map_or(Ok(Metadata::DepOnly), Metadata::try_from)?,
+ };
+
+ Ok(target)
}
/// Returns the name of the package.
@@ -142,3 +292,145 @@
}
}
}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn target_from_deponly() {
+ let manifest_target = manifest::Target {
+ namespace: manifest::ProviderNamespace::Local,
+ deps: vec!["libqg".into(), "cargo".into()],
+ desc: None,
+ };
+ let target = Target::from_manifest("qg", "qg-provider", true, manifest_target).unwrap();
+ assert_eq!(target.name(), "qg");
+ assert_eq!(target.provider(), "qg-provider");
+ assert_eq!(target.full_name(), "qg");
+ assert!(target.is_global());
+ assert_eq!(target.dependencies().count(), 2);
+ assert_eq!(target.type_string(), "dep_only");
+ }
+
+ #[test]
+ fn target_from_deponly_local() {
+ let manifest_target = manifest::Target {
+ namespace: manifest::ProviderNamespace::Local,
+ deps: vec!["cargo".into()],
+ desc: None,
+ };
+ let target = Target::from_manifest("qg", "qg-provider", false, manifest_target).unwrap();
+ assert_eq!(target.name(), "qg");
+ assert_eq!(target.provider(), "qg-provider");
+ assert_eq!(target.full_name(), "qg-provider:qg");
+ assert!(!target.is_global());
+ assert_eq!(target.dependencies().count(), 1);
+ assert_eq!(target.type_string(), "dep_only");
+ }
+
+ #[test]
+ fn target_from_download_basic_valid() {
+ let manifest_target = manifest::Target {
+ namespace: manifest::ProviderNamespace::Local,
+ deps: vec![],
+ desc: Some(manifest::TargetType::Download(
+ manifest::DownloadablePackage {
+ url: "https://qg.io/client".into(),
+ url_parameters: HashMap::new(),
+ variants: vec![],
+ format: Some("bin".into()),
+ bin_name: Some("qg".into()),
+ },
+ )),
+ };
+ let target = Target::from_manifest("qg", "qg-provider", true, manifest_target).unwrap();
+ assert_eq!(target.name(), "qg");
+ assert_eq!(target.provider(), "qg-provider");
+ assert_eq!(target.full_name(), "qg");
+ assert!(target.is_global());
+ assert_eq!(target.dependencies().count(), 0);
+ assert_eq!(target.type_string(), "download");
+ }
+
+ #[test]
+ fn target_from_download_params_valid() {
+ let manifest_target = manifest::Target {
+ namespace: manifest::ProviderNamespace::Local,
+ deps: vec![],
+ desc: Some(manifest::TargetType::Download(
+ manifest::DownloadablePackage {
+ url: "https://qg.io/client?version={ver}".into(),
+ url_parameters: HashMap::from([("ver".into(), "1.2.4".into())]),
+ variants: vec![],
+ format: Some("bin".into()),
+ bin_name: Some("qg".into()),
+ },
+ )),
+ };
+ let target = Target::from_manifest("qg", "qg-provider", true, manifest_target).unwrap();
+ assert_eq!(target.name(), "qg");
+ assert_eq!(target.provider(), "qg-provider");
+ assert_eq!(target.full_name(), "qg");
+ assert!(target.is_global());
+ assert_eq!(target.dependencies().count(), 0);
+ assert_eq!(target.type_string(), "download");
+ }
+
+ #[test]
+ fn target_from_download_params_missing() {
+ let manifest_target = manifest::Target {
+ namespace: manifest::ProviderNamespace::Local,
+ deps: vec![],
+ desc: Some(manifest::TargetType::Download(
+ manifest::DownloadablePackage {
+ url: "https://qg.io/client?version={ver}".into(),
+ url_parameters: HashMap::new(), // no `ver` defined
+ variants: vec![],
+ format: Some("bin".into()),
+ bin_name: Some("qg".into()),
+ },
+ )),
+ };
+ let err = Target::from_manifest("qg", "qg-provider", true, manifest_target).unwrap_err();
+ assert!(matches!(err, Error::GenericErrorPlaceholder));
+ }
+
+ #[test]
+ fn target_from_download_params_invalid_name() {
+ let manifest_target = manifest::Target {
+ namespace: manifest::ProviderNamespace::Local,
+ deps: vec![],
+ desc: Some(manifest::TargetType::Download(
+ manifest::DownloadablePackage {
+ url: "https://qg.io/client?version={0ver}".into(),
+ url_parameters: HashMap::from([("0ver".into(), "01.2.4".into())]),
+ variants: vec![],
+ format: Some("bin".into()),
+ bin_name: Some("qg".into()),
+ },
+ )),
+ };
+ let err = Target::from_manifest("qg", "qg-provider", true, manifest_target).unwrap_err();
+ assert!(matches!(err, Error::GenericErrorPlaceholder));
+ }
+
+ #[test]
+ fn target_from_download_params_empty_name() {
+ let manifest_target = manifest::Target {
+ namespace: manifest::ProviderNamespace::Local,
+ deps: vec![],
+ desc: Some(manifest::TargetType::Download(
+ manifest::DownloadablePackage {
+ url: "https://qg.io/client?version={}".into(),
+ url_parameters: HashMap::new(),
+ variants: vec![],
+ format: Some("bin".into()),
+ bin_name: Some("qg".into()),
+ },
+ )),
+ };
+ let err = Target::from_manifest("qg", "qg-provider", true, manifest_target).unwrap_err();
+ assert!(matches!(err, Error::GenericErrorPlaceholder));
+ }
+}
