Define downloadable targets
This adds a second type of target to manifests: downloadable targets.
These are targets which can be downloaded from a defined URL. URLs may
contain interpolated variables with platform-specific overrides.
This change only defines these targets in the manifest; it does not
implement interpolation or platform overriding.
Change-Id: I9d3552ac5c6495a17e53301ed9a475d3d5377fc3
Reviewed-on: https://pigweed-review.googlesource.com/c/pigweed/qg/+/124956
Commit-Queue: Alexei Frolov <frolv@google.com>
Reviewed-by: Keir Mierle <keir@google.com>
diff --git a/examples/simple_project/qg.toml b/examples/simple_project/qg.toml
new file mode 100644
index 0000000..3b41715
--- /dev/null
+++ b/examples/simple_project/qg.toml
@@ -0,0 +1,20 @@
+[project]
+name = "simple-qg-project"
+
+[targets.cipd]
+type = "download"
+namespace = "global"
+url = "https://chrome-infra-packages.appspot.com/client?platform={platform}&version={version}"
+url_parameters = { version = "git_revision:97c6e32ecb4f25a703bbe23aa459ea57886464c2" }
+
+[[targets.cipd.variants]]
+match = { os = "linux", arch = "x64" }
+url_parameters = { platform = "linux-amd64" }
+
+[[targets.cipd.variants]]
+match = { os = "macos", arch = "x64" }
+url_parameters = { platform = "mac-amd64" }
+
+[[targets.cipd.variants]]
+match = { os = "windows", arch = "x64" }
+url_parameters = { platform = "windows-amd64" }
diff --git a/qg-cli/src/target.rs b/qg-cli/src/target.rs
index 7fd8bef..497da63 100644
--- a/qg-cli/src/target.rs
+++ b/qg-cli/src/target.rs
@@ -32,8 +32,8 @@
registry.target_count()
);
- for (name, target) in registry.targets() {
- println!("* {name}");
+ for target in registry.targets() {
+ println!("* {}", target.full_name());
println!(" type: {}", target.type_string());
}
diff --git a/qg/src/project/manifest.rs b/qg/src/project/manifest.rs
index eef95a4..0d55272 100644
--- a/qg/src/project/manifest.rs
+++ b/qg/src/project/manifest.rs
@@ -55,6 +55,12 @@
}
}
+impl ProviderNamespace {
+ pub fn is_global(&self) -> bool {
+ self == &Self::Global
+ }
+}
+
/// Information about a target provider.
#[derive(Debug, Deserialize, Serialize)]
pub struct ProviderDescriptor {
@@ -85,6 +91,10 @@
#[serde(flatten)]
pub desc: Option<TargetType>,
+ /// Scope of the target. Overrides the default provider scope.
+ #[serde(default)]
+ pub namespace: ProviderNamespace,
+
/// List of other targets on which this one depends.
#[serde(default)]
pub deps: Vec<String>,
@@ -94,6 +104,7 @@
#[serde(tag = "type", rename_all = "snake_case")]
pub enum TargetType {
Cipd(CipdPackage),
+ Download(DownloadablePackage),
}
/// A downloadable CIPD package.
@@ -115,6 +126,47 @@
pub subdir: Option<PathBuf>,
}
+/// A package downloadable from a known URL.
+#[derive(Debug, Deserialize, Serialize)]
+pub struct DownloadablePackage {
+ /// The URL at which the package is located.
+ ///
+ /// The URL may define arbitrary parameters enclosed in braces, which will
+ /// be interpolated with values from the `url_parameters` field.
+ ///
+ /// For example, the URL `https://api.left-pad.io?str={string}&len={length}`
+ /// will substitute the values of `string` and `length` defined within
+ /// `url_parameters` into the appropriate query params.
+ pub url: String,
+
+ /// Variable substitutions for `url`. See its documentation for details.
+ #[serde(default)]
+ pub url_parameters: HashMap<String, String>,
+
+ /// Overrides for the `url_parameters` field. Overrides are processed in
+ /// order of definition, with the first match being applied.
+ #[serde(default)]
+ pub variants: Vec<DownloadVariants>,
+}
+
+/// An override of the default `url_parameters` for a downloadable package.
+#[derive(Debug, Deserialize, Serialize)]
+pub struct DownloadVariants {
+ /// Mapping of variable -> value against which this override is matched.
+ /// The matchable variables are:
+ ///
+ /// * `os`: The system on which the build is running. One of `linux`,
+ /// `macos`, or `windows`.
+ /// * `arch`: System architecture. Either `x64` or `arm64`.
+ ///
+ #[serde(default, rename = "match")]
+ pub match_vars: HashMap<String, String>,
+
+ /// Parameter overrides to apply if this variant is matched.
+ #[serde(default)]
+ pub url_parameters: HashMap<String, String>,
+}
+
impl Manifest {
pub fn new(name: &str) -> Self {
Self {
diff --git a/qg/src/project/mod.rs b/qg/src/project/mod.rs
index bbd2de8..68f32ec 100644
--- a/qg/src/project/mod.rs
+++ b/qg/src/project/mod.rs
@@ -168,6 +168,7 @@
registry.add_target(crate::Target::from_manifest(
&name,
&root_manifest.project.name,
+ target.namespace.is_global(),
target,
));
}
@@ -177,16 +178,25 @@
let provider_file = root_manifest_file.relative_file(manifest);
let provider_data = provider_file.deserialize_toml::<manifest::ProviderFile>()?;
- let is_global = if let Some(desc) = &provider_data.provider {
- desc.namespace == manifest::ProviderNamespace::Global
+ let global_provider = if let Some(desc) = &provider_data.provider {
+ desc.namespace.is_global()
} else {
false
};
- registry.add_provider(Provider::new(provider, provider_file.path(), is_global));
+ registry.add_provider(Provider::new(
+ provider,
+ provider_file.path(),
+ global_provider,
+ ));
for (name, target) in provider_data.targets {
- registry.add_target(crate::Target::from_manifest(&name, provider, target));
+ registry.add_target(crate::Target::from_manifest(
+ &name,
+ provider,
+ global_provider || target.namespace.is_global(),
+ target,
+ ));
}
}
}
diff --git a/qg/src/registry.rs b/qg/src/registry.rs
index 2ed229f..376e0f7 100644
--- a/qg/src/registry.rs
+++ b/qg/src/registry.rs
@@ -50,10 +50,8 @@
}
/// Returns an iterator over all known targets, in arbitrary order.
- pub fn targets(&self) -> impl Iterator<Item = (&str, &Target)> {
- self.targets
- .iter()
- .map(|(k, v)| (k.as_str(), Arc::as_ref(v)))
+ pub fn targets(&self) -> impl Iterator<Item = &Target> {
+ self.targets.values().map(Arc::as_ref)
}
pub(crate) fn add_provider(&mut self, provider: Provider) -> bool {
@@ -64,15 +62,11 @@
/// Inserts a target into the registry.
pub(crate) fn add_target(&mut self, target: Target) -> bool {
- let Some(provider) = self.providers.get(target.provider()) else {
+ if !self.providers.contains_key(target.provider()) {
return false;
};
- let slug = if provider.global {
- target.name().to_string()
- } else {
- format!("{}:{}", provider.name, target.name())
- };
+ let target_name = target.full_name();
for dep in target
.dependencies()
@@ -81,10 +75,10 @@
self.unresolved_dependencies.insert(dep.into());
}
- self.unresolved_dependencies.remove(&slug);
+ self.unresolved_dependencies.remove(&target_name);
let target = Arc::new(target);
- self.targets.insert(slug, target);
+ self.targets.insert(target_name, target);
true
}
diff --git a/qg/src/target.rs b/qg/src/target.rs
index dfb6bc5..a746cd3 100644
--- a/qg/src/target.rs
+++ b/qg/src/target.rs
@@ -50,6 +50,9 @@
/// The name of the target's provider.
provider: String,
+ /// Whether the target lives in the global namespace.
+ global: bool,
+
/// List of targets on which this one depends, storing their full namespaced
/// paths.
dependencies: Vec<String>,
@@ -64,22 +67,33 @@
#[derive(Debug)]
enum Metadata {
DepOnly,
+
+ // TODO(frolv): These shouldn't store the manifest data directly but convert
+ // it into a more usable in-memory format.
Cipd(manifest::CipdPackage),
+ Download(manifest::DownloadablePackage),
}
impl From<manifest::TargetType> for Metadata {
fn from(tt: manifest::TargetType) -> Self {
match tt {
manifest::TargetType::Cipd(data) => Self::Cipd(data),
+ manifest::TargetType::Download(data) => Self::Download(data),
}
}
}
impl Target {
- pub(crate) fn from_manifest(name: &str, provider: &str, target: manifest::Target) -> Self {
+ pub(crate) fn from_manifest(
+ name: &str,
+ provider: &str,
+ global: bool,
+ target: manifest::Target,
+ ) -> Self {
Self {
name: name.to_owned(),
provider: provider.to_owned(),
+ global,
dependencies: target.deps,
metadata: target.desc.map_or(Metadata::DepOnly, Metadata::from),
}
@@ -97,17 +111,34 @@
&self.provider
}
+ /// Returns true if the target is defined in the global namespace.
+ #[must_use]
+ pub fn is_global(&self) -> bool {
+ self.global
+ }
+
/// Returns the package's dependencies.
pub fn dependencies(&self) -> impl Iterator<Item = &str> {
self.dependencies.iter().map(String::as_str)
}
+ /// Returns the fully-qualified name of the target.
+ #[must_use]
+ pub fn full_name(&self) -> String {
+ if self.global {
+ self.name.clone()
+ } else {
+ format!("{}:{}", self.provider, self.name)
+ }
+ }
+
/// Returns the type of the package as a string.
#[must_use]
pub fn type_string(&self) -> &str {
match self.metadata {
Metadata::DepOnly => "dep_only",
Metadata::Cipd(_) => "cipd",
+ Metadata::Download(_) => "download",
}
}
}
