Simple manifest parsing
This defines the basic provider and package structure of manifest files,
and adds a function to parse manifests starting at a project root. This
parsing is initially simple, only visiting providers defined within the
root manifest file.
A `qg package` CLI command is added to demonstrate manifest parsing.
Change-Id: I171fc21bd6299a726dd13615c070a406b85b61cc
Reviewed-on: https://pigweed-review.googlesource.com/c/pigweed/qg/+/121904
Reviewed-by: Erik Gilling <konkers@google.com>
Commit-Queue: Alexei Frolov <frolv@google.com>
diff --git a/qg-cli/src/main.rs b/qg-cli/src/main.rs
index 33e2932..7824800 100644
--- a/qg-cli/src/main.rs
+++ b/qg-cli/src/main.rs
@@ -16,6 +16,7 @@
#![cfg_attr(feature = "strict", deny(warnings))]
mod hello;
+mod packages;
mod subcommands;
#[cfg(feature = "new_command")]
@@ -38,7 +39,7 @@
let matches = app.get_matches();
match Subcommands::from_arg_matches(&matches) {
- Ok(subcommands) => subcommands.run(),
+ Ok(subcommands) => subcommands.run().await,
Err(e) if e.kind() == clap::error::ErrorKind::MissingSubcommand => {
run_main_command(&matches);
Ok(())
diff --git a/qg-cli/src/packages.rs b/qg-cli/src/packages.rs
new file mode 100644
index 0000000..18d1c65
--- /dev/null
+++ b/qg-cli/src/packages.rs
@@ -0,0 +1,39 @@
+// Copyright 2022 The Pigweed Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not
+// use this file except in compliance with the License. You may obtain a copy of
+// the License at
+//
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations under
+// the License.
+
+use anyhow::Result;
+use qg::Project;
+
+/// Manages packages in a project.
+#[derive(clap::Parser, Debug)]
+#[command(about)]
+pub struct Command {}
+
+// TODO(frolv): Actually use async.
+#[allow(clippy::unused_async)]
+pub async fn run(_args: &Command) -> Result<()> {
+ let registry = Project::locate()?.parse_manifests()?;
+
+ for (name, packages) in registry.packages_by_name() {
+ if packages.len() == 1 {
+ println!("{name}");
+ } else {
+ for package in packages {
+ println!("{}", package.canonical_slug());
+ }
+ }
+ }
+
+ Ok(())
+}
diff --git a/qg-cli/src/subcommands.rs b/qg-cli/src/subcommands.rs
index 97a0c12..43b00c4 100644
--- a/qg-cli/src/subcommands.rs
+++ b/qg-cli/src/subcommands.rs
@@ -14,7 +14,7 @@
use anyhow::Result;
-use crate::hello;
+use crate::{hello, packages};
#[cfg(feature = "new_command")]
use crate::new;
@@ -32,6 +32,7 @@
#[cfg(feature = "new_command")]
New(new::Command),
+ Package(packages::Command),
#[cfg(feature = "python")]
PyDemo(py_demo::Command),
@@ -39,11 +40,14 @@
impl Subcommands {
/// Invokes a subcommand with its parsed arguments.
- pub fn run(&self) -> Result<()> {
+ pub async fn run(&self) -> Result<()> {
match self {
Self::Hello(args) => {
hello::run(args);
}
+ Self::Package(args) => {
+ packages::run(args).await?;
+ }
#[cfg(feature = "new_command")]
Self::New(args) => {
new::run(args)?;
diff --git a/qg/Cargo.toml b/qg/Cargo.toml
index e83dd5d..f0e12f8 100644
--- a/qg/Cargo.toml
+++ b/qg/Cargo.toml
@@ -11,7 +11,9 @@
num-traits = "0.2.15"
serde = { version = "1.0.147", features = ["derive"] }
thiserror = "1.0.37"
+tokio = { version = "1.21.2", features = ["full"] }
toml = "0.5.9"
+futures = "0.3.25"
[dependencies.rustpython]
git = "https://github.com/RustPython/RustPython"
diff --git a/qg/src/lib.rs b/qg/src/lib.rs
index beb126b..91e3893 100644
--- a/qg/src/lib.rs
+++ b/qg/src/lib.rs
@@ -14,10 +14,16 @@
// Allows presubmit to fail on warnings.
#![cfg_attr(feature = "strict", deny(warnings))]
+#![warn(clippy::pedantic)]
use std::path::{Path, PathBuf};
+pub mod package;
pub mod project;
+pub mod registry;
+
+#[doc(inline)]
+pub use package::Package;
#[doc(inline)]
pub use project::Project;
diff --git a/qg/src/package.rs b/qg/src/package.rs
new file mode 100644
index 0000000..44bf754
--- /dev/null
+++ b/qg/src/package.rs
@@ -0,0 +1,76 @@
+// Copyright 2022 The Pigweed Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not
+// use this file except in compliance with the License. You may obtain a copy of
+// the License at
+//
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations under
+// the License.
+
+use crate::project::manifest;
+
+/// An installable `qg` package.
+#[derive(Debug)]
+pub struct Package {
+ name: String,
+ provider: String,
+ metadata: Metadata,
+}
+
+#[derive(Debug)]
+enum Metadata {
+ Cipd(manifest::CipdPackage),
+}
+
+impl From<manifest::PackageType> for Metadata {
+ fn from(pt: manifest::PackageType) -> Self {
+ match pt {
+ manifest::PackageType::Cipd(data) => Self::Cipd(data),
+ }
+ }
+}
+
+impl Package {
+ pub(crate) fn from_manifest(name: &str, provider: &str, package: manifest::Package) -> Self {
+ Self {
+ name: name.to_owned(),
+ provider: provider.to_owned(),
+ metadata: package.desc.into(),
+ }
+ }
+
+ /// Returns the name of the package.
+ #[must_use]
+ pub fn name(&self) -> &str {
+ &self.name
+ }
+
+ /// Returns the name of the package's provider.
+ #[must_use]
+ pub fn provider(&self) -> &str {
+ &self.provider
+ }
+
+ /// Returns the type of the package as a string.
+ #[must_use]
+ pub fn type_string(&self) -> &str {
+ match self.metadata {
+ Metadata::Cipd(_) => "cipd",
+ }
+ }
+
+ /// Returns the globally-unique identifier for the package.
+ #[must_use]
+ pub fn canonical_slug(&self) -> String {
+ if self.provider.is_empty() {
+ self.name.clone()
+ } else {
+ format!("{}:{}", self.provider, self.name)
+ }
+ }
+}
diff --git a/qg/src/project/file.rs b/qg/src/project/file.rs
index 0d4f8df..24e5360 100644
--- a/qg/src/project/file.rs
+++ b/qg/src/project/file.rs
@@ -28,8 +28,6 @@
}
impl File {
- // TODO(konkers): Remove once this is referenced.
- #[allow(dead_code)]
pub(super) fn new(path: impl AsRef<Path>) -> Self {
Self {
path: path.as_ref().to_path_buf(),
@@ -42,6 +40,24 @@
&self.path
}
+ /// Reads the contents of the file into a string.
+ ///
+ /// # Errors
+ /// Returns an error if the file could not be read.
+ pub fn read_text(&self) -> Result<String> {
+ std::fs::read_to_string(&self.path).map_err(Error::from)
+ }
+
+ /// Writes a string to the file.
+ ///
+ /// Creates the file if it does not exist, or overwrites existing contents.
+ ///
+ /// # Errors
+ /// Returns an error if the file could not be written.
+ pub fn write_text(&self, contents: &str) -> Result<()> {
+ std::fs::write(&self.path, contents).map_err(Error::from)
+ }
+
/// Writes a serializable struct to the file as TOML.
///
/// Creates the file if it does not exist, or overwrites existing contents.
@@ -82,6 +98,15 @@
func(&mut value)?;
self.serialize_toml(&value)
}
+
+ pub(crate) fn relative_file(&self, path: &Path) -> Self {
+ Self::new(
+ self.path
+ .parent()
+ .expect("file must have parent")
+ .join(path),
+ )
+ }
}
#[cfg(test)]
diff --git a/qg/src/project/manifest.rs b/qg/src/project/manifest.rs
index 3207f2c..271130f 100644
--- a/qg/src/project/manifest.rs
+++ b/qg/src/project/manifest.rs
@@ -12,16 +12,65 @@
// License for the specific language governing permissions and limitations under
// the License.
+use std::{collections::HashMap, path::PathBuf};
+
use serde::{Deserialize, Serialize};
#[derive(Debug, Deserialize, Serialize)]
pub struct Manifest {
+ /// Information about the project to which the manifest belongs.
pub project: Project,
+
+ #[serde(default)]
+ pub providers: HashMap<String, ProviderDescriptor>,
+}
+
+/// A qg-based project.
+#[derive(Debug, Deserialize, Serialize)]
+pub struct Project {
+ /// The name of the project.
+ pub name: String,
}
#[derive(Debug, Deserialize, Serialize)]
-pub struct Project {
- pub name: String,
+pub struct ProviderDescriptor {
+ pub manifest: Option<PathBuf>,
+}
+
+#[derive(Debug, Deserialize, Serialize)]
+pub struct ProviderFile {
+ #[serde(default)]
+ pub packages: HashMap<String, Package>,
+}
+
+#[derive(Debug, Deserialize, Serialize)]
+pub struct Package {
+ #[serde(flatten)]
+ pub desc: PackageType,
+}
+
+#[derive(Debug, Deserialize, Serialize)]
+#[serde(tag = "type", rename_all = "snake_case")]
+pub enum PackageType {
+ Cipd(CipdPackage),
+}
+
+#[derive(Debug, Deserialize, Serialize)]
+pub struct CipdPackage {
+ /// The location on of the package in CIPD's repositories.
+ pub path: String,
+
+ /// Platforms the package supports.
+ #[serde(default)]
+ pub platforms: Vec<String>,
+
+ /// CIPD tags to match against when finding a package revision.
+ #[serde(default)]
+ pub tags: Vec<String>,
+
+ /// Local subdirectory under the CIPD installation root into which the
+ /// package is installed.
+ pub subdir: Option<PathBuf>,
}
impl Manifest {
@@ -30,6 +79,7 @@
project: Project {
name: name.to_owned(),
},
+ providers: HashMap::new(),
}
}
}
diff --git a/qg/src/project/mod.rs b/qg/src/project/mod.rs
index 31df184..32183a0 100644
--- a/qg/src/project/mod.rs
+++ b/qg/src/project/mod.rs
@@ -17,11 +17,12 @@
path::{Path, PathBuf},
};
+use crate::registry::Registry;
use crate::{Error, Result};
use manifest::Manifest;
pub mod file;
-mod manifest;
+pub(crate) mod manifest;
#[doc(inline)]
pub use file::File;
@@ -145,6 +146,34 @@
Self::relative_file(&self.cache_dir, Path::new(path.as_ref()))
}
+ /// Reads `qg` manifest files starting from the project root into a registry
+ /// of available packages.
+ ///
+ /// # Errors
+ /// Returns an error if manifest files could not be read or are improperly
+ /// structured.
+ pub fn parse_manifests(&self) -> Result<Registry> {
+ let root_manifest_file = self.root_manifest();
+ let root_manifest = root_manifest_file.deserialize_toml::<Manifest>()?;
+ let mut registry = Registry::new();
+
+ for (provider, desc) in &root_manifest.providers {
+ registry.add_provider(provider);
+
+ if let Some(manifest) = &desc.manifest {
+ let provider_file = root_manifest_file
+ .relative_file(manifest)
+ .deserialize_toml::<manifest::ProviderFile>()?;
+
+ for (name, package) in provider_file.packages {
+ registry.add_package(crate::Package::from_manifest(&name, provider, package));
+ }
+ }
+ }
+
+ Ok(registry)
+ }
+
fn root_manifest(&self) -> self::File {
self.file(Self::MANIFEST_FILE)
.expect("manifest path is not absolute")
diff --git a/qg/src/registry.rs b/qg/src/registry.rs
new file mode 100644
index 0000000..ada1f20
--- /dev/null
+++ b/qg/src/registry.rs
@@ -0,0 +1,81 @@
+// Copyright 2022 The Pigweed Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not
+// use this file except in compliance with the License. You may obtain a copy of
+// the License at
+//
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations under
+// the License.
+
+use std::{
+ collections::{HashMap, HashSet},
+ sync::Arc,
+};
+
+use crate::Package;
+
+/// A database of packages known to `qg`.
+#[derive(Debug)]
+pub struct Registry {
+ /// Mapping of package slug to package. Slugs are globally unique.
+ packages: HashMap<String, Arc<Package>>,
+
+ /// Mapping of package name to packages. The same package can come form
+ /// multiple providers.
+ packages_by_name: HashMap<String, Vec<Arc<Package>>>,
+
+ /// All known package providers by name.
+ /// TODO(frolv): Make this a map storing provider metadata.
+ providers: HashSet<String>,
+}
+
+impl Registry {
+ /// Initializes an empty package registry.
+ #[must_use]
+ pub fn new() -> Self {
+ Self {
+ packages: HashMap::new(),
+ packages_by_name: HashMap::new(),
+ providers: HashSet::new(),
+ }
+ }
+
+ /// Returns an iterator over all known packages grouped by package name.
+ pub fn packages_by_name(&self) -> impl Iterator<Item = (&str, Vec<&Package>)> {
+ self.packages_by_name
+ .iter()
+ .map(|(k, v)| (k.as_str(), v.iter().map(Arc::as_ref).collect()))
+ }
+
+ pub(crate) fn add_provider(&mut self, name: &str) -> bool {
+ self.providers.insert(name.to_owned())
+ }
+
+ pub(crate) fn add_package(&mut self, package: Package) -> bool {
+ if !self.providers.contains(package.provider()) {
+ return false;
+ }
+
+ let package = Arc::new(package);
+ self.packages
+ .insert(package.canonical_slug(), package.clone());
+
+ self.packages_by_name
+ .entry(package.name().into())
+ .or_default()
+ .push(package);
+
+ true
+ }
+}
+
+impl Default for Registry {
+ fn default() -> Self {
+ Self::new()
+ }
+}
