cmd/gazelle/update-repos.go

/* Copyright 2017 The Bazel Authors. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

   http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"errors"
	"flag"
	"fmt"
	"os"
	"path/filepath"
	"sort"
	"strings"

	"github.com/bazelbuild/bazel-gazelle/config"
	"github.com/bazelbuild/bazel-gazelle/internal/wspace"
	"github.com/bazelbuild/bazel-gazelle/language"
	"github.com/bazelbuild/bazel-gazelle/merger"
	"github.com/bazelbuild/bazel-gazelle/repo"
	"github.com/bazelbuild/bazel-gazelle/rule"
	bzl "github.com/bazelbuild/buildtools/build"
)

type updateReposConfig struct {
	repoFilePath  string
	importPaths   []string
	macroFileName string
	macroDefName  string
	pruneRules    bool
	workspace     *rule.File
	repoFileMap   map[string]*rule.File
}

const updateReposName = "_update-repos"

func getUpdateReposConfig(c *config.Config) *updateReposConfig {
	return c.Exts[updateReposName].(*updateReposConfig)
}

type updateReposConfigurer struct{}

type macroFlag struct {
	macroFileName *string
	macroDefName  *string
}

func (f macroFlag) Set(value string) error {
	args := strings.Split(value, "%")
	if len(args) != 2 {
		return fmt.Errorf("Failure parsing to_macro: %s, expected format is macroFile%%defName", value)
	}
	if strings.HasPrefix(args[0], "..") {
		return fmt.Errorf("Failure parsing to_macro: %s, macro file path %s should not start with \"..\"", value, args[0])
	}
	*f.macroFileName = args[0]
	*f.macroDefName = args[1]
	return nil
}

func (f macroFlag) String() string {
	return ""
}

func (*updateReposConfigurer) RegisterFlags(fs *flag.FlagSet, cmd string, c *config.Config) {
	uc := &updateReposConfig{}
	c.Exts[updateReposName] = uc
	fs.StringVar(&uc.repoFilePath, "from_file", "", "Gazelle will translate repositories listed in this file into repository rules in WORKSPACE or a .bzl macro function. Gopkg.lock and go.mod files are supported")
	fs.Var(macroFlag{macroFileName: &uc.macroFileName, macroDefName: &uc.macroDefName}, "to_macro", "Tells Gazelle to write repository rules into a .bzl macro function rather than the WORKSPACE file. . The expected format is: macroFile%defName")
	fs.BoolVar(&uc.pruneRules, "prune", false, "When enabled, Gazelle will remove rules that no longer have equivalent repos in the Gopkg.lock/go.mod file. Can only used with -from_file.")
}

func (*updateReposConfigurer) CheckFlags(fs *flag.FlagSet, c *config.Config) error {
	uc := getUpdateReposConfig(c)
	switch {
	case uc.repoFilePath != "":
		if len(fs.Args()) != 0 {
			return fmt.Errorf("got %d positional arguments with -from_file; wanted 0.\nTry -help for more information.", len(fs.Args()))
		}

	default:
		if len(fs.Args()) == 0 {
			return fmt.Errorf("no repositories specified\nTry -help for more information.")
		}
		if uc.pruneRules {
			return fmt.Errorf("the -prune option can only be used with -from_file")
		}
		uc.importPaths = fs.Args()
	}

	var err error
	workspacePath := wspace.FindWORKSPACEFile(c.RepoRoot)
	uc.workspace, err = rule.LoadWorkspaceFile(workspacePath, "")
	if err != nil {
		return fmt.Errorf("loading WORKSPACE file: %v", err)
	}
	c.Repos, uc.repoFileMap, err = repo.ListRepositories(uc.workspace)
	if err != nil {
		return fmt.Errorf("loading WORKSPACE file: %v", err)
	}

	return nil
}

func (*updateReposConfigurer) KnownDirectives() []string { return nil }

func (*updateReposConfigurer) Configure(c *config.Config, rel string, f *rule.File) {}

func updateRepos(args []string) (err error) {
	// Build configuration with all languages.
	cexts := make([]config.Configurer, 0, len(languages)+2)
	cexts = append(cexts, &config.CommonConfigurer{}, &updateReposConfigurer{})
	kinds := make(map[string]rule.KindInfo)
	loads := []rule.LoadInfo{}
	for _, lang := range languages {
		cexts = append(cexts, lang)
		loads = append(loads, lang.Loads()...)
		for kind, info := range lang.Kinds() {
			kinds[kind] = info
		}
	}
	c, err := newUpdateReposConfiguration(args, cexts)
	if err != nil {
		return err
	}
	uc := getUpdateReposConfig(c)

	// TODO(jayconrod): move Go-specific RemoteCache logic to language/go.
	var knownRepos []repo.Repo
	for _, r := range c.Repos {
		if r.Kind() == "go_repository" {
			knownRepos = append(knownRepos, repo.Repo{
				Name:     r.Name(),
				GoPrefix: r.AttrString("importpath"),
				Remote:   r.AttrString("remote"),
				VCS:      r.AttrString("vcs"),
			})
		}
	}
	rc, cleanup := repo.NewRemoteCache(knownRepos)
	defer func() {
		if cerr := cleanup(); err == nil && cerr != nil {
			err = cerr
		}
	}()

	// Fix the workspace file with each language.
	for _, lang := range filterLanguages(c, languages) {
		lang.Fix(c, uc.workspace)
	}

	// Generate rules from command language arguments or by importing a file.
	var gen, empty []*rule.Rule
	if uc.repoFilePath == "" {
		gen, err = updateRepoImports(c, rc)
	} else {
		gen, empty, err = importRepos(c, rc)
	}
	if err != nil {
		return err
	}

	// Organize generated and empty rules by file. A rule should go into the file
	// it came from (by name). New rules should go into WORKSPACE or the file
	// specified with -to_macro.
	var newGen []*rule.Rule
	genForFiles := make(map[*rule.File][]*rule.Rule)
	emptyForFiles := make(map[*rule.File][]*rule.Rule)
	genNames := make(map[string]*rule.Rule)
	for _, r := range gen {
		if existingRule := genNames[r.Name()]; existingRule != nil {
			import1 := existingRule.AttrString("importpath")
			import2 := r.AttrString("importpath")
			return fmt.Errorf("imports %s and %s resolve to the same repository rule name %s",
				import1, import2, r.Name())
		} else {
			genNames[r.Name()] = r
		}
		f := uc.repoFileMap[r.Name()]
		if f != nil {
			genForFiles[f] = append(genForFiles[f], r)
		} else {
			newGen = append(newGen, r)
		}
	}
	for _, r := range empty {
		f := uc.repoFileMap[r.Name()]
		if f == nil {
			panic(fmt.Sprintf("empty rule %q for deletion that was not found", r.Name()))
		}
		emptyForFiles[f] = append(emptyForFiles[f], r)
	}

	var newGenFile *rule.File
	var macroPath string
	if uc.macroFileName != "" {
		macroPath = filepath.Join(c.RepoRoot, filepath.Clean(uc.macroFileName))
	}
	for f := range genForFiles {
		if macroPath == "" && wspace.IsWORKSPACE(f.Path) ||
			macroPath != "" && f.Path == macroPath && f.DefName == uc.macroDefName {
			newGenFile = f
			break
		}
	}
	if newGenFile == nil {
		if uc.macroFileName == "" {
			newGenFile = uc.workspace
		} else {
			var err error
			newGenFile, err = rule.LoadMacroFile(macroPath, "", uc.macroDefName)
			if os.IsNotExist(err) {
				newGenFile, err = rule.EmptyMacroFile(macroPath, "", uc.macroDefName)
				if err != nil {
					return fmt.Errorf("error creating %q: %v", macroPath, err)
				}
			} else if err != nil {
				return fmt.Errorf("error loading %q: %v", macroPath, err)
			}
		}
	}
	genForFiles[newGenFile] = append(genForFiles[newGenFile], newGen...)

	// Merge rules and fix loads in each file.
	seenFile := make(map[*rule.File]bool)
	sortedFiles := make([]*rule.File, 0, len(genForFiles))
	for f := range genForFiles {
		if !seenFile[f] {
			seenFile[f] = true
			sortedFiles = append(sortedFiles, f)
		}
	}
	for f := range emptyForFiles {
		if !seenFile[f] {
			seenFile[f] = true
			sortedFiles = append(sortedFiles, f)
		}
	}
	if ensureMacroInWorkspace(uc) {
		if !seenFile[uc.workspace] {
			seenFile[uc.workspace] = true
			sortedFiles = append(sortedFiles, uc.workspace)
		}
	}
	sort.Slice(sortedFiles, func(i, j int) bool {
		if cmp := strings.Compare(sortedFiles[i].Path, sortedFiles[j].Path); cmp != 0 {
			return cmp < 0
		}
		return sortedFiles[i].DefName < sortedFiles[j].DefName
	})

	updatedFiles := make(map[string]*rule.File)
	for _, f := range sortedFiles {
		merger.MergeFile(f, emptyForFiles[f], genForFiles[f], merger.PreResolve, kinds)
		merger.FixLoads(f, loads)
		if f == uc.workspace {
			if err := merger.CheckGazelleLoaded(f); err != nil {
				return err
			}
		}
		f.Sync()
		if uf, ok := updatedFiles[f.Path]; ok {
			uf.SyncMacroFile(f)
		} else {
			updatedFiles[f.Path] = f
		}
	}

	// Write updated files to disk.
	for _, f := range sortedFiles {
		if uf := updatedFiles[f.Path]; uf != nil {
			if err := uf.Save(uf.Path); err != nil {
				return err
			}
			delete(updatedFiles, f.Path)
		}
	}

	return nil
}

func newUpdateReposConfiguration(args []string, cexts []config.Configurer) (*config.Config, error) {
	c := config.New()
	fs := flag.NewFlagSet("gazelle", flag.ContinueOnError)
	// Flag will call this on any parse error. Don't print usage unless
	// -h or -help were passed explicitly.
	fs.Usage = func() {}
	for _, cext := range cexts {
		cext.RegisterFlags(fs, "update-repos", c)
	}
	if err := fs.Parse(args); err != nil {
		if err == flag.ErrHelp {
			updateReposUsage(fs)
			return nil, err
		}
		// flag already prints the error; don't print it again.
		return nil, errors.New("Try -help for more information")
	}
	for _, cext := range cexts {
		if err := cext.CheckFlags(fs, c); err != nil {
			return nil, err
		}
	}
	return c, nil
}

func updateReposUsage(fs *flag.FlagSet) {
	fmt.Fprint(os.Stderr, `usage:

# Add/update repositories by import path
gazelle update-repos example.com/repo1 example.com/repo2

# Import repositories from lock file
gazelle update-repos -from_file=file

The update-repos command updates repository rules in the WORKSPACE file.
update-repos can add or update repositories explicitly by import path.
update-repos can also import repository rules from a vendoring tool's lock
file (currently only deps' Gopkg.lock is supported).

FLAGS:

`)
	fs.PrintDefaults()
}

func updateRepoImports(c *config.Config, rc *repo.RemoteCache) (gen []*rule.Rule, err error) {
	// TODO(jayconrod): let the user pick the language with a command line flag.
	// For now, only use the first language that implements the interface.
	uc := getUpdateReposConfig(c)
	var updater language.RepoUpdater
	for _, lang := range filterLanguages(c, languages) {
		if u, ok := lang.(language.RepoUpdater); ok {
			updater = u
			break
		}
	}
	if updater == nil {
		return nil, fmt.Errorf("no languages can update repositories")
	}
	res := updater.UpdateRepos(language.UpdateReposArgs{
		Config:  c,
		Imports: uc.importPaths,
		Cache:   rc,
	})
	return res.Gen, res.Error
}

func importRepos(c *config.Config, rc *repo.RemoteCache) (gen, empty []*rule.Rule, err error) {
	uc := getUpdateReposConfig(c)
	importSupported := false
	var importer language.RepoImporter
	for _, lang := range filterLanguages(c, languages) {
		if i, ok := lang.(language.RepoImporter); ok {
			importSupported = true
			if i.CanImport(uc.repoFilePath) {
				importer = i
				break
			}
		}
	}
	if importer == nil {
		if importSupported {
			return nil, nil, fmt.Errorf("unknown file format: %s", uc.repoFilePath)
		} else {
			return nil, nil, fmt.Errorf("no supported languages can import configuration files")
		}
	}
	res := importer.ImportRepos(language.ImportReposArgs{
		Config: c,
		Path:   uc.repoFilePath,
		Prune:  uc.pruneRules,
		Cache:  rc,
	})
	return res.Gen, res.Empty, res.Error
}

// ensureMacroInWorkspace adds a call to the repository macro if the -to_macro
// flag was used, and the macro was not called or declared with a
// '# gazelle:repository_macro' directive.
//
// ensureMacroInWorkspace returns true if the WORKSPACE file was updated
// and should be saved.
func ensureMacroInWorkspace(uc *updateReposConfig) (updated bool) {
	if uc.macroFileName == "" {
		return false
	}

	// Check whether the macro is already declared.
	// We won't add a call if the macro is declared but not called. It might
	// be called somewhere else.
	macroValue := uc.macroFileName + "%" + uc.macroDefName
	for _, d := range uc.workspace.Directives {
		if d.Key == "repository_macro" && d.Value == macroValue {
			return false
		}
	}

	// Try to find a load and a call.
	var load *bzl.LoadStmt
	var call *bzl.CallExpr
	var loadedDefName string
CallLoop:
	for _, stmt := range uc.workspace.File.Stmt {
		switch stmt := stmt.(type) {
		case *bzl.LoadStmt:
			mod := stmt.Module.Value
			if mod != ":"+uc.macroFileName &&
				mod != "//:"+uc.macroFileName &&
				mod != "@//:"+uc.macroFileName {
				break
			}
			for i, from := range stmt.From {
				if from.Name == uc.macroDefName {
					loadedDefName = stmt.To[i].Name
					load = stmt
					break
				}
			}

		case *bzl.CallExpr:
			if id, ok := stmt.X.(*bzl.Ident); ok && id.Name == loadedDefName {
				break CallLoop
			}
		}
	}

	// Add the load and call if they're missing.
	if call == nil {
		if load == nil {
			load = &bzl.LoadStmt{
				Module:       &bzl.StringExpr{Value: "//:" + uc.macroFileName},
				From:         []*bzl.Ident{{Name: uc.macroDefName}},
				To:           []*bzl.Ident{{Name: uc.macroDefName}},
				ForceCompact: true,
			}
			uc.workspace.File.Stmt = append(uc.workspace.File.Stmt, load)
			loadedDefName = uc.macroDefName
		} else if loadedDefName == "" {
			load.From = append(load.From, &bzl.Ident{Name: uc.macroDefName})
			load.To = append(load.To, &bzl.Ident{Name: uc.macroDefName})
			loadedDefName = uc.macroDefName
		}

		call = &bzl.CallExpr{X: &bzl.Ident{Name: loadedDefName}}
		uc.workspace.File.Stmt = append(uc.workspace.File.Stmt, call)
	}

	// Add the directive to the call.
	com := bzl.Comment{Token: "# gazelle:repository_macro " + macroValue}
	call.Before = append(call.Before, com)

	return true
}