feature: opm (index|registry) prune command

This makes it possible for a user to prune an index image or an operator
database referencing a large collection of operator bundles.

Author: djzager

cmd/opm/index/cmd.go

@@ -24,4 +24,5 @@ func AddCommand(parent *cobra.Command) {
 	cmd.AddCommand(newIndexDeleteCmd())
 	addIndexAddCmd(cmd)
 	cmd.AddCommand(newIndexExportCmd())
+	cmd.AddCommand(newIndexPruneCmd())
 }

cmd/opm/index/prune.go

@@ -0,0 +1,113 @@
+package index
+
+import (
+	"github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+
+	"github.com/operator-framework/operator-registry/pkg/lib/indexer"
+)
+
+func newIndexPruneCmd() *cobra.Command {
+	indexCmd := &cobra.Command{
+		Use:   "prune",
+		Short: "prune an index of all but specified packages",
+		Long:  `prune an index of all but specified packages`,
+
+		PreRunE: func(cmd *cobra.Command, args []string) error {
+			if debug, _ := cmd.Flags().GetBool("debug"); debug {
+				logrus.SetLevel(logrus.DebugLevel)
+			}
+			return nil
+		},
+
+		RunE: runIndexPruneCmdFunc,
+	}
+
+	indexCmd.Flags().Bool("debug", false, "enable debug logging")
+	indexCmd.Flags().Bool("generate", false, "if enabled, just creates the dockerfile and saves it to local disk")
+	indexCmd.Flags().StringP("out-dockerfile", "d", "", "if generating the dockerfile, this flag is used to (optionally) specify a dockerfile name")
+	indexCmd.Flags().StringP("from-index", "f", "", "index to prune")
+	if err := indexCmd.MarkFlagRequired("from-index"); err != nil {
+		logrus.Panic("Failed to set required `from-index` flag for `index prune`")
+	}
+	indexCmd.Flags().StringSliceP("packages", "p", nil, "comma separated list of packages to keep")
+	if err := indexCmd.MarkFlagRequired("packages"); err != nil {
+		logrus.Panic("Failed to set required `packages` flag for `index prune`")
+	}
+	indexCmd.Flags().StringP("binary-image", "i", "", "container image for on-image `opm` command")
+	indexCmd.Flags().StringP("container-tool", "c", "podman", "tool to interact with container images (save, build, etc.). One of: [docker, podman]")
+	indexCmd.Flags().StringP("tag", "t", "", "custom tag for container image being built")
+	indexCmd.Flags().Bool("permissive", false, "allow registry load errors")
+
+	if err := indexCmd.Flags().MarkHidden("debug"); err != nil {
+		logrus.Panic(err.Error())
+	}
+
+	return indexCmd
+
+}
+
+func runIndexPruneCmdFunc(cmd *cobra.Command, args []string) error {
+	generate, err := cmd.Flags().GetBool("generate")
+	if err != nil {
+		return err
+	}
+
+	outDockerfile, err := cmd.Flags().GetString("out-dockerfile")
+	if err != nil {
+		return err
+	}
+
+	fromIndex, err := cmd.Flags().GetString("from-index")
+	if err != nil {
+		return err
+	}
+
+	packages, err := cmd.Flags().GetStringSlice("packages")
+	if err != nil {
+		return err
+	}
+
+	binaryImage, err := cmd.Flags().GetString("binary-image")
+	if err != nil {
+		return err
+	}
+
+	containerTool, err := cmd.Flags().GetString("container-tool")
+	if err != nil {
+		return err
+	}
+
+	tag, err := cmd.Flags().GetString("tag")
+	if err != nil {
+		return err
+	}
+
+	permissive, err := cmd.Flags().GetBool("permissive")
+	if err != nil {
+		return err
+	}
+
+	logger := logrus.WithFields(logrus.Fields{"packages": packages})
+
+	logger.Info("building the index")
+
+	indexPruner := indexer.NewIndexPruner(containerTool, logger)
+
+	request := indexer.PruneFromIndexRequest{
+		Generate:          generate,
+		FromIndex:         fromIndex,
+		BinarySourceImage: binaryImage,
+		OutDockerfile:     outDockerfile,
+		Packages:          packages,
+		Tag:               tag,
+		Permissive:        permissive,
+	}
+
+	err = indexPruner.PruneFromIndex(request)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}

cmd/opm/registry/cmd.go

@@ -23,6 +23,7 @@ func NewOpmRegistryCmd() *cobra.Command {
 	rootCmd.AddCommand(newRegistryServeCmd())
 	rootCmd.AddCommand(newRegistryAddCmd())
 	rootCmd.AddCommand(newRegistryRmCmd())
+	rootCmd.AddCommand(newRegistryPruneCmd())
 
 	return rootCmd
 }

cmd/opm/registry/prune.go

@@ -0,0 +1,69 @@
+package registry
+
+import (
+	"github.com/operator-framework/operator-registry/pkg/lib/registry"
+
+	"github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+)
+
+func newRegistryPruneCmd() *cobra.Command {
+	rootCmd := &cobra.Command{
+		Use:   "prune",
+		Short: "prune an operator registry DB of all but specified packages",
+		Long:  `prune an operator registry DB of all but specified packages`,
+
+		PreRunE: func(cmd *cobra.Command, args []string) error {
+			if debug, _ := cmd.Flags().GetBool("debug"); debug {
+				logrus.SetLevel(logrus.DebugLevel)
+			}
+			return nil
+		},
+
+		RunE: runRegistryPruneCmdFunc,
+	}
+
+	rootCmd.Flags().Bool("debug", false, "enable debug logging")
+	rootCmd.Flags().StringP("database", "d", "bundles.db", "relative path to database file")
+	rootCmd.Flags().StringSliceP("packages", "p", []string{}, "comma separated list of package names to be kept")
+	if err := rootCmd.MarkFlagRequired("packages"); err != nil {
+		logrus.Panic("Failed to set required `packages` flag for `registry rm`")
+	}
+	rootCmd.Flags().Bool("permissive", false, "allow registry load errors")
+
+	return rootCmd
+}
+
+func runRegistryPruneCmdFunc(cmd *cobra.Command, args []string) error {
+	fromFilename, err := cmd.Flags().GetString("database")
+	if err != nil {
+		return err
+	}
+	packages, err := cmd.Flags().GetStringSlice("packages")
+	if err != nil {
+		return err
+	}
+	permissive, err := cmd.Flags().GetBool("permissive")
+	if err != nil {
+		return err
+	}
+
+	request := registry.PruneFromRegistryRequest{
+		Packages:      packages,
+		InputDatabase: fromFilename,
+		Permissive:    permissive,
+	}
+
+	logger := logrus.WithFields(logrus.Fields{"packages": packages})
+
+	logger.Info("pruning from the registry")
+
+	registryPruner := registry.NewRegistryPruner(logger)
+
+	err = registryPruner.PruneFromRegistry(request)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}

docs/design/opm-tooling.md

@@ -34,6 +34,14 @@ Great! The existing `test-registry.db` file is updated. Now we have a registry t
 
 Calling this on our existing test registry removes all versions of the prometheus operator entirely from the database.
 
+#### prune
+
+`opm` supports specifying which packages should be kept in an operator database. For example:
+
+`opm registry prune -p "prometheus" -d "test-registry.db"`
+
+Would remove all but the `prometheus` package from the operator database.
+
 #### serve
 
 `opm` also includes a command to connect to an existing database and serve a `gRPC` API that handles requests for data about the registry:
@@ -93,6 +101,14 @@ Like `opm registry rm`, this command will remove all versions an entire operator
 
 This will result in the tagged container image `quay.io/operator-framework/monitoring-index:1.0.2` with a registry that no longer contains the `prometheus` operator at all.
 
+#### prune
+
+`opm index prune` allows the user to specify which operator packages should be maintained in an index. For example:
+
+`opm index prune -p "prometheus" --from-index quay.io/operator-framework/example-index:1.0.0 --tag quay.io/operator-framework/example-index:1.0.1`
+
+Would remove all but the `prometheus` package from the index.
+
 #### export
 
 `opm index export` will export a package from an index image into a directory. The format of this directory will match the appregistry manifest format: containing all versions of the package in the index along with a `package.yaml` file. This command takes an `--index` flag that points to an index image, a `--package` flag that states a package name, an optional `--download-folder` as the export location (default is `./downloaded`), and just as the other index commands it takes a `--container-tool` flag. Ex:

pkg/lib/indexer/indexer.go

@@ -37,6 +37,7 @@ type ImageIndexer struct {
 	ImageReader         containertools.ImageReader
 	RegistryAdder       registry.RegistryAdder
 	RegistryDeleter     registry.RegistryDeleter
+	RegistryPruner      registry.RegistryPruner
 	ContainerTool       string
 	Logger              *logrus.Entry
 }
@@ -440,3 +441,57 @@ func generatePackageYaml(dbQuerier pregistry.Query, packageName, downloadPath st
 
 	return utilerrors.NewAggregate(errs)
 }
+
+// PruneFromIndexRequest defines the parameters to send to the PruneFromIndex API
+type PruneFromIndexRequest struct {
+	Generate          bool
+	Permissive        bool
+	BinarySourceImage string
+	FromIndex         string
+	OutDockerfile     string
+	Tag               string
+	Packages          []string
+}
+
+func (i ImageIndexer) PruneFromIndex(request PruneFromIndexRequest) error {
+	// set a temp directory
+	tmpDir, err := ioutil.TempDir("./", tmpDirPrefix)
+	if err != nil {
+		return err
+	}
+	defer os.RemoveAll(tmpDir)
+
+	databaseFile, err := i.getDatabaseFile(tmpDir, request.FromIndex)
+	if err != nil {
+		return err
+	}
+	workingDir := path.Dir(databaseFile)
+
+	// Run opm registry prune on the database
+	pruneFromRegistryReq := registry.PruneFromRegistryRequest{
+		Packages:      request.Packages,
+		InputDatabase: databaseFile,
+		Permissive:    request.Permissive,
+	}
+
+	// Prune the bundles from the registry
+	err = i.RegistryPruner.PruneFromRegistry(pruneFromRegistryReq)
+	if err != nil {
+		return err
+	}
+
+	// write the dockerfile to disk if generate is set, otherwise shell out to build the image
+	if request.Generate {
+		err = i.generateDockerfile(request.BinarySourceImage, request.OutDockerfile, databaseFile)
+		if err != nil {
+			return err
+		}
+	} else {
+		err = i.buildDockerfile(request.BinarySourceImage, workingDir, request.Tag)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

pkg/lib/indexer/interfaces.go

@@ -64,3 +64,19 @@ func NewIndexExporter(containerTool string, logger *logrus.Entry) IndexExporter
 		Logger:              logger,
 	}
 }
+
+// IndexPruner prunes operators out of an index
+type IndexPruner interface {
+	PruneFromIndex(PruneFromIndexRequest) error
+}
+
+func NewIndexPruner(containerTool string, logger *logrus.Entry) IndexPruner {
+	return ImageIndexer{
+		DockerfileGenerator: containertools.NewDockerfileGenerator(containerTool, logger),
+		CommandRunner:       containertools.NewCommandRunner(containerTool, logger),
+		LabelReader:         containertools.NewLabelReader(containerTool, logger),
+		ImageReader:         containertools.NewImageReader(containerTool, logger),
+		ContainerTool:       containerTool,
+		Logger:              logger,
+	}
+}

pkg/lib/registry/interfaces.go

@@ -26,3 +26,13 @@ func NewRegistryDeleter(logger *logrus.Entry) RegistryDeleter {
 		Logger: logger,
 	}
 }
+
+type RegistryPruner interface {
+	PruneFromRegistry(PruneFromRegistryRequest) error
+}
+
+func NewRegistryPruner(logger *logrus.Entry) RegistryPruner {
+	return RegistryUpdater{
+		Logger: logger,
+	}
+}

pkg/lib/registry/registry.go

@@ -135,3 +135,55 @@ func (r RegistryUpdater) DeleteFromRegistry(request DeleteFromRegistryRequest) e
 
 	return nil
 }
+
+type PruneFromRegistryRequest struct {
+	Permissive    bool
+	InputDatabase string
+	Packages      []string
+}
+
+func (r RegistryUpdater) PruneFromRegistry(request PruneFromRegistryRequest) error {
+	db, err := sql.Open("sqlite3", request.InputDatabase)
+	if err != nil {
+		return err
+	}
+	defer db.Close()
+
+	dbLoader, err := sqlite.NewSQLLiteLoader(db)
+	if err != nil {
+		return err
+	}
+	if err := dbLoader.Migrate(context.TODO()); err != nil {
+		return err
+	}
+
+	// get all the packages
+	lister := sqlite.NewSQLLiteQuerierFromDb(db)
+	packages, err := lister.ListPackages(context.TODO())
+	if err != nil {
+		return err
+	}
+
+	// make it inexpensive to find packages
+	pkgMap := make(map[string]bool)
+	for _, pkg := range request.Packages {
+		pkgMap[pkg] = true
+	}
+
+	// prune packages from registry
+	for _, pkg := range packages {
+		if _, found := pkgMap[pkg]; !found {
+			remover := sqlite.NewSQLRemoverForPackages(dbLoader, pkg)
+			if err := remover.Remove(); err != nil {
+				err = fmt.Errorf("error deleting packages from database: %s", err)
+				if !request.Permissive {
+					logrus.WithError(err).Fatal("permissive mode disabled")
+					return err
+				}
+				logrus.WithError(err).Warn("permissive mode enabled")
+			}
+		}
+	}
+
+	return nil
+}