bug 1827748: feature: opm (index|registry) prune command

cmd/opm/index/cmd.go

@@ -24,4 +24,5 @@ func AddCommand(parent *cobra.Command) {
 	cmd.AddCommand(newIndexDeleteCmd())
 	addIndexAddCmd(cmd)
 	cmd.AddCommand(newIndexExportCmd())
+	cmd.AddCommand(newIndexPruneCmd())
 }

cmd/opm/index/prune.go

@@ -0,0 +1,120 @@
+package index
+
+import (
+	"fmt"
+
+	"github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+
+	"github.com/operator-framework/operator-registry/pkg/containertools"
+	"github.com/operator-framework/operator-registry/pkg/lib/indexer"
+)
+
+func newIndexPruneCmd() *cobra.Command {
+	indexCmd := &cobra.Command{
+		Use:   "prune",
+		Short: "prune an index of all but specified packages",
+		Long:  `prune an index of all but specified packages`,
+
+		PreRunE: func(cmd *cobra.Command, args []string) error {
+			if debug, _ := cmd.Flags().GetBool("debug"); debug {
+				logrus.SetLevel(logrus.DebugLevel)
+			}
+			return nil
+		},
+
+		RunE: runIndexPruneCmdFunc,
+	}
+
+	indexCmd.Flags().Bool("debug", false, "enable debug logging")
+	indexCmd.Flags().Bool("generate", false, "if enabled, just creates the dockerfile and saves it to local disk")
+	indexCmd.Flags().StringP("out-dockerfile", "d", "", "if generating the dockerfile, this flag is used to (optionally) specify a dockerfile name")
+	indexCmd.Flags().StringP("from-index", "f", "", "index to prune")
+	if err := indexCmd.MarkFlagRequired("from-index"); err != nil {
+		logrus.Panic("Failed to set required `from-index` flag for `index prune`")
+	}
+	indexCmd.Flags().StringSliceP("packages", "p", nil, "comma separated list of packages to keep")
+	if err := indexCmd.MarkFlagRequired("packages"); err != nil {
+		logrus.Panic("Failed to set required `packages` flag for `index prune`")
+	}
+	indexCmd.Flags().StringP("binary-image", "i", "", "container image for on-image `opm` command")
+	indexCmd.Flags().StringP("container-tool", "c", "podman", "tool to interact with container images (save, build, etc.). One of: [docker, podman]")
+	indexCmd.Flags().StringP("tag", "t", "", "custom tag for container image being built")
+	indexCmd.Flags().Bool("permissive", false, "allow registry load errors")
+
+	if err := indexCmd.Flags().MarkHidden("debug"); err != nil {
+		logrus.Panic(err.Error())
+	}
+
+	return indexCmd
+
+}
+
+func runIndexPruneCmdFunc(cmd *cobra.Command, args []string) error {
+	generate, err := cmd.Flags().GetBool("generate")
+	if err != nil {
+		return err
+	}
+
+	outDockerfile, err := cmd.Flags().GetString("out-dockerfile")
+	if err != nil {
+		return err
+	}
+
+	fromIndex, err := cmd.Flags().GetString("from-index")
+	if err != nil {
+		return err
+	}
+
+	packages, err := cmd.Flags().GetStringSlice("packages")
+	if err != nil {
+		return err
+	}
+
+	binaryImage, err := cmd.Flags().GetString("binary-image")
+	if err != nil {
+		return err
+	}
+
+	containerTool, err := cmd.Flags().GetString("container-tool")
+	if err != nil {
+		return err
+	}
+
+	if containerTool == "none" {
+		return fmt.Errorf("none is not a valid container-tool for index prune")
+	}
+
+	tag, err := cmd.Flags().GetString("tag")
+	if err != nil {
+		return err
+	}
+
+	permissive, err := cmd.Flags().GetBool("permissive")
+	if err != nil {
+		return err
+	}
+
+	logger := logrus.WithFields(logrus.Fields{"packages": packages})
+
+	logger.Info("pruning the index")
+
+	indexPruner := indexer.NewIndexPruner(containertools.NewContainerTool(containerTool, containertools.PodmanTool), logger)
+
+	request := indexer.PruneFromIndexRequest{
+		Generate:          generate,
+		FromIndex:         fromIndex,
+		BinarySourceImage: binaryImage,
+		OutDockerfile:     outDockerfile,
+		Packages:          packages,
+		Tag:               tag,
+		Permissive:        permissive,
+	}
+
+	err = indexPruner.PruneFromIndex(request)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}

cmd/opm/registry/cmd.go

@@ -23,6 +23,7 @@ func NewOpmRegistryCmd() *cobra.Command {
 	rootCmd.AddCommand(newRegistryServeCmd())
 	rootCmd.AddCommand(newRegistryAddCmd())
 	rootCmd.AddCommand(newRegistryRmCmd())
+	rootCmd.AddCommand(newRegistryPruneCmd())
 
 	return rootCmd
 }

cmd/opm/registry/prune.go

@@ -0,0 +1,69 @@
+package registry
+
+import (
+	"github.com/operator-framework/operator-registry/pkg/lib/registry"
+
+	"github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+)
+
+func newRegistryPruneCmd() *cobra.Command {
+	rootCmd := &cobra.Command{
+		Use:   "prune",
+		Short: "prune an operator registry DB of all but specified packages",
+		Long:  `prune an operator registry DB of all but specified packages`,
+
+		PreRunE: func(cmd *cobra.Command, args []string) error {
+			if debug, _ := cmd.Flags().GetBool("debug"); debug {
+				logrus.SetLevel(logrus.DebugLevel)
+			}
+			return nil
+		},
+
+		RunE: runRegistryPruneCmdFunc,
+	}
+
+	rootCmd.Flags().Bool("debug", false, "enable debug logging")
+	rootCmd.Flags().StringP("database", "d", "bundles.db", "relative path to database file")
+	rootCmd.Flags().StringSliceP("packages", "p", []string{}, "comma separated list of package names to be kept")
+	if err := rootCmd.MarkFlagRequired("packages"); err != nil {
+		logrus.Panic("Failed to set required `packages` flag for `registry rm`")
+	}
+	rootCmd.Flags().Bool("permissive", false, "allow registry load errors")
+
+	return rootCmd
+}
+
+func runRegistryPruneCmdFunc(cmd *cobra.Command, args []string) error {
+	fromFilename, err := cmd.Flags().GetString("database")
+	if err != nil {
+		return err
+	}
+	packages, err := cmd.Flags().GetStringSlice("packages")
+	if err != nil {
+		return err
+	}
+	permissive, err := cmd.Flags().GetBool("permissive")
+	if err != nil {
+		return err
+	}
+
+	request := registry.PruneFromRegistryRequest{
+		Packages:      packages,
+		InputDatabase: fromFilename,
+		Permissive:    permissive,
+	}
+
+	logger := logrus.WithFields(logrus.Fields{"packages": packages})
+
+	logger.Info("pruning from the registry")
+
+	registryPruner := registry.NewRegistryPruner(logger)
+
+	err = registryPruner.PruneFromRegistry(request)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}

docs/design/opm-tooling.md

@@ -34,6 +34,14 @@ Great! The existing `test-registry.db` file is updated. Now we have a registry t
 
 Calling this on our existing test registry removes all versions of the prometheus operator entirely from the database.
 
+#### prune
+
+`opm` supports specifying which packages should be kept in an operator database. For example:
+
+`opm registry prune -p "prometheus" -d "test-registry.db"`
+
+Would remove all but the `prometheus` package from the operator database.
+
 #### serve
 
 `opm` also includes a command to connect to an existing database and serve a `gRPC` API that handles requests for data about the registry:
@@ -93,6 +101,14 @@ Like `opm registry rm`, this command will remove all versions an entire operator
 
 This will result in the tagged container image `quay.io/operator-framework/monitoring-index:1.0.2` with a registry that no longer contains the `prometheus` operator at all.
 
+#### prune
+
+`opm index prune` allows the user to specify which operator packages should be maintained in an index. For example:
+
+`opm index prune -p "prometheus" --from-index quay.io/operator-framework/example-index:1.0.0 --tag quay.io/operator-framework/example-index:1.0.1`
+
+Would remove all but the `prometheus` package from the index.
+
 #### export
 
 `opm index export` will export a package from an index image into a directory. The format of this directory will match the appregistry manifest format: containing all versions of the package in the index along with a `package.yaml` file. This command takes an `--index` flag that points to an index image, a `--package` flag that states a package name, an optional `--download-folder` as the export location (default is `./downloaded`), and just as the other index commands it takes a `--container-tool` flag. Ex:

pkg/lib/indexer/indexer.go

@@ -41,6 +41,7 @@ type ImageIndexer struct {
 	ImageReader         containertools.ImageReader
 	RegistryAdder       registry.RegistryAdder
 	RegistryDeleter     registry.RegistryDeleter
+	RegistryPruner      registry.RegistryPruner
 	ContainerTool       containertools.ContainerTool
 	Logger              *logrus.Entry
 }
@@ -205,6 +206,82 @@ func (i ImageIndexer) DeleteFromIndex(request DeleteFromIndexRequest) error {
 	return nil
 }
 
+// PruneFromIndexRequest defines the parameters to send to the PruneFromIndex API
+type PruneFromIndexRequest struct {
+	Generate          bool
+	Permissive        bool
+	BinarySourceImage string
+	FromIndex         string
+	OutDockerfile     string
+	Tag               string
+	Packages          []string
+}
+
+func (i ImageIndexer) PruneFromIndex(request PruneFromIndexRequest) error {
+	buildDir, outDockerfile, cleanup, err := buildContext(request.Generate, request.OutDockerfile)
+	defer cleanup()
+	if err != nil {
+		return err
+	}
+
+	// set a temp directory for unpacking an image
+	// this is in its own function context so that the deferred cleanup runs before we do a docker build
+	// which prevents the full contents of the previous image from being in the build context
+	var databasePath string
+	if err := func () error {
+		tmpDir, err := ioutil.TempDir("./", tmpDirPrefix)
+		if err != nil {
+
+			return err
+		}
+		defer os.RemoveAll(tmpDir)
+
+		databaseFile, err := i.getDatabaseFile(tmpDir, request.FromIndex)
+		if err != nil {
+			return err
+		}
+		// copy the index to the database folder in the build directory
+		if databasePath, err = copyDatabaseTo(databaseFile, filepath.Join(buildDir, defaultDatabaseFolder)); err != nil {
+			return err
+		}
+		return nil
+	}(); err != nil {
+		return err
+	}
+
+	// Run opm registry prune on the database
+	pruneFromRegistryReq := registry.PruneFromRegistryRequest{
+		Packages:      request.Packages,
+		InputDatabase: databasePath,
+		Permissive:    request.Permissive,
+	}
+
+	// Prune the bundles from the registry
+	err = i.RegistryPruner.PruneFromRegistry(pruneFromRegistryReq)
+	if err != nil {
+		return err
+	}
+
+	// generate the dockerfile
+	dockerfile := i.DockerfileGenerator.GenerateIndexDockerfile(request.BinarySourceImage, databasePath)
+	err = write(dockerfile, outDockerfile, i.Logger)
+	if err != nil {
+		return err
+	}
+
+	if request.Generate {
+		return nil
+	}
+
+	// build the dockerfile
+	err = build(outDockerfile, request.Tag, i.CommandRunner, i.Logger)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
 func (i ImageIndexer) getDatabaseFile(workingDir, fromIndex string) (string, error) {
 	if fromIndex == "" {
 		return path.Join(workingDir, defaultDatabaseFile), nil

pkg/lib/indexer/interfaces.go

@@ -63,3 +63,20 @@ func NewIndexExporter(containerTool containertools.ContainerTool, logger *logrus
 		Logger:              logger,
 	}
 }
+
+// IndexPruner prunes operators out of an index
+type IndexPruner interface {
+	PruneFromIndex(PruneFromIndexRequest) error
+}
+
+func NewIndexPruner(containerTool containertools.ContainerTool, logger *logrus.Entry) IndexPruner {
+	return ImageIndexer{
+		DockerfileGenerator: containertools.NewDockerfileGenerator(logger),
+		CommandRunner:       containertools.NewCommandRunner(containerTool, logger),
+		LabelReader:         containertools.NewLabelReader(containerTool, logger),
+		RegistryPruner:      registry.NewRegistryPruner(logger),
+		ImageReader:         containertools.NewImageReader(containerTool, logger),
+		ContainerTool:       containerTool,
+		Logger:              logger,
+	}
+}

pkg/lib/registry/interfaces.go

@@ -26,3 +26,13 @@ func NewRegistryDeleter(logger *logrus.Entry) RegistryDeleter {
 		Logger: logger,
 	}
 }
+
+type RegistryPruner interface {
+	PruneFromRegistry(PruneFromRegistryRequest) error
+}
+
+func NewRegistryPruner(logger *logrus.Entry) RegistryPruner {
+	return RegistryUpdater{
+		Logger: logger,
+	}
+}