packaging: merge packaging repository

git-subtree-dir: tools/packaging
git-subtree-mainline: f818b46a41
git-subtree-split: 1f22d72d5d

Signed-off-by: Peng Tao <bergwolf@hyper.sh>

tools/packaging/cmd/kata-pkgsync/cli/config.go

@@ -0,0 +1,39 @@
+// Copyright (c) 2017-2018 Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package main
+
+type CfgOBSProject struct {
+	Name string
+	Auth struct {
+		User     string
+		Password string
+	}
+	Releases []string
+	Archs    []string `yaml:"architectures"`
+}
+
+type CfgPackagecloud struct {
+	Auth struct {
+		User  string
+		Token string
+	}
+	Repo string
+}
+
+type config struct {
+	OBSProjects  map[string]CfgOBSProject `yaml:"obsprojects"`
+	Packagecloud CfgPackagecloud
+	// Mapping from OBS "Repositories" to Packagecloud "Distros"
+	DistroMapping map[string]string `yaml:"distro-mapping"`
+}
+
+func getConfig(configFile string) (config, error) {
+	var cfg config
+	if err := yamlUnmarshal(configFile, &cfg); err != nil {
+		return cfg, err
+	}
+	return cfg, nil
+}

tools/packaging/cmd/kata-pkgsync/cli/main.go

@@ -0,0 +1,295 @@
+// Copyright (c) 2019 SUSE LLC
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package main
+
+import (
+	"flag"
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/marcov/obsgo"
+	"github.com/mlafeldt/pkgcloud"
+	"github.com/sirupsen/logrus"
+	pb "gopkg.in/cheggaaa/pb.v1"
+)
+
+//nolint[:gochecknoglobals]
+var (
+	// empty variables are set by "go build -ldflags" option
+	name           = ""
+	version        = ""
+	commit         = ""
+	defaultConfig  = "config.yaml"
+	defaultOBSDest = "obs-packages"
+)
+
+func usage() {
+	fmt.Fprintf(flag.CommandLine.Output(), `NAME:
+   %s - Synchronize packages from OBS to Packagecloud
+
+USAGE:
+   %s [options] [config file]
+
+By default %s reads the configuration from a file named %s. This can be
+overridden by passing the path to a config file.
+
+Options:
+`, name, name, name, defaultConfig)
+
+	flag.PrintDefaults()
+}
+
+func getOBSProjects(cfgProjects map[string]CfgOBSProject) []obsgo.Project {
+	var projects []obsgo.Project
+
+	for n, p := range cfgProjects {
+		proj := obsgo.Project{
+			User:     p.Auth.User,
+			Password: p.Auth.Password,
+		}
+
+		if len(p.Archs) == 0 {
+			p.Archs = append(p.Archs, "")
+		}
+
+		if len(p.Releases) == 0 {
+			p.Releases = append(p.Releases, "")
+		}
+
+		// Kata projects names format is "project:release:architecture"
+		for _, arch := range p.Archs {
+			for _, release := range p.Releases {
+				var fullname strings.Builder
+
+				fullname.WriteString(n)
+				if arch != "" {
+					fmt.Fprintf(&fullname, ":%s", arch)
+				}
+				if release != "" {
+					fmt.Fprintf(&fullname, ":%s", release)
+				}
+
+				proj.Name = fullname.String()
+				projects = append(projects, proj)
+			}
+		}
+	}
+
+	return projects
+}
+
+func getXferBinaries(pkg obsgo.PackageInfo, pcDistro string,
+	pcPackages []pkgcloud.Package, pcPackagesNeeded *[]bool) []obsgo.PkgBinary {
+
+	var xferBins []obsgo.PkgBinary
+	for _, src := range pkg.Files {
+		found := false
+		for i, dst := range pcPackages {
+			logrus.WithFields(logrus.Fields{
+				"source-file":        src.Filename,
+				"source-distro":      pcDistro,
+				"destination-file":   dst.Filename,
+				"destination-distro": dst.DistroVersion,
+			}).Debug("Checking package")
+			if pcDistro == dst.DistroVersion && src.Filename == dst.Filename {
+				logrus.WithFields(logrus.Fields{
+					"filename": src.Filename,
+					"distro":   pcDistro,
+				}).Debug("Package file already on Packagecloud")
+				found = true
+				(*pcPackagesNeeded)[i] = true
+				break
+			}
+		}
+		if !found {
+			logrus.WithFields(logrus.Fields{
+				"filename": src.Filename,
+				"distro":   pcDistro,
+			}).Debug("Package file NOT on Packagecloud")
+			xferBins = append(xferBins, src)
+		}
+	}
+
+	return xferBins
+}
+
+func main() {
+	flag.Usage = usage
+	verbose := flag.Bool("debug", false, "debug mode")
+	dlPath := flag.String("dir", defaultOBSDest, "Destination directory of packages download from OBS")
+	dryRun := flag.Bool("dry-run", false, "dry-run mode (do not download/upload files)")
+	pcDelete := flag.Bool("delete", false, "Delete Packagecloud packages that are not published on OBS")
+	showVersion := flag.Bool("version", false, "show the version")
+
+	flag.Parse()
+
+	if *showVersion {
+		fmt.Printf("%s %s (commit %v)\n", name, version, commit)
+		os.Exit(0)
+	}
+
+	if *verbose {
+		fmt.Println("Starting in debug mode...")
+		logrus.SetLevel(logrus.DebugLevel)
+	}
+
+	var configFile string
+	if len(flag.Args()) > 0 {
+		configFile = flag.Args()[0]
+	} else {
+		configFile = defaultConfig
+	}
+	logrus.Debugf("Using config file %s", configFile)
+
+	cfg, err := getConfig(configFile)
+	if err != nil {
+		logrus.WithError(err).Error("Failed to read config")
+		os.Exit(-1)
+	}
+	logrus.Debugf("Configuration file content: %+v", cfg)
+
+	var pc PCClient
+	if err := pc.PackagecloudClient(
+		cfg.Packagecloud.Auth.User,
+		cfg.Packagecloud.Auth.Token,
+		cfg.Packagecloud.Repo); err != nil {
+		logrus.WithError(err).Error("Failed to create a Packagecloud client instance")
+		os.Exit(-1)
+	}
+	logrus.WithFields(logrus.Fields{
+		"repo": pc.Repo,
+	}).Info("Retrieving Packagecloud list of files")
+
+	pcPackages, err := pc.PackagecloudList()
+	if err != nil {
+		logrus.WithFields(logrus.Fields{
+			"repo":  pc.Repo,
+			"error": err,
+		}).Error("Failed to retrieve Packagecloud list of files")
+		os.Exit(-1)
+	}
+
+	// lookup table for pcPackages packages that should NOT be deleted from Packagecloud
+	pcPackagesNeeded := make([]bool, len(pcPackages))
+
+	projects := getOBSProjects(cfg.OBSProjects)
+	for _, proj := range projects {
+		logrus.WithFields(logrus.Fields{
+			"OBS project": proj.Name,
+		}).Info("Retrieving packages info")
+
+		obsPackages, err := proj.FindAllPackages()
+		if err != nil {
+			logrus.WithError(err).WithFields(logrus.Fields{
+				"project": proj.Name,
+			}).Error("Failed to get OBS packages")
+			os.Exit(-1)
+		}
+
+		logrus.WithFields(logrus.Fields{
+			"OBS project": proj.Name,
+		}).Infof("Found %d packages", len(obsPackages))
+
+		totalXferred := 0
+		for _, pkg := range obsPackages {
+			pcDistro, found := cfg.DistroMapping[pkg.Repo]
+			if !found {
+				logrus.WithFields(logrus.Fields{
+					"OBS Repo": pkg.Repo,
+				}).Warn("No mapped Packagecloud distro specified")
+				os.Exit(-1)
+			}
+
+			if pcDistro == "" {
+				logrus.WithFields(logrus.Fields{
+					"package":  pkg.Name,
+					"OBS Repo": pkg.Repo,
+				}).Warn("Repo not supported by Packagecloud")
+				continue
+			}
+
+			xferBins := getXferBinaries(pkg, pcDistro, pcPackages, &pcPackagesNeeded)
+			if len(xferBins) == 0 {
+				logrus.WithFields(logrus.Fields{
+					"package":  pkg.Name,
+					"OBS Repo": pkg.Repo,
+				}).Infof("All %d files already on Packagecloud", len(pkg.Files))
+				continue
+			}
+
+			logrus.WithFields(logrus.Fields{
+				"pkg":     pkg.Name,
+				"repo":    pkg.Repo,
+				"# files": len(xferBins),
+			}).Info("Downloading from OBS")
+
+			if *dryRun {
+				continue
+			}
+
+			pkg.Files = xferBins
+			paths, err := proj.DownloadPackageFiles(pkg, *dlPath)
+			if err != nil {
+				logrus.WithError(err).Warnf("Failed to download binaries for %s on %s/%s", pkg.Name, pc.Repo, pcDistro)
+				continue
+			}
+
+			logrus.WithFields(logrus.Fields{
+				"pkg":     pkg.Name,
+				"distro":  pcDistro,
+				"# files": len(xferBins),
+			}).Info("Uploading to Packagecloud")
+
+			err = pc.PackagecloudPush(paths, pcDistro)
+			if err != nil {
+				logrus.WithError(err).WithFields(logrus.Fields{
+					"package": pkg.Name,
+					"distro":  pcDistro,
+				}).Error("Failed to push binaries to Packagecloud")
+			}
+
+			totalXferred += len(xferBins)
+		}
+
+		logrus.WithFields(logrus.Fields{
+			"OBS project":       proj.Name,
+			"Packagecloud Repo": pc.Repo,
+		}).Infof("Successfully transferred %d files", totalXferred)
+	}
+
+	if !*pcDelete {
+		return
+	}
+
+	logrus.WithFields(logrus.Fields{
+		"Repo": pc.Repo,
+	}).Info("Finding and deleting Packagecloud files not published on OBS")
+	progressBar := pb.New(len(pcPackages))
+	progressBar.SetMaxWidth(100)
+	progressBar.Start()
+	totalDeleted := 0
+	for i, pkg := range pcPackages {
+		if !pcPackagesNeeded[i] {
+			if *dryRun {
+				continue
+			}
+			if err := pc.PackagecloudDelete(pkg.Filename, pkg.DistroVersion); err != nil {
+				logrus.WithError(err).WithFields(logrus.Fields{
+					"name":   pkg.Filename,
+					"distro": pkg.DistroVersion,
+				}).Error("Failed to delete package on Packagecloud")
+			}
+			totalDeleted++
+		}
+		progressBar.Increment()
+	}
+	progressBar.Finish()
+
+	logrus.WithFields(logrus.Fields{
+		"Packagecloud Repo": pc.Repo,
+	}).Infof("Deleted %d files", totalDeleted)
+}

tools/packaging/cmd/kata-pkgsync/cli/packagecloud.go

@@ -0,0 +1,69 @@
+// Copyright (c) 2017-2018 Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package main
+
+import (
+	"path"
+
+	"github.com/mlafeldt/pkgcloud"
+	"github.com/sirupsen/logrus"
+)
+
+type PCClient struct {
+	*pkgcloud.Client
+	Repo string
+}
+
+func (pc *PCClient) PackagecloudClient(user string, token string, repo string) error {
+	client, err := pkgcloud.NewClient(token)
+	if err != nil {
+		return err
+	}
+
+	*pc = PCClient{client, path.Join(user, repo)}
+	pc.ShowProgress(true)
+
+	return nil
+}
+
+func (pc PCClient) PackagecloudList() ([]pkgcloud.Package, error) {
+	logrus.WithFields(logrus.Fields{
+		"repo": pc.Repo,
+	}).Debug("Packagecloud listing package(s)")
+	return pc.All(pc.Repo)
+}
+
+func (pc PCClient) PackagecloudSearchPackage(pkg string) ([]pkgcloud.Package, error) {
+	logrus.WithFields(logrus.Fields{
+		"repo": pc.Repo,
+		"pkg":  pkg,
+	}).Debug("Packagecloud searching package")
+	return pc.Search(pc.Repo, pkg, "", "", 0)
+}
+
+func (pc PCClient) PackagecloudPush(packages []string, distro string) error {
+	logrus.WithFields(logrus.Fields{
+		"repo":   pc.Repo,
+		"#":      len(packages),
+		"distro": distro,
+	}).Debug("Packagecloud pushing package")
+	for _, pkg := range packages {
+		if err := pc.CreatePackage(pc.Repo, distro, pkg); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (pc PCClient) PackagecloudDelete(filename string, distro string) error {
+	logrus.WithFields(logrus.Fields{
+		"repo":     pc.Repo,
+		"filename": filename,
+		"distro":   distro,
+	}).Debug("Packagecloud delete package")
+	return pc.Destroy(pc.Repo, path.Join(distro, filename))
+}

tools/packaging/cmd/kata-pkgsync/cli/utils.go

@@ -0,0 +1,26 @@
+// Copyright (c) 2019 SUSE LLC
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package main
+
+import (
+	"fmt"
+	"io/ioutil"
+
+	"github.com/pkg/errors"
+	"gopkg.in/yaml.v2"
+)
+
+func yamlUnmarshal(yamlFile string, cfg interface{}) error {
+	source, err := ioutil.ReadFile(yamlFile)
+	if err != nil {
+		return err
+	}
+	err = yaml.Unmarshal(source, cfg)
+	if err != nil {
+		return errors.Wrapf(err, fmt.Sprintf("cannot unmarshal %s", yamlFile))
+	}
+	return nil
+}