soft-serve

  1package jobs
  2
  3import (
  4	"context"
  5	"fmt"
  6	"path/filepath"
  7	"runtime"
  8	"strings"
  9
 10	"charm.land/log/v2"
 11	"github.com/charmbracelet/soft-serve/git"
 12	"github.com/charmbracelet/soft-serve/pkg/backend"
 13	"github.com/charmbracelet/soft-serve/pkg/config"
 14	"github.com/charmbracelet/soft-serve/pkg/db"
 15	"github.com/charmbracelet/soft-serve/pkg/lfs"
 16	"github.com/charmbracelet/soft-serve/pkg/proto"
 17	"github.com/charmbracelet/soft-serve/pkg/store"
 18	"github.com/charmbracelet/soft-serve/pkg/sync"
 19	"github.com/spf13/cobra"
 20)
 21
 22func init() {
 23	Register("mirror-pull", mirrorPull{})
 24}
 25
 26type mirrorPull struct{}
 27
 28// Spec derives the spec used for pull mirrors and implements Runner.
 29func (m mirrorPull) Spec(ctx context.Context) string {
 30	cfg := config.FromContext(ctx)
 31	if cfg.Jobs.MirrorPull != "" {
 32		return cfg.Jobs.MirrorPull
 33	}
 34	return "@every 10m"
 35}
 36
 37func (m mirrorPull) Command() *cobra.Command {
 38	cmd := &cobra.Command{
 39		Use:   "mirror_pull",
 40		Short: "Fetch upstream of mirror repos",
 41		RunE:  m.Func,
 42	}
 43	return cmd
 44}
 45
 46// Func runs the (pull) mirror job task and implements Runner.
 47func (m mirrorPull) Func(cmd *cobra.Command, args []string) error {
 48	var (
 49		repos []proto.Repository
 50		err   error
 51	)
 52	ctx := cmd.Context()
 53	cfg := config.FromContext(ctx)
 54	logger := log.FromContext(ctx).WithPrefix("jobs.mirror")
 55	b := backend.FromContext(ctx)
 56	dbx := db.FromContext(ctx)
 57	datastore := store.FromContext(ctx)
 58	if len(args) != 0 {
 59		repo, err := b.Repository(ctx, args[0])
 60		if err != nil {
 61			logger.Error("error getting repositorie", "repo", repo, "err", err)
 62			return err
 63		}
 64		repos = []proto.Repository{}
 65	} else {
 66		repos, err = b.Repositories(ctx)
 67		if err != nil {
 68			logger.Error("error getting repositories", "err", err)
 69			return err
 70		}
 71	}
 72
 73	// Divide the work up among the number of CPUs.
 74	wq := sync.NewWorkPool(ctx, runtime.GOMAXPROCS(0),
 75		sync.WithWorkPoolLogger(logger.Errorf),
 76	)
 77
 78	logger.Debug("updating mirror repos")
 79	for _, repo := range repos {
 80		if repo.IsMirror() {
 81			r, err := repo.Open()
 82			if err != nil {
 83				logger.Error("error opening repository", "repo", repo.Name(), "err", err)
 84				continue
 85			}
 86
 87			name := repo.Name()
 88			wq.Add(name, func() {
 89				repo := repo
 90
 91				cmds := []string{
 92					"fetch --prune",         // fetch prune before updating remote
 93					"remote update --prune", // update remote and prune remote refs
 94				}
 95
 96				for _, c := range cmds {
 97					args := strings.Split(c, " ")
 98					gitCmd := git.NewCommand(args...).WithContext(ctx)
 99					gitCmd.AddEnvs(
100						fmt.Sprintf(`GIT_SSH_COMMAND=ssh -o UserKnownHostsFile="%s" -o StrictHostKeyChecking=no -i "%s"`,
101							filepath.Join(cfg.DataPath, "ssh", "known_hosts"),
102							cfg.SSH.ClientKeyPath,
103						),
104					)
105
106					if err := gitCmd.RunInDirWithOptions(r.Path, git.RunInDirOptions{
107						Stdin:  nil,
108						Stdout: cmd.OutOrStdout(),
109						Stderr: cmd.OutOrStderr(),
110					}); err != nil {
111						logger.Error("error running git remote update", "repo", name, "err", err)
112					}
113				}
114
115				if cfg.LFS.Enabled {
116					rcfg, err := r.Config()
117					if err != nil {
118						logger.Error("error getting git config", "repo", name, "err", err)
119						return
120					}
121
122					lfsEndpoint := rcfg.Section("lfs").Option("url")
123					if lfsEndpoint == "" {
124						// If there is no LFS url defined, means the repo
125						// doesn't use LFS and we can skip it.
126						return
127					}
128
129					ep, err := lfs.NewEndpoint(lfsEndpoint)
130					if err != nil {
131						logger.Error("error creating LFS endpoint", "repo", name, "err", err)
132						return
133					}
134
135					client := lfs.NewClient(ep)
136					if client == nil {
137						logger.Errorf("failed to create lfs client: unsupported endpoint %s", lfsEndpoint)
138						return
139					}
140
141					if err := backend.StoreRepoMissingLFSObjects(ctx, repo, dbx, datastore, client); err != nil {
142						logger.Error("failed to store missing lfs objects", "err", err, "path", r.Path)
143						return
144					}
145				}
146			})
147		}
148	}
149
150	wq.Run()
151	return nil
152}