package local import ( "encoding/json" "errors" "fmt" "math/rand" "net" "os" "path" "path/filepath" "sort" "strconv" "strings" "time" "github.com/cooperspencer/gickup/gitcmd" "github.com/cooperspencer/gickup/logger" "github.com/cooperspencer/gickup/types" "github.com/cooperspencer/gickup/zip" "github.com/go-git/go-git/v5" "github.com/go-git/go-git/v5/config" "github.com/go-git/go-git/v5/plumbing" "github.com/go-git/go-git/v5/plumbing/transport" "github.com/go-git/go-git/v5/plumbing/transport/http" "github.com/go-git/go-git/v5/plumbing/transport/ssh" "github.com/melbahja/goph" "github.com/rs/zerolog" gossh "golang.org/x/crypto/ssh" ) var ( gitc = gitcmd.GitCmd{} sub zerolog.Logger ) // tokenAuth returns the HTTP basic auth for a repo that authenticates via token. func tokenAuth(repo types.Repo) *http.BasicAuth { if repo.Token == "" { return nil } if repo.NoTokenUser { return &http.BasicAuth{ Username: "x-access-token", Password: repo.Token, } } return &http.BasicAuth{ Username: repo.Origin.User, Password: repo.Token, } } func toGitCmdAuth(auth transport.AuthMethod) *gitcmd.Auth { if basicAuth, ok := auth.(*http.BasicAuth); ok && basicAuth != nil { return &gitcmd.Auth{ Username: basicAuth.Username, Password: basicAuth.Password, } } return nil } // Locally TODO. func Locally(repo types.Repo, l types.Local, dry bool) bool { sub = logger.CreateSubLogger("stage", "locally", "path", l.Path) if l.LFS { g, err := gitcmd.New() if err != nil { sub.Error(). Msg(err.Error()) } gitc = g } date := time.Now() if l.Structured { repo.Name = path.Join(repo.Hoster, repo.Owner, repo.Name) } if l.Bare || l.Mirror { repo.Name += ".git" } if l.Keep > 0 { repo.Name = path.Join(repo.Name, fmt.Sprint(date.Unix())) } _, err := os.Stat(l.Path) if os.IsNotExist(err) && !dry { if err = os.MkdirAll(l.Path, 0o777); err != nil { sub.Error(). Msg(err.Error()) return false } _, err = os.Stat(l.Path) } if err != nil { sub.Error(). Msg(err.Error()) return false } tries := 5 var auth transport.AuthMethod switch { case repo.Origin.SSH: if repo.Origin.SSHKey == "" { home := os.Getenv("HOME") repo.Origin.SSHKey = path.Join(home, ".ssh", "id_rsa") } auth, err = ssh.NewPublicKeysFromFile("git", repo.Origin.SSHKey, "") if err != nil { sub.Error(). Msg(err.Error()) return false } case repo.Token != "": auth = tokenAuth(repo) case repo.Origin.Username != "" && repo.Origin.Password != "": auth = &http.BasicAuth{ Username: repo.Origin.Username, Password: repo.Origin.Password, } } for x := 1; x <= tries; x++ { stat, err := os.Stat(filepath.Join(l.Path, repo.Name)) if os.IsNotExist(err) { sub.Info(). Msgf("cloning %s", types.Green(repo.Name)) err := cloneRepository(repo, auth, dry, l) if err != nil { if err.Error() == "repository not found" { sub.Warn(). Str("repo", repo.Name). Msg(err.Error()) return false } if x == tries { sub.Warn(). Str("repo", repo.Name). Msg(err.Error()) return false } if strings.Contains(err.Error(), "ERR access denied or repository not exported") { sub.Warn(). Str("repo", repo.Name). Msgf("%s doesn't exist.", repo.Name) return false } if strings.Contains(err.Error(), "remote repository is empty") { sub.Warn(). Str("repo", repo.Name). Msg(err.Error()) break } /* err = os.RemoveAll(filepath.Join(l.Path, repo.Name)) if err != nil { dir, _ := filepath.Abs(filepath.Join(l.Path, repo.Name)) sub.Warn(). Str("repo", repo.Name).Err(err). Msgf("couldn't remove %s", types.Red(dir)) } */ sub.Warn().Err(err). Msgf("retry %s from %s", types.Red(x), types.Red(tries)) time.Sleep(5 * time.Second) continue } } else { if !stat.IsDir() { sub.Warn(). Str("repo", repo.Name). Msgf("%s is a file", types.Red(repo.Name)) } else { sub.Info(). Msgf("opening %s locally", types.Green(repo.Name)) err := updateRepository(repo, auth, dry, l) if err != nil { switch { case errors.Is(err, git.NoErrAlreadyUpToDate): sub.Info(). Msg(err.Error()) case x == tries: sub.Warn(). Str("repo", repo.Name). Msg(err.Error()) return false default: sub.Warn(). Str("repo", repo.Name).Err(err). Msgf("retry %s from %s", types.Red(x), types.Red(tries)) time.Sleep(5 * time.Second) continue } } } } if len(repo.Issues) > 0 { _, err := os.Stat(filepath.Join(l.Path, fmt.Sprintf("%s.issues", repo.Name))) if os.IsNotExist(err) && !dry { if err := os.MkdirAll(filepath.Join(l.Path, fmt.Sprintf("%s.issues", repo.Name)), 0o777); err != nil { sub.Error(). Msg(err.Error()) } } issuesDir, err := filepath.Abs(filepath.Join(l.Path, fmt.Sprintf("%s.issues", repo.Name))) if err != nil { sub.Error(). Msg(err.Error()) } else { sub.Info().Str("repo", repo.Name).Msg("backing up issues") if !dry { for k, v := range repo.Issues { jsonData, err := json.Marshal(v) if err != nil { sub.Error(). Msg(err.Error()) } else { err = os.WriteFile(filepath.Join(issuesDir, fmt.Sprintf("%s.json", k)), jsonData, 0o644) if err != nil { sub.Error(). Msg(err.Error()) } } } } } } if l.Zip { tozip := []string{filepath.Join(l.Path, repo.Name)} if len(repo.Issues) > 0 { tozip = append(tozip, filepath.Join(l.Path, fmt.Sprintf("%s.issues", repo.Name))) } sub.Info(). Msgf("zipping %s", types.Green(repo.Name)) if _, err := os.Stat(fmt.Sprintf("%s.zip", filepath.Join(l.Path, repo.Name))); !os.IsNotExist(err) { sub.Warn().Str("repo", repo.Name).Msgf("will overwrite %s.zip", filepath.Join(l.Path, repo.Name)) } err := zip.Zip(filepath.Join(l.Path, repo.Name), tozip) if err != nil { sub.Error(). Str("repo", repo.Name). Msg(err.Error()) return false } } if l.Keep > 0 { if err := pruneOldBackups(filepath.Dir(filepath.Join(l.Path, repo.Name)), repo.Name, l); err != nil { sub.Warn(). Str("repo", repo.Name).Msg(err.Error()) return false } } x = 5 } return true } // pruneOldBackups enforces l.Keep by removing the oldest timestamped backups func pruneOldBackups(parentdir string, repoName string, l types.Local) error { files, err := os.ReadDir(parentdir) if err != nil { return err } // A backup generation is the "" directory plus "." // (like ".issues") // They are grouped by their timestamp so that together they only count as one backup against l.Keep. generations := map[int64][]string{} for _, file := range files { stamp, _, _ := strings.Cut(file.Name(), ".") ts, err := strconv.ParseInt(stamp, 10, 64) if err != nil { sub.Warn(). Str("repo", repoName). Msgf("couldn't parse timestamp! %s", types.Red(file.Name())) continue } generations[ts] = append(generations[ts], file.Name()) } if len(generations) <= l.Keep { return nil } stamps := make([]int64, 0, len(generations)) for ts := range generations { stamps = append(stamps, ts) } sort.Slice(stamps, func(i, j int) bool { return stamps[i] > stamps[j] }) for _, ts := range stamps[l.Keep:] { for _, file := range generations[ts] { sub.Info(). Msgf("removing %s", types.Red(filepath.Join(parentdir, file))) err := os.RemoveAll(filepath.Join(parentdir, file)) if err != nil { sub.Warn(). Str("repo", repoName).Msg(err.Error()) } } } return nil } func sanitizeRemote(r *git.Repository, cleanURL string) error { if cleanURL == "" { return nil } cfg, err := r.Config() if err != nil { return err } rem, ok := cfg.Remotes["origin"] if !ok || rem == nil { return nil } changed := false for i, u := range rem.URLs { if (strings.HasPrefix(u, "http://") || strings.HasPrefix(u, "https://")) && strings.Contains(u, "@") { rem.URLs[i] = cleanURL changed = true } } if changed { return r.SetConfig(cfg) } return nil } func updateRepository(repo types.Repo, auth transport.AuthMethod, dry bool, l types.Local) error { r, err := git.PlainOpen(filepath.Join(l.Path, repo.Name)) if err != nil { return err } if !dry { if l.LFS { _, err = os.Stat(filepath.Join(l.Path, repo.Name)) if err != nil { return err } if !repo.Origin.SSH && repo.URL != "" { if err := sanitizeRemote(r, repo.URL); err != nil { sub.Warn().Str("repo", repo.Name).Msgf("could not sanitize remote: %s", err) } } sub.Info(). Msgf("pulling %s", types.Green(repo.Name)) err = gitc.Pull(l.Bare, l.Mirror, filepath.Join(l.Path, repo.Name), toGitCmdAuth(auth)) if err != nil { return err } if l.Bare || l.Mirror { sub.Info(). Msgf("fetching lfs files for %s", types.Green(repo.Name)) err = gitc.LFSFetch(filepath.Join(l.Path, repo.Name), toGitCmdAuth(auth)) if err != nil { return err } } } else { // fetch to see if there are any unpullable commits, for example a force push err = r.Fetch(&git.FetchOptions{Auth: auth, RemoteName: "origin"}) if err != nil && !errors.Is(err, git.NoErrAlreadyUpToDate) { return err } sub.Info(). Msgf("pulling %s", types.Green(repo.Name)) if !l.Bare && !l.Mirror { w, worktreeErr := r.Worktree() if worktreeErr != nil && !errors.Is(worktreeErr, git.NoErrAlreadyUpToDate) { return worktreeErr } err = w.Pull(&git.PullOptions{Auth: auth, RemoteName: "origin", SingleBranch: false}) if err != nil && !errors.Is(err, git.NoErrAlreadyUpToDate) { return err } } // if everything was ok, fetch everything err = r.Fetch(&git.FetchOptions{Auth: auth, RemoteName: "origin", RefSpecs: []config.RefSpec{"+refs/*:refs/*"}}) if err != nil { return err } } } return err } func cloneRepository(repo types.Repo, auth transport.AuthMethod, dry bool, l types.Local) error { if dry { return nil } url := repo.URL if repo.Origin.SSH { url = repo.SSHURL site := types.Site{} err := site.GetValues(url) if err != nil { sub.Fatal().Str("repo", repo.Name).Msg(err.Error()) } sshAuth, err := goph.Key(repo.Origin.SSHKey, "") if err != nil { sub.Fatal().Str("repo", repo.Name).Msg(err.Error()) } err = testSSHConnection(site, sshAuth) if err != nil { sub.Fatal().Str("repo", repo.Name).Msg(err.Error()) } } remoteConfig := config.RemoteConfig{ Name: "origin", URLs: []string{url}, } rem := git.NewRemote(nil, &remoteConfig) _, err := rem.List(&git.ListOptions{Auth: auth}) if err != nil { return err } if l.LFS { err = gitc.Clone(url, filepath.Join(l.Path, repo.Name), l.Bare, l.Mirror, toGitCmdAuth(auth)) if err != nil { return err } if l.Bare || l.Mirror { sub.Info(). Msgf("fetching lfs files for %s", types.Green(repo.Name)) err = gitc.LFSFetch(filepath.Join(l.Path, repo.Name), toGitCmdAuth(auth)) if err != nil { return err } } } else { r := &git.Repository{} r, err = git.PlainClone(filepath.Join(l.Path, repo.Name), l.Bare, &git.CloneOptions{ URL: url, Auth: auth, SingleBranch: false, Mirror: l.Mirror, }) if err != nil { return err } err = r.Fetch(&git.FetchOptions{ RefSpecs: []config.RefSpec{"refs/*:refs/*"}, Auth: auth, Force: true, }) if errors.Is(err, git.NoErrAlreadyUpToDate) { err = nil } } return err } func testSSHConnection(site types.Site, sshAuth goph.Auth) error { _, err := goph.NewConn(&goph.Config{ User: site.User, Addr: site.URL, Port: uint(site.Port), Auth: sshAuth, Callback: VerifyHost, }) return err } // VerifyHost TODO. func VerifyHost(host string, remote net.Addr, key gossh.PublicKey) error { // Got from the example from // https://github.com/melbahja/goph/blob/master/examples/goph/main.go // // If you want to connect to new hosts. // here your should check new connections public keys // if the key not trusted you shuld return an error // // hostFound: is host in known hosts file. // err: error if key not in known hosts file // OR host in known hosts file but key changed! hostFound, err := goph.CheckKnownHost(host, remote, key, "") // Host in known hosts but key mismatch! // Maybe because of MAN IN THE MIDDLE ATTACK! /* if hostFound && err != nil { return err } */ // handshake because public key already exists. if hostFound && err == nil { return nil } // Add the new host to known hosts file. return goph.AddKnownHost(host, remote, key, "") } func TempClone(repo types.Repo, tempdir string) (*git.Repository, error) { return tempCloneBase(repo, tempdir, false) } func TempCloneBare(repo types.Repo, tempdir string) (*git.Repository, error) { return tempCloneBase(repo, tempdir, true) } func tempCloneBase(repo types.Repo, tempdir string, isBare bool) (*git.Repository, error) { var auth transport.AuthMethod if repo.Token != "" { auth = tokenAuth(repo) } else if repo.Origin.Username != "" && repo.Origin.Password != "" { auth = &http.BasicAuth{ Username: repo.Origin.Username, Password: repo.Origin.Password, } } if repo.Origin.LFS { g, err := gitcmd.New() if err != nil { return nil, err } gitc = g err = gitc.Clone(repo.URL, tempdir, isBare, false, toGitCmdAuth(auth)) if err != nil { return nil, err } r, err := git.PlainOpen(tempdir) if err != nil { return nil, err } err = r.Fetch(&git.FetchOptions{ RefSpecs: []config.RefSpec{"refs/*:refs/*"}, Auth: auth, Force: true, }) if err != nil && !errors.Is(err, git.NoErrAlreadyUpToDate) { return r, err } // A regular clone only downloads the LFS objects needed for the // checked-out branch. Without fetching the rest, objects referenced // by other branches/tags or older commits are missing locally, // which makes the later `git push --all` fail with // "missing or corrupt local objects". sub.Info(). Msgf("fetching lfs files for %s", types.Green(repo.Name)) if lfsErr := gitc.LFSFetch(tempdir, toGitCmdAuth(auth)); lfsErr != nil { return r, lfsErr } // Bare backups already contain all refs and LFS objects. // Checkout and pull require a working tree and must not run for these clones. if isBare { return r, nil } // Get the symbolic reference for HEAD headRef, err := r.Head() if err != nil { return nil, err } // Retrieve the list of branches refs, err := r.Branches() if err != nil { return nil, err } // Print the names of branches err = refs.ForEach(func(ref *plumbing.Reference) error { if ref.Name().Short() != headRef.Name().Short() { return gitc.Checkout(tempdir, ref.Name().Short()) } return nil }) if err != nil { return nil, err } err = gitc.Checkout(tempdir, headRef.Name().Short()) if err != nil { return nil, err } err = gitc.MirrorPull(tempdir, toGitCmdAuth(auth)) if err != nil { return nil, err } return r, err } r, err := git.PlainClone(tempdir, isBare, &git.CloneOptions{ URL: repo.URL, Auth: auth, SingleBranch: false, }) if err != nil { return nil, err } err = r.Fetch(&git.FetchOptions{ RefSpecs: []config.RefSpec{"refs/*:refs/*"}, Auth: auth, Force: true, }) if errors.Is(err, git.NoErrAlreadyUpToDate) { return r, nil } return r, err } // CreateRemotePush pushes to url using the caller's resolved token or SSH configuration. func CreateRemotePush(repo *git.Repository, destination types.GenRepo, url, token string, lfs bool) error { sub = logger.CreateSubLogger("stage", "tempclone", "url", url) var auth transport.AuthMethod if destination.SSH { if destination.SSHKey == "" { home := os.Getenv("HOME") destination.SSHKey = path.Join(home, ".ssh", "id_rsa") } site := types.Site{} err := site.GetValues(url) if err != nil { sub.Fatal().Msg(err.Error()) } sshAuth, err := goph.Key(destination.SSHKey, "") if err != nil { sub.Fatal().Msg(err.Error()) } err = testSSHConnection(site, sshAuth) if err != nil { sub.Fatal().Msg(err.Error()) } if destination.SSHKey == "" { home := os.Getenv("HOME") destination.SSHKey = path.Join(home, ".ssh", "id_rsa") } auth, err = ssh.NewPublicKeysFromFile("git", destination.SSHKey, "") if err != nil { return err } } else { auth = &http.BasicAuth{ Username: "xyz", Password: token, } } if lfs { g, err := gitcmd.New() if err != nil { return err } gitc = g worktree, err := repo.Worktree() if err != nil { return err } remote := RandomString(8) if destination.SSH { err = gitc.NewRemote(remote, url, worktree.Filesystem.Root()) if err != nil { return err } err = gitc.SSHPush(worktree.Filesystem.Root(), remote, destination.SSHKey) if err != nil { return err } } else { err = gitc.NewRemote(remote, url, worktree.Filesystem.Root()) if err != nil { return err } err = gitc.Push(worktree.Filesystem.Root(), remote, toGitCmdAuth(auth)) if err != nil { return err } } return nil } remoteconfig := config.RemoteConfig{Name: RandomString(8), URLs: []string{url}} remote, err := repo.CreateRemote(&remoteconfig) if err != nil { return err } headref, _ := repo.Head() pushoptions := git.PushOptions{Force: destination.Force, Auth: auth, RemoteName: remote.Config().Name, RefSpecs: []config.RefSpec{config.RefSpec(fmt.Sprintf("%s:%s", headref.Name(), headref.Name()))}} err = repo.Push(&pushoptions) if err == nil || errors.Is(err, git.NoErrAlreadyUpToDate) { refspecs, err := getPushRefSpecs(repo, headref.Name()) if err != nil { return err } var pushErrors []error // Keep valid refs moving when a destination hook rejects one branch or tag. for _, refspec := range refspecs { pushoptions.RefSpecs = []config.RefSpec{refspec} err = repo.Push(&pushoptions) if err != nil && !errors.Is(err, git.NoErrAlreadyUpToDate) { pushErrors = append(pushErrors, err) } } return errors.Join(pushErrors...) } return err } func getPushRefSpecs(repo *git.Repository, skip plumbing.ReferenceName) ([]config.RefSpec, error) { refs, err := repo.References() if err != nil { return nil, err } refspecs := []config.RefSpec{} err = refs.ForEach(func(ref *plumbing.Reference) error { name := ref.Name() if name == skip || (!name.IsBranch() && !name.IsTag()) { return nil } refspecs = append(refspecs, config.RefSpec(fmt.Sprintf("%s:%s", name, name))) return nil }) return refspecs, err } func RandomString(length int) string { charset := "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" seededRand := rand.New(rand.NewSource(time.Now().UnixNano())) b := make([]byte, length) for i := range b { b[i] = charset[seededRand.Intn(len(charset))] } return string(b) }