mirror of
https://github.com/cooperspencer/gickup.git
synced 2026-10-19 16:16:46 +00:00
Om Lahore
1774a02416
Both ScanAndCollect calls dropped their error. ScanAndCollect returns whatever it managed to collect alongside the error, so a failed or partial ListDescendantGroups left subgroups short without saying so, and the include/exclude org maps were built from an incomplete list. For includeorgs that silently drops repos from the backup set. The GetGroup call seven lines above each of these already logs and continues on error, so this matches the surrounding code.
656 lines
17 KiB
Go
656 lines
17 KiB
Go
package gitlab
|
|
|
|
import (
|
|
"fmt"
|
|
"net/http"
|
|
"path"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/cooperspencer/gickup/logger"
|
|
"github.com/cooperspencer/gickup/types"
|
|
"github.com/rs/zerolog"
|
|
gitlab "gitlab.com/gitlab-org/api/client-go"
|
|
)
|
|
|
|
var sub zerolog.Logger
|
|
|
|
func getRepoVisibility(visibility string, private bool) gitlab.VisibilityValue {
|
|
switch strings.ToLower(strings.TrimSpace(visibility)) {
|
|
case "private":
|
|
return gitlab.PrivateVisibility
|
|
case "public":
|
|
return gitlab.PublicVisibility
|
|
default:
|
|
if private {
|
|
return gitlab.PrivateVisibility
|
|
}
|
|
return gitlab.PublicVisibility
|
|
}
|
|
}
|
|
|
|
// Backup TODO.
|
|
func Backup(r types.Repo, d types.GenRepo, dry bool) bool {
|
|
var gitlabclient *gitlab.Client
|
|
token := d.GetToken()
|
|
var err error
|
|
if d.URL == "" {
|
|
d.URL = "https://gitlab.com"
|
|
gitlabclient, err = gitlab.NewClient(token)
|
|
} else {
|
|
gitlabclient, err = gitlab.NewClient(token, gitlab.WithBaseURL(d.URL))
|
|
}
|
|
sub = logger.CreateSubLogger("stage", "gitlab", "url", d.URL)
|
|
|
|
if err != nil {
|
|
sub.Error().
|
|
Msg(err.Error())
|
|
return false
|
|
}
|
|
|
|
sub.Info().
|
|
Msgf("mirroring %s to %s", types.Blue(r.Name), d.URL)
|
|
|
|
True := true
|
|
|
|
opt := gitlab.ListProjectsOptions{
|
|
Search: &r.Name,
|
|
Owned: &True,
|
|
}
|
|
|
|
projects, _, err := gitlabclient.Projects.ListProjects(&opt)
|
|
if err != nil {
|
|
sub.Error().Msg(err.Error())
|
|
return false
|
|
}
|
|
|
|
found := false
|
|
for _, p := range projects {
|
|
if p.Name == r.Name {
|
|
found = true
|
|
}
|
|
}
|
|
|
|
if dry || found {
|
|
return true
|
|
}
|
|
|
|
if r.Token != "" {
|
|
splittedurl := strings.Split(r.URL, "//")
|
|
|
|
r.URL = fmt.Sprintf("%s//%s:%s@%s",
|
|
splittedurl[0], r.Owner, r.Token, splittedurl[1])
|
|
}
|
|
|
|
visibility := getRepoVisibility(d.Visibility.Repositories, r.Private)
|
|
|
|
opts := &gitlab.CreateProjectOptions{
|
|
Mirror: &True,
|
|
ImportURL: &r.URL,
|
|
Name: &r.Name,
|
|
Description: &r.Description,
|
|
Visibility: gitlab.Ptr(visibility),
|
|
}
|
|
|
|
_, _, err = gitlabclient.Projects.CreateProject(opts)
|
|
if err != nil {
|
|
sub.Error().
|
|
Msg(err.Error())
|
|
return false
|
|
}
|
|
|
|
return true
|
|
}
|
|
|
|
// Get TODO.
|
|
func Get(conf *types.Conf) ([]types.Repo, bool) {
|
|
ran := false
|
|
repos := []types.Repo{}
|
|
inSlice := map[string]bool{}
|
|
for _, repo := range conf.Source.Gitlab {
|
|
if repo.URL == "" {
|
|
repo.URL = "https://gitlab.com"
|
|
}
|
|
err := repo.Filter.ParseDuration()
|
|
sub = logger.CreateSubLogger("stage", "gitlab", "url", repo.URL)
|
|
if err != nil {
|
|
sub.Warn().
|
|
Msg(err.Error())
|
|
}
|
|
ran = true
|
|
|
|
token := repo.GetToken()
|
|
client, err := gitlab.NewClient(token, gitlab.WithBaseURL(repo.URL))
|
|
if err != nil {
|
|
sub.Error().
|
|
Msg(err.Error())
|
|
continue
|
|
}
|
|
|
|
userNotInConfig := false
|
|
|
|
if repo.User == "" {
|
|
user, _, err := client.Users.CurrentUser()
|
|
if err != nil {
|
|
sub.Error().
|
|
Msg(err.Error())
|
|
continue
|
|
}
|
|
repo.User = user.Username
|
|
userNotInConfig = true
|
|
}
|
|
|
|
sub.Info().
|
|
Msgf("grabbing repositories from %s", repo.User)
|
|
gitlabrepos := []*gitlab.Project{}
|
|
gitlabgrouprepos := map[string][]*gitlab.Project{}
|
|
|
|
opt := &gitlab.ListProjectsOptions{Membership: gitlab.Ptr(true)}
|
|
users, _, err := client.Users.ListUsers(&gitlab.ListUsersOptions{Username: &repo.User})
|
|
if err != nil {
|
|
sub.Error().
|
|
Msg(err.Error())
|
|
continue
|
|
}
|
|
if len(users) == 0 {
|
|
sub.Error().Msgf("couldn't find user %s", repo.User)
|
|
break
|
|
}
|
|
|
|
opt.PerPage = 50
|
|
for _, user := range users {
|
|
if user.Username == repo.User {
|
|
i := int64(1)
|
|
for {
|
|
opt.Page = i
|
|
var projects []*gitlab.Project
|
|
if userNotInConfig {
|
|
projects, _, err = client.Projects.ListProjects(opt)
|
|
} else {
|
|
projects, _, err = client.Projects.ListUserProjects(user.ID, opt)
|
|
}
|
|
if err != nil {
|
|
sub.Error().
|
|
Msg(err.Error())
|
|
}
|
|
if len(projects) == 0 {
|
|
break
|
|
}
|
|
gitlabrepos = append(gitlabrepos, projects...)
|
|
i++
|
|
}
|
|
}
|
|
}
|
|
|
|
if repo.Starred {
|
|
for _, user := range users {
|
|
if user.Username == repo.User {
|
|
i := 1
|
|
for {
|
|
opt.Page = int64(i)
|
|
projects, _, err := client.Projects.ListUserStarredProjects(user.ID, opt)
|
|
if err != nil {
|
|
sub.Error().
|
|
Msg(err.Error())
|
|
}
|
|
if len(projects) == 0 {
|
|
break
|
|
}
|
|
gitlabrepos = append(gitlabrepos, projects...)
|
|
i++
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
include := types.GetMap(repo.Include)
|
|
includeorgs := types.GetMap(repo.IncludeOrgs)
|
|
exclude := types.GetMap(repo.Exclude)
|
|
excludeorgs := types.GetMap(repo.ExcludeOrgs)
|
|
languages := types.GetMap(repo.Filter.Languages)
|
|
|
|
for _, org := range repo.IncludeOrgs {
|
|
group, _, err := client.Groups.GetGroup(org, &gitlab.GetGroupOptions{})
|
|
if err != nil {
|
|
sub.Error().
|
|
Msg(err.Error())
|
|
continue
|
|
}
|
|
|
|
subgroups, err := gitlab.ScanAndCollect(func(p gitlab.PaginationOptionFunc) ([]*gitlab.Group, *gitlab.Response, error) {
|
|
return client.Groups.ListDescendantGroups(group.ID, &gitlab.ListDescendantGroupsOptions{}, p)
|
|
})
|
|
if err != nil {
|
|
sub.Error().
|
|
Msg(err.Error())
|
|
continue
|
|
}
|
|
|
|
for _, sub := range subgroups {
|
|
includeorgs[sub.FullPath] = true
|
|
}
|
|
}
|
|
|
|
for _, org := range repo.ExcludeOrgs {
|
|
group, _, err := client.Groups.GetGroup(org, &gitlab.GetGroupOptions{})
|
|
if err != nil {
|
|
sub.Error().
|
|
Msg(err.Error())
|
|
continue
|
|
}
|
|
|
|
subgroups, err := gitlab.ScanAndCollect(func(p gitlab.PaginationOptionFunc) ([]*gitlab.Group, *gitlab.Response, error) {
|
|
return client.Groups.ListDescendantGroups(group.ID, &gitlab.ListDescendantGroupsOptions{}, p)
|
|
})
|
|
if err != nil {
|
|
sub.Error().
|
|
Msg(err.Error())
|
|
continue
|
|
}
|
|
|
|
for _, sub := range subgroups {
|
|
excludeorgs[sub.FullPath] = true
|
|
}
|
|
}
|
|
|
|
for _, r := range gitlabrepos {
|
|
if !inSlice[r.PathWithNamespace] {
|
|
sub.Debug().Str("repo-type", "user").Msg(r.WebURL)
|
|
if repo.Filter.ExcludeForks {
|
|
if r.ForkedFromProject != nil {
|
|
continue
|
|
}
|
|
}
|
|
if repo.Filter.ExcludeArchived {
|
|
if r.Archived {
|
|
continue
|
|
}
|
|
}
|
|
|
|
if len(repo.Filter.Languages) > 0 {
|
|
langs, _, err := client.Projects.GetProjectLanguages(r.ID)
|
|
if err != nil {
|
|
sub.Error().
|
|
Msg(err.Error())
|
|
continue
|
|
}
|
|
language := ""
|
|
percentage := float32(0)
|
|
|
|
for lang, percent := range *langs {
|
|
if percent > percentage {
|
|
language = lang
|
|
percentage = percent
|
|
}
|
|
}
|
|
if !languages[strings.ToLower(language)] {
|
|
continue
|
|
}
|
|
}
|
|
|
|
if r.StarCount < int64(repo.Filter.Stars) {
|
|
continue
|
|
}
|
|
if time.Since(*r.LastActivityAt) > repo.Filter.LastActivityDuration && repo.Filter.LastActivityDuration != 0 {
|
|
continue
|
|
}
|
|
if exclude[r.Name] {
|
|
continue
|
|
}
|
|
if excludeorgs[r.Namespace.FullPath] {
|
|
continue
|
|
}
|
|
if include[r.Name] {
|
|
if r.RepositoryAccessLevel != gitlab.DisabledAccessControl {
|
|
repos = append(repos, types.Repo{
|
|
Name: r.Path,
|
|
URL: r.HTTPURLToRepo,
|
|
SSHURL: r.SSHURLToRepo,
|
|
Token: token,
|
|
Origin: repo,
|
|
Owner: r.Namespace.FullPath,
|
|
Hoster: types.GetHost(repo.URL),
|
|
Description: r.Description,
|
|
Private: r.Visibility == gitlab.PrivateVisibility,
|
|
Issues: GetIssues(r, client, repo),
|
|
})
|
|
}
|
|
|
|
if r.WikiEnabled && repo.Wiki {
|
|
if activeWiki(r, client, repo) {
|
|
httpURLToRepo := types.DotGitRx.ReplaceAllString(r.HTTPURLToRepo, ".wiki.git")
|
|
sshURLToRepo := types.DotGitRx.ReplaceAllString(r.SSHURLToRepo, ".wiki.git")
|
|
repos = append(repos, types.Repo{
|
|
Name: r.Path + ".wiki",
|
|
URL: httpURLToRepo,
|
|
SSHURL: sshURLToRepo,
|
|
Token: token,
|
|
Origin: repo,
|
|
Owner: r.Namespace.FullPath,
|
|
Hoster: types.GetHost(repo.URL),
|
|
Description: r.Description,
|
|
Private: r.Visibility == gitlab.PrivateVisibility,
|
|
})
|
|
}
|
|
}
|
|
|
|
continue
|
|
}
|
|
if len(include) == 0 {
|
|
if r.RepositoryAccessLevel != gitlab.DisabledAccessControl {
|
|
repos = append(repos, types.Repo{
|
|
Name: r.Path,
|
|
URL: r.HTTPURLToRepo,
|
|
SSHURL: r.SSHURLToRepo,
|
|
Token: token,
|
|
Origin: repo,
|
|
Owner: r.Namespace.FullPath,
|
|
Hoster: types.GetHost(repo.URL),
|
|
Description: r.Description,
|
|
Private: r.Visibility == gitlab.PrivateVisibility,
|
|
Issues: GetIssues(r, client, repo),
|
|
})
|
|
}
|
|
|
|
if r.WikiEnabled && repo.Wiki {
|
|
if activeWiki(r, client, repo) {
|
|
httpURLToRepo := types.DotGitRx.ReplaceAllString(r.HTTPURLToRepo, ".wiki.git")
|
|
sshURLToRepo := types.DotGitRx.ReplaceAllString(r.SSHURLToRepo, ".wiki.git")
|
|
repos = append(repos, types.Repo{
|
|
Name: r.Path + ".wiki",
|
|
URL: httpURLToRepo,
|
|
SSHURL: sshURLToRepo,
|
|
Token: token,
|
|
Origin: repo,
|
|
Owner: r.Namespace.FullPath,
|
|
Hoster: types.GetHost(repo.URL),
|
|
Description: r.Description,
|
|
Private: r.Visibility == gitlab.PrivateVisibility,
|
|
})
|
|
}
|
|
}
|
|
}
|
|
inSlice[r.PathWithNamespace] = true
|
|
}
|
|
}
|
|
|
|
if token != "" {
|
|
groups := []*gitlab.Group{}
|
|
i := int64(1)
|
|
for {
|
|
g, _, err := client.Groups.ListGroups(&gitlab.ListGroupsOptions{
|
|
ListOptions: gitlab.ListOptions{
|
|
Page: i,
|
|
PerPage: 50,
|
|
},
|
|
})
|
|
if err != nil {
|
|
sub.Error().Msg(err.Error())
|
|
}
|
|
|
|
if len(g) == 0 {
|
|
break
|
|
}
|
|
|
|
groups = append(groups, g...)
|
|
i++
|
|
}
|
|
|
|
gopt := &gitlab.ListGroupProjectsOptions{}
|
|
for _, group := range groups {
|
|
i = 1
|
|
gopt.PerPage = 50
|
|
gopt.Page = i
|
|
for {
|
|
projects, _, err := client.Groups.ListGroupProjects(group.ID, gopt)
|
|
if err != nil {
|
|
sub.Error().
|
|
Msg(err.Error())
|
|
}
|
|
if len(projects) == 0 {
|
|
break
|
|
}
|
|
splfullpath := strings.Split(group.FullPath, "/")
|
|
fullpath := path.Join(splfullpath...)
|
|
if _, ok := gitlabgrouprepos[fullpath]; !ok {
|
|
gitlabgrouprepos[fullpath] = []*gitlab.Project{}
|
|
}
|
|
|
|
gitlabgrouprepos[fullpath] = append(gitlabgrouprepos[fullpath], projects...)
|
|
i++
|
|
gopt.Page = i
|
|
}
|
|
}
|
|
for k, gr := range gitlabgrouprepos {
|
|
for _, r := range gr {
|
|
if !inSlice[r.PathWithNamespace] {
|
|
sub.Debug().Str("repo-type", "group").Msg(r.WebURL)
|
|
if repo.Filter.ExcludeForks {
|
|
if r.ForkedFromProject != nil {
|
|
continue
|
|
}
|
|
}
|
|
if repo.Filter.ExcludeArchived {
|
|
if r.Archived {
|
|
continue
|
|
}
|
|
}
|
|
|
|
if len(repo.Filter.Languages) > 0 {
|
|
langs, _, err := client.Projects.GetProjectLanguages(r.ID)
|
|
if err != nil {
|
|
sub.Error().
|
|
Msg(err.Error())
|
|
continue
|
|
}
|
|
language := ""
|
|
percentage := float32(0)
|
|
|
|
for lang, percent := range *langs {
|
|
if percent > percentage {
|
|
language = lang
|
|
percentage = percent
|
|
}
|
|
}
|
|
if !languages[strings.ToLower(language)] {
|
|
continue
|
|
}
|
|
}
|
|
|
|
if r.StarCount < int64(repo.Filter.Stars) {
|
|
continue
|
|
}
|
|
if time.Since(*r.LastActivityAt) > repo.Filter.LastActivityDuration && repo.Filter.LastActivityDuration != 0 {
|
|
continue
|
|
}
|
|
|
|
if exclude[r.Name] {
|
|
continue
|
|
}
|
|
if excludeorgs[r.Namespace.FullPath] {
|
|
continue
|
|
}
|
|
if include[r.Name] {
|
|
if r.RepositoryAccessLevel != gitlab.DisabledAccessControl {
|
|
repos = append(repos, types.Repo{
|
|
Name: r.Path,
|
|
URL: r.HTTPURLToRepo,
|
|
SSHURL: r.SSHURLToRepo,
|
|
Token: token,
|
|
Origin: repo,
|
|
Owner: k,
|
|
Hoster: types.GetHost(repo.URL),
|
|
Description: r.Description,
|
|
Private: r.Visibility == gitlab.PrivateVisibility,
|
|
Issues: GetIssues(r, client, repo),
|
|
})
|
|
}
|
|
|
|
if r.WikiEnabled && repo.Wiki {
|
|
if activeWiki(r, client, repo) {
|
|
httpURLToRepo := types.DotGitRx.ReplaceAllString(r.HTTPURLToRepo, ".wiki.git")
|
|
sshURLToRepo := types.DotGitRx.ReplaceAllString(r.SSHURLToRepo, ".wiki.git")
|
|
repos = append(repos, types.Repo{
|
|
Name: r.Path + ".wiki",
|
|
URL: httpURLToRepo,
|
|
SSHURL: sshURLToRepo,
|
|
Token: token,
|
|
Origin: repo,
|
|
Owner: k,
|
|
Hoster: types.GetHost(repo.URL),
|
|
Description: r.Description,
|
|
Private: r.Visibility == gitlab.PrivateVisibility,
|
|
})
|
|
}
|
|
}
|
|
|
|
continue
|
|
}
|
|
if len(include) == 0 {
|
|
if len(includeorgs) == 0 || includeorgs[r.Namespace.FullPath] {
|
|
if r.RepositoryAccessLevel != gitlab.DisabledAccessControl {
|
|
repos = append(repos, types.Repo{
|
|
Name: r.Path,
|
|
URL: r.HTTPURLToRepo,
|
|
SSHURL: r.SSHURLToRepo,
|
|
Token: token,
|
|
Origin: repo,
|
|
Owner: k,
|
|
Hoster: types.GetHost(repo.URL),
|
|
Description: r.Description,
|
|
Private: r.Visibility == gitlab.PrivateVisibility,
|
|
Issues: GetIssues(r, client, repo),
|
|
})
|
|
}
|
|
|
|
if r.WikiEnabled && repo.Wiki {
|
|
if activeWiki(r, client, repo) {
|
|
httpURLToRepo := types.DotGitRx.ReplaceAllString(r.HTTPURLToRepo, ".wiki.git")
|
|
sshURLToRepo := types.DotGitRx.ReplaceAllString(r.SSHURLToRepo, ".wiki.git")
|
|
repos = append(repos, types.Repo{
|
|
Name: r.Path + ".wiki",
|
|
URL: httpURLToRepo,
|
|
SSHURL: sshURLToRepo,
|
|
Token: token,
|
|
Origin: repo,
|
|
Owner: k,
|
|
Hoster: types.GetHost(repo.URL),
|
|
Description: r.Description,
|
|
Private: r.Visibility == gitlab.PrivateVisibility,
|
|
})
|
|
}
|
|
}
|
|
}
|
|
}
|
|
inSlice[r.PathWithNamespace] = true
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
return repos, ran
|
|
}
|
|
|
|
func activeWiki(r *gitlab.Project, client *gitlab.Client, _ types.GenRepo) bool {
|
|
wikilistoptions := &gitlab.ListWikisOptions{
|
|
WithContent: gitlab.Ptr(true),
|
|
}
|
|
|
|
wikis, _, err := client.Wikis.ListWikis(r.ID, wikilistoptions)
|
|
if err != nil {
|
|
sub.Warn().
|
|
Msg(err.Error())
|
|
}
|
|
|
|
return len(wikis) > 0
|
|
}
|
|
|
|
// GetIssues get issues
|
|
func GetIssues(repo *gitlab.Project, client *gitlab.Client, conf types.GenRepo) map[string]interface{} {
|
|
issues := map[string]interface{}{}
|
|
if conf.Issues {
|
|
listOptions := &gitlab.ListProjectIssuesOptions{ListOptions: gitlab.ListOptions{PerPage: 100}}
|
|
errorcount := 0
|
|
for {
|
|
i, response, err := client.Issues.ListProjectIssues(repo.ID, listOptions)
|
|
if err != nil {
|
|
if response.StatusCode == http.StatusForbidden {
|
|
sub.Error().Err(err).Str("repo", repo.Name).Msg("can't fetch issues")
|
|
return issues
|
|
}
|
|
if errorcount < 5 {
|
|
sub.Warn().Err(err).Str("repo", repo.Name).Msg("can't fetch issues")
|
|
time.Sleep(5 * time.Second)
|
|
errorcount++
|
|
} else {
|
|
return issues
|
|
}
|
|
} else {
|
|
if len(i) > 0 {
|
|
for _, issue := range i {
|
|
issues[strconv.FormatInt(issue.IID, 10)] = issue
|
|
}
|
|
} else {
|
|
break
|
|
}
|
|
listOptions.Page++
|
|
}
|
|
}
|
|
}
|
|
return issues
|
|
}
|
|
|
|
// GetOrCreate Get or create a repository
|
|
func GetOrCreate(destination types.GenRepo, repo types.Repo) (string, error) {
|
|
visibility := getRepoVisibility(destination.Visibility.Repositories, repo.Private)
|
|
|
|
sub = logger.CreateSubLogger("stage", "gitlab", "url", destination.URL)
|
|
|
|
token := destination.GetToken()
|
|
client, err := gitlab.NewClient(token, gitlab.WithBaseURL(destination.URL))
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
user, _, err := client.Users.CurrentUser()
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
targetNamespace := user.Username
|
|
if destination.User != "" {
|
|
targetNamespace = destination.User
|
|
}
|
|
|
|
fullPath := fmt.Sprintf("%s/%s", targetNamespace, repo.Name)
|
|
existingProject, _, err := client.Projects.GetProject(fullPath, &gitlab.GetProjectOptions{})
|
|
if err == nil && existingProject != nil {
|
|
return existingProject.HTTPURLToRepo, nil
|
|
}
|
|
|
|
opts := gitlab.CreateProjectOptions{
|
|
Name: gitlab.Ptr(repo.Name),
|
|
Visibility: gitlab.Ptr(visibility),
|
|
Description: gitlab.Ptr(repo.Description),
|
|
}
|
|
|
|
if targetNamespace != user.Username {
|
|
group, _, err := client.Groups.GetGroup(targetNamespace, &gitlab.GetGroupOptions{})
|
|
if err == nil {
|
|
opts.NamespaceID = &group.ID
|
|
} else {
|
|
users, _, err := client.Users.ListUsers(&gitlab.ListUsersOptions{Username: &targetNamespace})
|
|
if err != nil || len(users) == 0 {
|
|
return "", fmt.Errorf("target namespace %s not found", targetNamespace)
|
|
}
|
|
}
|
|
}
|
|
|
|
r, _, err := client.Projects.CreateProject(&opts)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
|
|
return r.HTTPURLToRepo, nil
|
|
}
|