diff --git a/.github/workflows/validate-lint.yml b/.github/workflows/validate-lint.yml new file mode 100644 index 0000000..9a9ba23 --- /dev/null +++ b/.github/workflows/validate-lint.yml @@ -0,0 +1,55 @@ +name: Validate lint + +on: + pull_request: + branches: + - '*' + +env: + GO_VERSION: 1.17 + GOLANGCI_LINT_VERSION: v1.45.0 + +jobs: + + lint: + runs-on: ubuntu-20.04 + + defaults: + run: + working-directory: ${{ github.workspace }}/go/src/github.com/cooperspencer/gickup + + steps: + - name: Set up Go ${{ env.GO_VERSION }} + uses: actions/setup-go@v2 + with: + go-version: ${{ env.GO_VERSION }} + + - name: Check out code + uses: actions/checkout@v2 + with: + path: go/src/github.com/cooperspencer/gickup + fetch-depth: 0 + + - name: Cache Go modules + uses: actions/cache@v2 + with: + path: | + ~/go/pkg/mod + ~/.cache/go-build + key: ${{ runner.os }}-lint-go-${{ hashFiles('**/go.sum') }} + restore-keys: ${{ runner.os }}-lint-go + + - name: Setup GOPATH + run: go env -w GOPATH=${{ github.workspace }}/go + + - name: go mod tidy + run: | + go mod tidy + git diff --exit-code + + - name: golangci-lint + uses: golangci/golangci-lint-action@v2 + with: + version: ${{ env.GOLANGCI_LINT_VERSION }} + working-directory: ${{ github.workspace }}/go/src/github.com/cooperspencer/gickup + diff --git a/.gitignore b/.gitignore index e567cb2..f6006c4 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,5 @@ bla.conf gickup.exe gickup conf.yml -.idea \ No newline at end of file +.idea +dist/ diff --git a/.golangci.yml b/.golangci.yml new file mode 100644 index 0000000..25e6ccf --- /dev/null +++ b/.golangci.yml @@ -0,0 +1,75 @@ +linters: + enable-all: true + disable: + - cyclop # duplicate of gocyclo + - exhaustivestruct # not revelent + - gochecknoglobals + - gocognit # too strict + - goerr113 # too strict + - golint # deprecated + - gomnd # too strict + - interfacer # deprecated + - ireturn # not revelent + - maligned # deprecated + - scopelint # deprecated + - tagliatelle # breaking + - testpackage # too strict + - varnamelen # not revelent + - wrapcheck # too strict + - wsl # too strict + - maintidx # kind of duplicate of gocyclo + +issues: + exclude-rules: + - path: main.go + text: bad syntax for struct tag pair + - path: main.go # FIXME + text: Function 'main' is too long + - path: (.+)_test.go + linters: + - typecheck + - path: metrics/prometheus/prometheus.go # FIXME + text: counter metrics should have "_total" suffix + - path: local/local.go # FIXME + text: has complex nested blocks + - path: local/local.go # FIXME + text: Function 'Locally' has too many statements + - path: gitlab/gitlab.go # FIXME + text: Function 'Backup' is too long + - path: gitlab/gitlab.go # FIXME + text: Function 'Get' has too many statements + - path: gitlab/gitlab.go # FIXME + text: cyclomatic complexity \d+ of func `Get` is high + - path: gitlab/gitlab.go # FIXME + text: has complex nested blocks + - path: gogs/gogs.go # FIXME + text: has complex nested blocks + - path: gogs/gogs.go # FIXME + text: Function 'Backup' is too long + - path: gogs/gogs.go # FIXME + text: Function 'Get' has too many statements + - path: bitbucket/bitbucket.go # FIXME + text: Function 'Get' is too long + - path: bitbucket/bitbucket.go # FIXME + text: type assertion must be checked + - path: gitea/gitea.go + text: .code.gitea.io/sdk/gitea.ListOrgReposOptions. composite literal uses unkeyed fields + - path: gitea/gitea.go # FIXME + text: has complex nested blocks + - path: gitea/gitea.go # FIXME + text: Function 'Backup' is too long + - path: gitea/gitea.go # FIXME + text: Function 'Get' has too many statements + - path: gitea/gitea.go # FIXME + text: cyclomatic complexity \d+ of func `Get` is high + - path: gitea/gitea.go # FIXME + text: lines are duplicate of `gitea/gitea.go:\d+-\d+` + - path: github/github.go # FIXME + text: Function 'Get' has too many statements + - path: github/github.go # FIXME + text: has complex nested blocks + +linters-settings: + forbidigo: + forbid: + - '^print(ln)?$' diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..7448331 --- /dev/null +++ b/Makefile @@ -0,0 +1,24 @@ +test: + go test ./... +.PHONY: test + +dist: + mkdir -p dist + +dist/gickup: dist + go build -o dist/gickup ./main.go + +build: dist/gickup +.PHONY: build + +clean: + $(RM) -r dist +.PHONY: clean + +install-tools: + go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.45.0 +.PHONY: install-tools + +lint: + golangci-lint run +.PHONY: lint diff --git a/bitbucket/bitbucket.go b/bitbucket/bitbucket.go index 31b5201..c06147a 100644 --- a/bitbucket/bitbucket.go +++ b/bitbucket/bitbucket.go @@ -1,13 +1,14 @@ package bitbucket import ( - "gickup/types" "net/url" + "github.com/cooperspencer/gickup/types" "github.com/ktrysmt/go-bitbucket" "github.com/rs/zerolog/log" ) +// Get TODO. func Get(conf *types.Conf) []types.Repo { repos := []types.Repo{} for _, repo := range conf.Source.BitBucket { @@ -15,20 +16,31 @@ func Get(conf *types.Conf) []types.Repo { if repo.User == "" { repo.User = repo.Username } - if repo.Url == "" { - repo.Url = bitbucket.DEFAULT_BITBUCKET_API_BASE_URL + + if repo.URL == "" { + repo.URL = bitbucket.DEFAULT_BITBUCKET_API_BASE_URL } else { - bitbucketUrl, err := url.Parse(repo.Url) + bitbucketURL, err := url.Parse(repo.URL) if err != nil { - log.Fatal().Str("stage", "bitbucket").Str("url", repo.Url).Msg(err.Error()) + log.Fatal(). + Str("stage", "bitbucket"). + Str("url", repo.URL). + Msg(err.Error()) } - client.SetApiBaseURL(*bitbucketUrl) + client.SetApiBaseURL(*bitbucketURL) } - log.Info().Str("stage", "bitbucket").Str("url", repo.Url).Msgf("grabbing repositories from %s", repo.User) + + log.Info(). + Str("stage", "bitbucket"). + Str("url", repo.URL). + Msgf("grabbing repositories from %s", repo.User) repositories, err := client.Repositories.ListForAccount(&bitbucket.RepositoriesOptions{Owner: repo.User}) if err != nil { - log.Fatal().Str("stage", "bitbucket").Str("url", repo.Url).Msg(err.Error()) + log.Fatal(). + Str("stage", "bitbucket"). + Str("url", repo.URL). + Msg(err.Error()) } include := types.GetMap(repo.Include) @@ -41,17 +53,40 @@ func Get(conf *types.Conf) []types.Repo { user = r.Owner["nickname"].(string) } } + if include[r.Name] { - repos = append(repos, types.Repo{Name: r.Name, Url: r.Links["clone"].([]interface{})[0].(map[string]interface{})["href"].(string), SshUrl: r.Links["clone"].([]interface{})[1].(map[string]interface{})["href"].(string), Token: "", Defaultbranch: r.Mainbranch.Name, Origin: repo, Owner: user, Hoster: types.GetHost(repo.Url)}) + repos = append(repos, types.Repo{ + Name: r.Name, + URL: r.Links["clone"].([]interface{})[0].(map[string]interface{})["href"].(string), + SSHURL: r.Links["clone"].([]interface{})[1].(map[string]interface{})["href"].(string), + Token: "", + Defaultbranch: r.Mainbranch.Name, + Origin: repo, + Owner: user, + Hoster: types.GetHost(repo.URL), + }) + continue } + if exclude[r.Name] { continue } + if len(include) == 0 { - repos = append(repos, types.Repo{Name: r.Name, Url: r.Links["clone"].([]interface{})[0].(map[string]interface{})["href"].(string), SshUrl: r.Links["clone"].([]interface{})[1].(map[string]interface{})["href"].(string), Token: "", Defaultbranch: r.Mainbranch.Name, Origin: repo, Owner: user, Hoster: types.GetHost(repo.Url)}) + repos = append(repos, types.Repo{ + Name: r.Name, + URL: r.Links["clone"].([]interface{})[0].(map[string]interface{})["href"].(string), + SSHURL: r.Links["clone"].([]interface{})[1].(map[string]interface{})["href"].(string), + Token: "", + Defaultbranch: r.Mainbranch.Name, + Origin: repo, + Owner: user, + Hoster: types.GetHost(repo.URL), + }) } } } + return repos } diff --git a/gitea/gitea.go b/gitea/gitea.go index 2ced20b..76b30df 100644 --- a/gitea/gitea.go +++ b/gitea/gitea.go @@ -2,75 +2,135 @@ package gitea import ( "code.gitea.io/sdk/gitea" - "gickup/types" + "github.com/cooperspencer/gickup/types" "github.com/rs/zerolog/log" ) +// Backup TODO. func Backup(r types.Repo, d types.GenRepo, dry bool) { - if d.Url == "" { - d.Url = "https://gitea.com/" + if d.URL == "" { + d.URL = "https://gitea.com/" } - log.Info().Str("stage", "gitea").Str("url", d.Url).Msgf("mirroring %s to %s", types.Blue(r.Name), d.Url) - giteaclient, err := gitea.NewClient(d.Url) + + log.Info(). + Str("stage", "gitea"). + Str("url", d.URL). + Msgf("mirroring %s to %s", types.Blue(r.Name), d.URL) + + giteaclient, err := gitea.NewClient(d.URL) if err != nil { - log.Fatal().Str("stage", "gitea").Str("url", d.Url).Msg(err.Error()) + log.Fatal().Str("stage", "gitea").Str("url", d.URL).Msg(err.Error()) } + giteaclient.SetBasicAuth(d.GetToken(), "") + user, _, err := giteaclient.GetMyUserInfo() if err != nil { - log.Fatal().Str("stage", "gitea").Str("url", d.Url).Msg(err.Error()) + log.Fatal(). + Str("stage", "gitea"). + Str("url", d.URL). + Msg(err.Error()) } - if !dry { - repo, _, err := giteaclient.GetRepo(user.UserName, r.Name) - if err != nil { - opts := gitea.MigrateRepoOption{RepoName: r.Name, RepoOwner: user.UserName, Mirror: true, CloneAddr: r.Url, AuthToken: r.Token} - if r.Token == "" { - opts = gitea.MigrateRepoOption{RepoName: r.Name, RepoOwner: user.UserName, Mirror: true, CloneAddr: r.Url, AuthUsername: r.Origin.User, AuthPassword: r.Origin.Password} - } - _, _, err := giteaclient.MigrateRepo(opts) - if err != nil { - log.Fatal().Str("stage", "gitea").Str("url", d.Url).Msg(err.Error()) - } - log.Info().Str("stage", "gitea").Str("url", d.Url).Msgf("mirrored %s to %s", types.Blue(r.Name), d.Url) - } else { - if repo.Mirror { - log.Info().Str("stage", "gitea").Str("url", d.Url).Msgf("mirror of %s already exists, syncing instead", types.Blue(r.Name)) - _, err := giteaclient.MirrorSync(user.UserName, repo.Name) - if err != nil { - log.Fatal().Str("stage", "gitea").Str("url", d.Url).Msg(err.Error()) - } - log.Info().Str("stage", "gitea").Str("url", d.Url).Msgf("successfully synced %s.", types.Blue(r.Name)) + + if dry { + return + } + + repo, _, err := giteaclient.GetRepo(user.UserName, r.Name) + if err != nil { + opts := gitea.MigrateRepoOption{ + RepoName: r.Name, + RepoOwner: user.UserName, + Mirror: true, + CloneAddr: r.URL, + AuthToken: r.Token, + } + + if r.Token == "" { + opts = gitea.MigrateRepoOption{ + RepoName: r.Name, + RepoOwner: user.UserName, + Mirror: true, + CloneAddr: r.URL, + AuthUsername: r.Origin.User, + AuthPassword: r.Origin.Password, } } + + _, _, err := giteaclient.MigrateRepo(opts) + if err != nil { + log.Fatal(). + Str("stage", "gitea"). + Str("url", d.URL). + Msg(err.Error()) + } + + log.Info(). + Str("stage", "gitea"). + Str("url", d.URL). + Msgf("mirrored %s to %s", types.Blue(r.Name), d.URL) + + return + } + if repo.Mirror { + log.Info(). + Str("stage", "gitea"). + Str("url", d.URL). + Msgf("mirror of %s already exists, syncing instead", types.Blue(r.Name)) + + _, err := giteaclient.MirrorSync(user.UserName, repo.Name) + if err != nil { + log.Fatal(). + Str("stage", "gitea"). + Str("url", d.URL). + Msg(err.Error()) + } + + log.Info(). + Str("stage", "gitea"). + Str("url", d.URL). + Msgf("successfully synced %s.", types.Blue(r.Name)) } } +// Get TODO. func Get(conf *types.Conf) []types.Repo { repos := []types.Repo{} for _, repo := range conf.Source.Gitea { - if repo.Url == "" { - repo.Url = "https://gitea.com" + if repo.URL == "" { + repo.URL = "https://gitea.com" } - log.Info().Str("stage", "gitea").Str("url", repo.Url).Msgf("grabbing repositories from %s", repo.User) + log.Info(). + Str("stage", "gitea"). + Str("url", repo.URL). + Msgf("grabbing repositories from %s", repo.User) opt := gitea.ListReposOptions{} opt.PageSize = 50 opt.Page = 1 gitearepos := []*gitea.Repository{} - client := &gitea.Client{} + + var client *gitea.Client var err error token := repo.GetToken() if token != "" { - client, err = gitea.NewClient(repo.Url, gitea.SetToken(token)) + client, err = gitea.NewClient(repo.URL, gitea.SetToken(token)) } else { - client, err = gitea.NewClient(repo.Url) + client, err = gitea.NewClient(repo.URL) } + for { if err != nil { - log.Fatal().Str("stage", "gitea").Str("url", repo.Url).Msg(err.Error()) + log.Fatal(). + Str("stage", "gitea"). + Str("url", repo.URL). + Msg(err.Error()) } repos, _, err := client.ListUserRepos(repo.User, opt) if err != nil { - log.Fatal().Str("stage", "gitea").Str("url", repo.Url).Msg(err.Error()) + log.Fatal(). + Str("stage", "gitea"). + Str("url", repo.URL). + Msg(err.Error()) } if len(repos) == 0 { break @@ -82,7 +142,10 @@ func Get(conf *types.Conf) []types.Repo { if repo.Starred { starredrepos, _, err := client.GetMyStarredRepos() if err != nil { - log.Fatal().Str("stage", "gitea").Str("url", repo.Url).Msg(err.Error()) + log.Fatal(). + Str("stage", "gitea"). + Str("url", repo.URL). + Msg(err.Error()) } gitearepos = append(gitearepos, starredrepos...) } @@ -94,19 +157,58 @@ func Get(conf *types.Conf) []types.Repo { for _, r := range gitearepos { if include[r.Name] { - repos = append(repos, types.Repo{Name: r.Name, Url: r.CloneURL, SshUrl: r.SSHURL, Token: token, Defaultbranch: r.DefaultBranch, Origin: repo, Owner: r.Owner.UserName, Hoster: types.GetHost(repo.Url)}) + repos = append(repos, types.Repo{ + Name: r.Name, + URL: r.CloneURL, + SSHURL: r.SSHURL, + Token: token, + Defaultbranch: r.DefaultBranch, + Origin: repo, + Owner: r.Owner.UserName, + Hoster: types.GetHost(repo.URL), + }) if r.HasWiki && repo.Wiki && types.StatRemote(r.CloneURL, r.SSHURL, repo) { - repos = append(repos, types.Repo{Name: r.Name + ".wiki", Url: types.DotGitRx.ReplaceAllString(r.CloneURL, ".wiki.git"), SshUrl: types.DotGitRx.ReplaceAllString(r.SSHURL, ".wiki.git"), Token: token, Defaultbranch: r.DefaultBranch, Origin: repo, Owner: r.Owner.UserName, Hoster: types.GetHost(repo.Url)}) + repos = append(repos, types.Repo{ + Name: r.Name + ".wiki", + URL: types.DotGitRx.ReplaceAllString(r.CloneURL, ".wiki.git"), + SSHURL: types.DotGitRx.ReplaceAllString(r.SSHURL, ".wiki.git"), + Token: token, + Defaultbranch: r.DefaultBranch, + Origin: repo, + Owner: r.Owner.UserName, + Hoster: types.GetHost(repo.URL), + }) } + continue } + if exclude[r.Name] { continue } + if len(repo.Include) == 0 { - repos = append(repos, types.Repo{Name: r.Name, Url: r.CloneURL, SshUrl: r.SSHURL, Token: token, Defaultbranch: r.DefaultBranch, Origin: repo, Owner: r.Owner.UserName, Hoster: types.GetHost(repo.Url)}) + repos = append(repos, types.Repo{ + Name: r.Name, + URL: r.CloneURL, + SSHURL: r.SSHURL, + Token: token, + Defaultbranch: r.DefaultBranch, + Origin: repo, + Owner: r.Owner.UserName, + Hoster: types.GetHost(repo.URL), + }) if r.HasWiki && repo.Wiki && types.StatRemote(r.CloneURL, r.SSHURL, repo) { - repos = append(repos, types.Repo{Name: r.Name + ".wiki", Url: types.DotGitRx.ReplaceAllString(r.CloneURL, ".wiki.git"), SshUrl: types.DotGitRx.ReplaceAllString(r.SSHURL, ".wiki.git"), Token: token, Defaultbranch: r.DefaultBranch, Origin: repo, Owner: r.Owner.UserName, Hoster: types.GetHost(repo.Url)}) + repos = append(repos, types.Repo{ + Name: r.Name + ".wiki", + URL: types.DotGitRx.ReplaceAllString(r.CloneURL, ".wiki.git"), + SSHURL: types.DotGitRx.ReplaceAllString(r.SSHURL, ".wiki.git"), + Token: token, + Defaultbranch: r.DefaultBranch, + Origin: repo, + Owner: r.Owner.UserName, + Hoster: types.GetHost(repo.URL), + }) } } } @@ -115,7 +217,10 @@ func Get(conf *types.Conf) []types.Repo { for { o, _, err := client.ListUserOrgs(repo.User, gitea.ListOrgsOptions{ListOptions: orgopt}) if err != nil { - log.Fatal().Str("stage", "gitea").Str("url", repo.Url).Msg(err.Error()) + log.Fatal(). + Str("stage", "gitea"). + Str("url", repo.URL). + Msg(err.Error()) } if len(o) == 0 { break @@ -151,30 +256,74 @@ func Get(conf *types.Conf) []types.Repo { } for _, r := range orgrepos { if include[r.Name] { - repos = append(repos, types.Repo{Name: r.Name, Url: r.CloneURL, SshUrl: r.SSHURL, Token: token, Defaultbranch: r.DefaultBranch, Origin: repo, Owner: r.Owner.UserName, Hoster: types.GetHost(repo.Url)}) + repos = append(repos, types.Repo{ + Name: r.Name, + URL: r.CloneURL, + SSHURL: r.SSHURL, + Token: token, + Defaultbranch: r.DefaultBranch, + Origin: repo, + Owner: r.Owner.UserName, + Hoster: types.GetHost(repo.URL), + }) if r.HasWiki && repo.Wiki && types.StatRemote(r.CloneURL, r.SSHURL, repo) { - repos = append(repos, types.Repo{Name: r.Name + ".wiki", Url: types.DotGitRx.ReplaceAllString(r.CloneURL, ".wiki.git"), SshUrl: types.DotGitRx.ReplaceAllString(r.SSHURL, ".wiki.git"), Token: token, Defaultbranch: r.DefaultBranch, Origin: repo, Owner: r.Owner.UserName, Hoster: types.GetHost(repo.Url)}) + repos = append(repos, types.Repo{ + Name: r.Name + ".wiki", + URL: types.DotGitRx.ReplaceAllString(r.CloneURL, ".wiki.git"), + SSHURL: types.DotGitRx.ReplaceAllString(r.SSHURL, ".wiki.git"), + Token: token, + Defaultbranch: r.DefaultBranch, + Origin: repo, + Owner: r.Owner.UserName, + Hoster: types.GetHost(repo.URL), + }) } + continue } + if exclude[r.Name] { continue } + if len(repo.Include) == 0 { - repos = append(repos, types.Repo{Name: r.Name, Url: r.CloneURL, SshUrl: r.SSHURL, Token: token, Defaultbranch: r.DefaultBranch, Origin: repo, Owner: r.Owner.UserName, Hoster: types.GetHost(repo.Url)}) + repos = append(repos, types.Repo{ + Name: r.Name, + URL: r.CloneURL, + SSHURL: r.SSHURL, + Token: token, + Defaultbranch: r.DefaultBranch, + Origin: repo, + Owner: r.Owner.UserName, + Hoster: types.GetHost(repo.URL), + }) if r.HasWiki && repo.Wiki && types.StatRemote(r.CloneURL, r.SSHURL, repo) { - repos = append(repos, types.Repo{Name: r.Name + ".wiki", Url: types.DotGitRx.ReplaceAllString(r.CloneURL, ".wiki.git"), SshUrl: types.DotGitRx.ReplaceAllString(r.SSHURL, ".wiki.git"), Token: token, Defaultbranch: r.DefaultBranch, Origin: repo, Owner: r.Owner.UserName, Hoster: types.GetHost(repo.Url)}) + repos = append(repos, types.Repo{ + Name: r.Name + ".wiki", + URL: types.DotGitRx.ReplaceAllString(r.CloneURL, ".wiki.git"), + SSHURL: types.DotGitRx.ReplaceAllString(r.SSHURL, ".wiki.git"), + Token: token, + Defaultbranch: r.DefaultBranch, + Origin: repo, + Owner: r.Owner.UserName, + Hoster: types.GetHost(repo.URL), + }) } } } } + return repos } -func getOrgRepos(client *gitea.Client, org *gitea.Organization, orgopt gitea.ListOptions, repo types.GenRepo) []*gitea.Repository { - o, _, err := client.ListOrgRepos(org.UserName, gitea.ListOrgReposOptions{orgopt}) +func getOrgRepos(client *gitea.Client, org *gitea.Organization, + orgopt gitea.ListOptions, repo types.GenRepo, +) []*gitea.Repository { + o, _, err := client.ListOrgRepos(org.UserName, + gitea.ListOrgReposOptions{orgopt}) if err != nil { - log.Fatal().Str("stage", "gitea").Str("url", repo.Url).Msg(err.Error()) + log.Fatal().Str("stage", "gitea").Str("url", repo.URL).Msg(err.Error()) } + return o } diff --git a/github/github.go b/github/github.go index c36520f..46bc84c 100644 --- a/github/github.go +++ b/github/github.go @@ -2,28 +2,51 @@ package github import ( "context" - "gickup/types" + + "github.com/cooperspencer/gickup/types" "github.com/google/go-github/v41/github" "github.com/rs/zerolog/log" "golang.org/x/oauth2" ) func addWiki(r github.Repository, repo types.GenRepo, token string) types.Repo { - if r.GetHasWiki() && repo.Wiki && types.StatRemote(r.GetCloneURL(), r.GetSSHURL(), repo) { - return types.Repo{Name: *r.Name + ".wiki", Url: types.DotGitRx.ReplaceAllString(r.GetCloneURL(), ".wiki.git"), SshUrl: types.DotGitRx.ReplaceAllString(r.GetSSHURL(), ".wiki.git"), Token: token, Defaultbranch: r.GetDefaultBranch(), Origin: repo, Owner: r.GetOwner().GetLogin(), Hoster: "github.com"} + if !(r.GetHasWiki() && repo.Wiki && + types.StatRemote(r.GetCloneURL(), r.GetSSHURL(), repo)) { + return types.Repo{} + } + + return types.Repo{ + Name: *r.Name + ".wiki", + URL: types.DotGitRx.ReplaceAllString(r.GetCloneURL(), ".wiki.git"), + SSHURL: types.DotGitRx.ReplaceAllString(r.GetSSHURL(), ".wiki.git"), + Token: token, + Defaultbranch: r.GetDefaultBranch(), + Origin: repo, + Owner: r.GetOwner().GetLogin(), + Hoster: "github.com", } - return types.Repo{} } +// Get TODO. func Get(conf *types.Conf) []types.Repo { repos := []types.Repo{} for _, repo := range conf.Source.Github { - log.Info().Str("stage", "github").Str("url", "https://github.com").Msgf("grabbing the repositories from %s", repo.User) - client := &github.Client{} - opt := &github.RepositoryListOptions{ListOptions: github.ListOptions{PerPage: 50}} + log.Info(). + Str("stage", "github"). + Str("url", "https://github.com"). + Msgf("grabbing the repositories from %s", repo.User) + + opt := &github.RepositoryListOptions{ + ListOptions: github.ListOptions{ + PerPage: 50, + }, + } + i := 1 githubrepos := []*github.Repository{} token := repo.GetToken() + + var client *github.Client if token == "" { client = github.NewClient(nil) } else { @@ -31,23 +54,34 @@ func Get(conf *types.Conf) []types.Repo { &oauth2.Token{AccessToken: token}, ) tc := oauth2.NewClient(context.TODO(), ts) + client = github.NewClient(tc) } + if token != "" { user, _, err := client.Users.Get(context.TODO(), "") if err != nil { - log.Fatal().Str("stage", "github").Str("url", "https://github.com").Msg(err.Error()) + log.Fatal(). + Str("stage", "github"). + Str("url", "https://github.com"). + Msg(err.Error()) } + if repo.User == user.GetLogin() { repo.User = "" } } + for { opt.Page = i repos, _, err := client.Repositories.List(context.TODO(), repo.User, opt) if err != nil { - log.Fatal().Str("stage", "github").Str("url", "https://github.com").Msg(err.Error()) + log.Fatal(). + Str("stage", "github"). + Str("url", "https://github.com"). + Msg(err.Error()) } + if len(repos) == 0 { break } @@ -57,12 +91,20 @@ func Get(conf *types.Conf) []types.Repo { if repo.Starred { i = 1 - opt := &github.ActivityListStarredOptions{ListOptions: github.ListOptions{PerPage: 50}} + opt := &github.ActivityListStarredOptions{ + ListOptions: github.ListOptions{ + PerPage: 50, + }, + } + for { opt.ListOptions.Page = i repos, _, err := client.Activity.ListStarred(context.TODO(), repo.User, opt) if err != nil { - log.Fatal().Str("stage", "github").Str("url", "https://github.com").Msg(err.Error()) + log.Fatal(). + Str("stage", "github"). + Str("url", "https://github.com"). + Msg(err.Error()) } if len(repos) == 0 { break @@ -81,30 +123,61 @@ func Get(conf *types.Conf) []types.Repo { for _, r := range githubrepos { if include[*r.Name] { - repos = append(repos, types.Repo{Name: r.GetName(), Url: r.GetCloneURL(), SshUrl: r.GetSSHURL(), Token: token, Defaultbranch: r.GetDefaultBranch(), Origin: repo, Owner: r.GetOwner().GetLogin(), Hoster: "github.com"}) + repos = append(repos, types.Repo{ + Name: r.GetName(), + URL: r.GetCloneURL(), + SSHURL: r.GetSSHURL(), + Token: token, + Defaultbranch: r.GetDefaultBranch(), + Origin: repo, + Owner: r.GetOwner().GetLogin(), + Hoster: "github.com", + }) wiki := addWiki(*r, repo, token) if wiki.Name != "" { repos = append(repos, wiki) } + continue } + if exclude[*r.Name] { continue } + if excludeorgs[r.GetOwner().GetLogin()] { continue } + if len(repo.Include) == 0 { if len(includeorgs) > 0 { if includeorgs[r.GetOwner().GetLogin()] { - repos = append(repos, types.Repo{Name: r.GetName(), Url: r.GetCloneURL(), SshUrl: r.GetSSHURL(), Token: token, Defaultbranch: r.GetDefaultBranch(), Origin: repo, Owner: r.GetOwner().GetLogin(), Hoster: "github.com"}) + repos = append(repos, types.Repo{ + Name: r.GetName(), + URL: r.GetCloneURL(), + SSHURL: r.GetSSHURL(), + Token: token, + Defaultbranch: r.GetDefaultBranch(), + Origin: repo, + Owner: r.GetOwner().GetLogin(), + Hoster: "github.com", + }) wiki := addWiki(*r, repo, token) if wiki.Name != "" { repos = append(repos, wiki) } } } else { - repos = append(repos, types.Repo{Name: r.GetName(), Url: r.GetCloneURL(), SshUrl: r.GetSSHURL(), Token: token, Defaultbranch: r.GetDefaultBranch(), Origin: repo, Owner: r.GetOwner().GetLogin(), Hoster: "github.com"}) + repos = append(repos, types.Repo{ + Name: r.GetName(), + URL: r.GetCloneURL(), + SSHURL: r.GetSSHURL(), + Token: token, + Defaultbranch: r.GetDefaultBranch(), + Origin: repo, + Owner: r.GetOwner().GetLogin(), + Hoster: "github.com", + }) wiki := addWiki(*r, repo, token) if wiki.Name != "" { repos = append(repos, wiki) @@ -113,5 +186,6 @@ func Get(conf *types.Conf) []types.Repo { } } } + return repos } diff --git a/gitlab/gitlab.go b/gitlab/gitlab.go index bc70427..55a3b4e 100644 --- a/gitlab/gitlab.go +++ b/gitlab/gitlab.go @@ -2,34 +2,48 @@ package gitlab import ( "fmt" - "gickup/types" "path" "strings" + "github.com/cooperspencer/gickup/types" "github.com/rs/zerolog/log" "github.com/xanzy/go-gitlab" ) +// Backup TODO. func Backup(r types.Repo, d types.GenRepo, dry bool) { - gitlabclient := &gitlab.Client{} + var gitlabclient *gitlab.Client token := d.GetToken() var err error - if d.Url == "" { - d.Url = "https://gitlab.com" + if d.URL == "" { + d.URL = "https://gitlab.com" gitlabclient, err = gitlab.NewClient(token) } else { - gitlabclient, err = gitlab.NewClient(token, gitlab.WithBaseURL(d.Url)) - } - log.Info().Str("stage", "gitlab").Str("url", d.Url).Msgf("mirroring %s to %s", types.Blue(r.Name), d.Url) - if err != nil { - log.Fatal().Str("stage", "gitlab").Str("url", d.Url).Msg(err.Error()) + gitlabclient, err = gitlab.NewClient(token, gitlab.WithBaseURL(d.URL)) } + if err != nil { + log.Fatal(). + Str("stage", "gitlab"). + Str("url", d.URL). + Msg(err.Error()) + } + + log.Info(). + Str("stage", "gitlab"). + Str("url", d.URL). + Msgf("mirroring %s to %s", types.Blue(r.Name), d.URL) + True := true - opt := gitlab.ListProjectsOptions{Search: &r.Name, Owned: &True} + + opt := gitlab.ListProjectsOptions{ + Search: &r.Name, + Owned: &True, + } + projects, _, err := gitlabclient.Projects.ListProjects(&opt) if err != nil { - log.Fatal().Str("stage", "gitlab").Str("url", d.Url).Msg(err.Error()) + log.Fatal().Str("stage", "gitlab").Str("url", d.URL).Msg(err.Error()) } found := false @@ -39,42 +53,62 @@ func Backup(r types.Repo, d types.GenRepo, dry bool) { } } - if !dry { - if !found { - if r.Token != "" { - splittedurl := strings.Split(r.Url, "//") - r.Url = fmt.Sprintf("%s//%s@%s", splittedurl[0], r.Token, splittedurl[1]) - if r.Token == "" { - r.Url = fmt.Sprintf("%s//%s:%s@%s", splittedurl[0], r.Origin.User, r.Origin.Password, splittedurl[1]) - } - } - opts := &gitlab.CreateProjectOptions{Mirror: &True, ImportURL: &r.Url, Name: &r.Name} - _, _, err := gitlabclient.Projects.CreateProject(opts) - if err != nil { - log.Fatal().Str("stage", "gitlab").Str("url", d.Url).Msg(err.Error()) - } - } + if dry || found { + return + } + + if r.Token != "" { + splittedurl := strings.Split(r.URL, "//") + + r.URL = fmt.Sprintf("%s//%s:%s@%s", + splittedurl[0], r.Origin.User, r.Origin.Password, splittedurl[1]) + } + + opts := &gitlab.CreateProjectOptions{ + Mirror: &True, + ImportURL: &r.URL, + Name: &r.Name, + } + + _, _, err = gitlabclient.Projects.CreateProject(opts) + if err != nil { + log.Fatal(). + Str("stage", "gitlab"). + Str("url", d.URL). + Msg(err.Error()) } } +// Get TODO. func Get(conf *types.Conf) []types.Repo { repos := []types.Repo{} for _, repo := range conf.Source.Gitlab { - if repo.Url == "" { - repo.Url = "https://gitlab.com" + if repo.URL == "" { + repo.URL = "https://gitlab.com" } - log.Info().Str("stage", "gitlab").Str("url", repo.Url).Msgf("grabbing repositories from %s", repo.User) + + log.Info(). + Str("stage", "gitlab"). + Str("url", repo.URL). + Msgf("grabbing repositories from %s", repo.User) gitlabrepos := []*gitlab.Project{} gitlabgrouprepos := map[string][]*gitlab.Project{} token := repo.GetToken() - client, err := gitlab.NewClient(token, gitlab.WithBaseURL(repo.Url)) + client, err := gitlab.NewClient(token, gitlab.WithBaseURL(repo.URL)) if err != nil { - log.Fatal().Str("stage", "gitlab").Str("url", repo.Url).Msg(err.Error()) + log.Fatal(). + Str("stage", "gitlab"). + Str("url", repo.URL). + Msg(err.Error()) } + opt := &gitlab.ListProjectsOptions{} users, _, err := client.Users.ListUsers(&gitlab.ListUsersOptions{Username: &repo.User}) if err != nil { - log.Fatal().Str("stage", "gitlab").Str("url", repo.Url).Msg(err.Error()) + log.Fatal(). + Str("stage", "gitlab"). + Str("url", repo.URL). + Msg(err.Error()) } opt.PerPage = 50 @@ -85,7 +119,10 @@ func Get(conf *types.Conf) []types.Repo { opt.Page = i projects, _, err := client.Projects.ListUserProjects(user.ID, opt) if err != nil { - log.Fatal().Str("stage", "gitlab").Str("url", repo.Url).Msg(err.Error()) + log.Fatal(). + Str("stage", "gitlab"). + Str("url", repo.URL). + Msg(err.Error()) } if len(projects) == 0 { break @@ -104,7 +141,10 @@ func Get(conf *types.Conf) []types.Repo { opt.Page = i projects, _, err := client.Projects.ListUserStarredProjects(user.ID, opt) if err != nil { - log.Fatal().Str("stage", "gitlab").Str("url", repo.Url).Msg(err.Error()) + log.Fatal(). + Str("stage", "gitlab"). + Str("url", repo.URL). + Msg(err.Error()) } if len(projects) == 0 { break @@ -124,14 +164,32 @@ func Get(conf *types.Conf) []types.Repo { for _, r := range gitlabrepos { if include[r.Name] { if r.RepositoryAccessLevel != gitlab.DisabledAccessControl { - repos = append(repos, types.Repo{Name: r.Path, Url: r.HTTPURLToRepo, SshUrl: r.SSHURLToRepo, Token: token, Defaultbranch: r.DefaultBranch, Origin: repo, Owner: r.Namespace.FullPath, Hoster: types.GetHost(repo.Url)}) + repos = append(repos, types.Repo{ + Name: r.Path, + URL: r.HTTPURLToRepo, + SSHURL: r.SSHURLToRepo, + Token: token, + Defaultbranch: r.DefaultBranch, + Origin: repo, + Owner: r.Namespace.FullPath, + Hoster: types.GetHost(repo.URL), + }) } if r.WikiEnabled && repo.Wiki { if activeWiki(r, client, repo) { - httpUrlToRepo := types.DotGitRx.ReplaceAllString(r.HTTPURLToRepo, ".wiki.git") - sshUrlToRepo := types.DotGitRx.ReplaceAllString(r.SSHURLToRepo, ".wiki.git") - repos = append(repos, types.Repo{Name: r.Path + ".wiki", Url: httpUrlToRepo, SshUrl: sshUrlToRepo, Token: token, Defaultbranch: r.DefaultBranch, Origin: repo, Owner: r.Namespace.FullPath, Hoster: types.GetHost(repo.Url)}) + httpURLToRepo := types.DotGitRx.ReplaceAllString(r.HTTPURLToRepo, ".wiki.git") + sshURLToRepo := types.DotGitRx.ReplaceAllString(r.SSHURLToRepo, ".wiki.git") + repos = append(repos, types.Repo{ + Name: r.Path + ".wiki", + URL: httpURLToRepo, + SSHURL: sshURLToRepo, + Token: token, + Defaultbranch: r.DefaultBranch, + Origin: repo, + Owner: r.Namespace.FullPath, + Hoster: types.GetHost(repo.URL), + }) } } @@ -142,29 +200,57 @@ func Get(conf *types.Conf) []types.Repo { } if len(include) == 0 { if r.RepositoryAccessLevel != gitlab.DisabledAccessControl { - repos = append(repos, types.Repo{Name: r.Path, Url: r.HTTPURLToRepo, SshUrl: r.SSHURLToRepo, Token: token, Defaultbranch: r.DefaultBranch, Origin: repo, Owner: r.Namespace.FullPath, Hoster: types.GetHost(repo.Url)}) + repos = append(repos, types.Repo{ + Name: r.Path, + URL: r.HTTPURLToRepo, + SSHURL: r.SSHURLToRepo, + Token: token, + Defaultbranch: r.DefaultBranch, + Origin: repo, + Owner: r.Namespace.FullPath, + Hoster: types.GetHost(repo.URL), + }) } if r.WikiEnabled && repo.Wiki { if activeWiki(r, client, repo) { - httpUrlToRepo := types.DotGitRx.ReplaceAllString(r.HTTPURLToRepo, ".wiki.git") - sshUrlToRepo := types.DotGitRx.ReplaceAllString(r.SSHURLToRepo, ".wiki.git") - repos = append(repos, types.Repo{Name: r.Path + ".wiki", Url: httpUrlToRepo, SshUrl: sshUrlToRepo, Token: token, Defaultbranch: r.DefaultBranch, Origin: repo, Owner: r.Namespace.FullPath, Hoster: types.GetHost(repo.Url)}) + httpURLToRepo := types.DotGitRx.ReplaceAllString(r.HTTPURLToRepo, ".wiki.git") + sshURLToRepo := types.DotGitRx.ReplaceAllString(r.SSHURLToRepo, ".wiki.git") + repos = append(repos, types.Repo{ + Name: r.Path + ".wiki", + URL: httpURLToRepo, + SSHURL: sshURLToRepo, + Token: token, + Defaultbranch: r.DefaultBranch, + Origin: repo, + Owner: r.Namespace.FullPath, + Hoster: types.GetHost(repo.URL), + }) } } } } + if token != "" { groups := []*gitlab.Group{} i := 1 for { - g, _, err := client.Groups.ListGroups(&gitlab.ListGroupsOptions{ListOptions: gitlab.ListOptions{Page: i, PerPage: 50}}) + g, _, err := client.Groups.ListGroups(&gitlab.ListGroupsOptions{ + ListOptions: gitlab.ListOptions{ + Page: i, + PerPage: 50, + }, + }) if err != nil { - log.Fatal().Str("stage", "gitlab").Str("url", repo.Url).Msg(err.Error()) + log.Fatal(). + Str("stage", "gitlab"). + Str("url", repo.URL).Msg(err.Error()) } + if len(g) == 0 { break } + groups = append(groups, g...) i++ } @@ -177,7 +263,10 @@ func Get(conf *types.Conf) []types.Repo { for { projects, _, err := client.Groups.ListGroupProjects(group.ID, gopt) if err != nil { - log.Fatal().Str("stage", "gitlab").Str("url", repo.Url).Msg(err.Error()) + log.Fatal(). + Str("stage", "gitlab"). + Str("url", repo.URL). + Msg(err.Error()) } if len(projects) == 0 { break @@ -197,16 +286,35 @@ func Get(conf *types.Conf) []types.Repo { for _, r := range gr { if include[r.Name] { if r.RepositoryAccessLevel != gitlab.DisabledAccessControl { - repos = append(repos, types.Repo{Name: r.Path, Url: r.HTTPURLToRepo, SshUrl: r.SSHURLToRepo, Token: token, Defaultbranch: r.DefaultBranch, Origin: repo, Owner: k, Hoster: types.GetHost(repo.Url)}) + repos = append(repos, types.Repo{ + Name: r.Path, + URL: r.HTTPURLToRepo, + SSHURL: r.SSHURLToRepo, + Token: token, + Defaultbranch: r.DefaultBranch, + Origin: repo, + Owner: k, + Hoster: types.GetHost(repo.URL), + }) } if r.WikiEnabled && repo.Wiki { if activeWiki(r, client, repo) { - httpUrlToRepo := types.DotGitRx.ReplaceAllString(r.HTTPURLToRepo, ".wiki.git") - sshUrlToRepo := types.DotGitRx.ReplaceAllString(r.SSHURLToRepo, ".wiki.git") - repos = append(repos, types.Repo{Name: r.Path + ".wiki", Url: httpUrlToRepo, SshUrl: sshUrlToRepo, Token: token, Defaultbranch: r.DefaultBranch, Origin: repo, Owner: k, Hoster: types.GetHost(repo.Url)}) + httpURLToRepo := types.DotGitRx.ReplaceAllString(r.HTTPURLToRepo, ".wiki.git") + sshURLToRepo := types.DotGitRx.ReplaceAllString(r.SSHURLToRepo, ".wiki.git") + repos = append(repos, types.Repo{ + Name: r.Path + ".wiki", + URL: httpURLToRepo, + SSHURL: sshURLToRepo, + Token: token, + Defaultbranch: r.DefaultBranch, + Origin: repo, + Owner: k, + Hoster: types.GetHost(repo.URL), + }) } } + continue } if exclude[r.Name] { @@ -218,33 +326,55 @@ func Get(conf *types.Conf) []types.Repo { if len(include) == 0 { if len(includeorgs) == 0 || includeorgs[r.Namespace.FullPath] { if r.RepositoryAccessLevel != gitlab.DisabledAccessControl { - repos = append(repos, types.Repo{Name: r.Path, Url: r.HTTPURLToRepo, SshUrl: r.SSHURLToRepo, Token: token, Defaultbranch: r.DefaultBranch, Origin: repo, Owner: k, Hoster: types.GetHost(repo.Url)}) + repos = append(repos, types.Repo{ + Name: r.Path, + URL: r.HTTPURLToRepo, + SSHURL: r.SSHURLToRepo, + Token: token, + Defaultbranch: r.DefaultBranch, + Origin: repo, + Owner: k, + Hoster: types.GetHost(repo.URL), + }) } if r.WikiEnabled && repo.Wiki { if activeWiki(r, client, repo) { - httpUrlToRepo := types.DotGitRx.ReplaceAllString(r.HTTPURLToRepo, ".wiki.git") - sshUrlToRepo := types.DotGitRx.ReplaceAllString(r.SSHURLToRepo, ".wiki.git") - repos = append(repos, types.Repo{Name: r.Path + ".wiki", Url: httpUrlToRepo, SshUrl: sshUrlToRepo, Token: token, Defaultbranch: r.DefaultBranch, Origin: repo, Owner: k, Hoster: types.GetHost(repo.Url)}) + httpURLToRepo := types.DotGitRx.ReplaceAllString(r.HTTPURLToRepo, ".wiki.git") + sshURLToRepo := types.DotGitRx.ReplaceAllString(r.SSHURLToRepo, ".wiki.git") + repos = append(repos, types.Repo{ + Name: r.Path + ".wiki", + URL: httpURLToRepo, + SSHURL: sshURLToRepo, + Token: token, Defaultbranch: r.DefaultBranch, + Origin: repo, + Owner: k, + Hoster: types.GetHost(repo.URL), + }) } } - } + } } } } } } + return repos } func activeWiki(r *gitlab.Project, client *gitlab.Client, repo types.GenRepo) bool { - wikilistoptions := &gitlab.ListWikisOptions{WithContent: gitlab.Bool(true)} + wikilistoptions := &gitlab.ListWikisOptions{ + WithContent: gitlab.Bool(true), + } + wikis, _, err := client.Wikis.ListWikis(r.ID, wikilistoptions) if err != nil { - log.Warn().Str("stage", "gitlab").Str("url", repo.Url).Msg(err.Error()) + log.Warn(). + Str("stage", "gitlab"). + Str("url", repo.URL). + Msg(err.Error()) } - if len(wikis) > 0 { - return true - } - return false + + return len(wikis) > 0 } diff --git a/go.mod b/go.mod index d886efc..8cb362d 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module gickup +module github.com/cooperspencer/gickup go 1.16 @@ -9,7 +9,6 @@ require ( github.com/alecthomas/kong v0.4.1 github.com/go-git/go-git/v5 v5.4.2 github.com/gogs/go-gogs-client v0.0.0-20210131175652-1d7215cd8d85 - github.com/golang/protobuf v1.5.2 // indirect github.com/google/go-github/v41 v41.0.0 github.com/gookit/color v1.5.0 github.com/hashicorp/go-cleanhttp v0.5.2 // indirect diff --git a/go.sum b/go.sum index efe25c4..f6bf3a5 100644 --- a/go.sum +++ b/go.sum @@ -31,8 +31,6 @@ cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohl cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= code.gitea.io/gitea-vet v0.2.1/go.mod h1:zcNbT/aJEmivCAhfmkHOlT645KNOf9W2KnkLgFjGGfE= -code.gitea.io/sdk/gitea v0.15.0 h1:tsNhxDM/2N1Ohv1Xq5UWrht/esg0WmtRj4wsHVHriTg= -code.gitea.io/sdk/gitea v0.15.0/go.mod h1:klY2LVI3s3NChzIk/MzMn7G1FHrfU7qd63iSMVoHRBA= code.gitea.io/sdk/gitea v0.15.1 h1:WJreC7YYuxbn0UDaPuWIe/mtiNKTvLN8MLkaw71yx/M= code.gitea.io/sdk/gitea v0.15.1/go.mod h1:klY2LVI3s3NChzIk/MzMn7G1FHrfU7qd63iSMVoHRBA= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= @@ -42,21 +40,16 @@ github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA= github.com/Microsoft/go-winio v0.4.16/go.mod h1:XB6nPKklQyQ7GC9LdcBEcBl8PF76WugXOPRXwdLnMv0= github.com/Microsoft/go-winio v0.5.0/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= -github.com/Microsoft/go-winio v0.5.1 h1:aPJp2QD7OOrhO5tQXqQoGSJc+DjDtWTGLOmNyAm6FgY= -github.com/Microsoft/go-winio v0.5.1/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= github.com/Microsoft/go-winio v0.5.2 h1:a9IhgEQBCUEk6QCdml9CiJGhAws+YwffDHEMp1VMrpA= github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= github.com/ProtonMail/go-crypto v0.0.0-20210428141323-04723f9f07d7/go.mod h1:z4/9nQmJSSwwds7ejkxaJwO37dru3geImFUdJlaLzQo= -github.com/ProtonMail/go-crypto v0.0.0-20211112122917-428f8eabeeb3 h1:XcF0cTDJeiuZ5NU8w7WUDge0HRwwNRmxj/GGk6KSA6g= -github.com/ProtonMail/go-crypto v0.0.0-20211112122917-428f8eabeeb3/go.mod h1:z4/9nQmJSSwwds7ejkxaJwO37dru3geImFUdJlaLzQo= github.com/ProtonMail/go-crypto v0.0.0-20220113124808-70ae35bab23f h1:J2FzIrXN82q5uyUraeJpLIm7U6PffRwje2ORho5yIik= github.com/ProtonMail/go-crypto v0.0.0-20220113124808-70ae35bab23f/go.mod h1:z4/9nQmJSSwwds7ejkxaJwO37dru3geImFUdJlaLzQo= github.com/acomagu/bufpipe v1.0.3 h1:fxAGrHZTgQ9w5QqVItgzwj235/uYZYgbXitB+dLupOk= github.com/acomagu/bufpipe v1.0.3/go.mod h1:mxdxdup/WdsKVreO5GpW4+M/1CE2sMG4jeGJ2sYmHc4= -github.com/alecthomas/kong v0.2.19 h1:qBDfByO5XgWUXyNB4D6OOhGh5Z1eNOwWayDPQJFNWdc= -github.com/alecthomas/kong v0.2.19/go.mod h1:ka3VZ8GZNPXv9Ov+j4YNLkI8mTuhXyr/0ktSlqIydQQ= github.com/alecthomas/kong v0.4.1 h1:0sFnMts+ijOiFuSHsMB9MlDi3NGINBkx9KIw1/gcuDw= github.com/alecthomas/kong v0.4.1/go.mod h1:uzxf/HUh0tj43x1AyJROl3JT7SgsZ5m+icOv1csRhc0= +github.com/alecthomas/repr v0.0.0-20210801044451-80ca428c5142 h1:8Uy0oSf5co/NZXje7U1z8Mpep++QJOldL2hs/sBQf48= github.com/alecthomas/repr v0.0.0-20210801044451-80ca428c5142/go.mod h1:2kn6fqh/zIyPLmm3ugklbEi5hg5wS435eygvNfaDQL8= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= @@ -72,7 +65,6 @@ github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+Ce github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= @@ -163,7 +155,6 @@ github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-github/v41 v41.0.0 h1:HseJrM2JFf2vfiZJ8anY2hqBjdfY1Vlj/K27ueww4gg= github.com/google/go-github/v41 v41.0.0/go.mod h1:XgmCA5H323A9rtgExdTcnDkcqp6S30AVACCBDOonIxg= -github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -190,8 +181,6 @@ github.com/hashicorp/go-retryablehttp v0.6.8/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER github.com/hashicorp/go-retryablehttp v0.7.0 h1:eu1EI/mbirUgP5C8hVsTNaGZreBDlYiwC1FZWkvQPQ4= github.com/hashicorp/go-retryablehttp v0.7.0/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY= github.com/hashicorp/go-version v1.2.1/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= -github.com/hashicorp/go-version v1.3.0 h1:McDWVJIU/y+u1BRV06dPaLfLCaT7fUTJLp5r04x7iNw= -github.com/hashicorp/go-version v1.3.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/go-version v1.4.0 h1:aAQzgqIrRKRa7w75CKpbBxYsmUoPjzVm1W59ca1L0J4= github.com/hashicorp/go-version v1.4.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= @@ -229,12 +218,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/ktrysmt/go-bitbucket v0.9.32 h1:IVk0m0gdB4OzRRLgxFnqNsfWKPXNdrcvgdpp9BojTpI= -github.com/ktrysmt/go-bitbucket v0.9.32/go.mod h1:FWxy2UK7GlK5b0NSJGc5hPqnssVlkNnsChvyuOf/Xno= github.com/ktrysmt/go-bitbucket v0.9.36 h1:tXuZarVceStMxkmFxkSDMPfbm/Zff6wd8ctmus4d/s4= github.com/ktrysmt/go-bitbucket v0.9.36/go.mod h1:FWxy2UK7GlK5b0NSJGc5hPqnssVlkNnsChvyuOf/Xno= -github.com/lib/pq v1.10.4 h1:SO9z7FRPzA03QhHKJrH5BXA6HU1rS4V2nIVrrNC1iYk= -github.com/lib/pq v1.10.4/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/matryer/is v1.2.0 h1:92UTHpy8CDwaJ08GqLDzhhuixiBUUD1p3AU6PHddz4A= github.com/matryer/is v1.2.0/go.mod h1:2fLPjFQM9rhQ15aVEtbuwhJinnOqrmgXPNdZsdwlWXA= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= @@ -267,7 +252,6 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= -github.com/prometheus/client_golang v1.11.0 h1:HNkLOAEQMIDv/K+04rukrLx6ch7msSRwf3/SASFAGtQ= github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= github.com/prometheus/client_golang v1.12.1 h1:ZiaPsmm9uiBeaSMRznKsCDNtPCS0T3JVDGF+06gjBzk= github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= @@ -278,29 +262,21 @@ github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2 github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= -github.com/prometheus/common v0.26.0 h1:iMAkS2TDoNWnKM+Kopnx/8tnEStIfpYA0ur0xQzzhMQ= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= github.com/prometheus/common v0.32.1 h1:hWIdL3N2HoUx3B8j3YN9mWor0qhY/NlEKZEaXxuIRh4= github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= -github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.7.3 h1:4jVXhlkAyzOScmCkXBTOLRLTz8EeU+eyjrwB/EPq0VU= github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/robfig/cron/v3 v3.0.0 h1:kQ6Cb7aHOHTSzNVNEhmp8EcWKLb4CbiMW9h9VyIhO4E= -github.com/robfig/cron/v3 v3.0.0/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rs/xid v1.3.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= -github.com/rs/zerolog v1.26.0 h1:ORM4ibhEZeTeQlCojCK2kPz1ogAY4bGs4tD+SaAdGaE= -github.com/rs/zerolog v1.26.0/go.mod h1:yBiM87lvSqX8h0Ww4sdzNSkVYZ8dL2xjZJG1lAuGZEo= github.com/rs/zerolog v1.26.1 h1:/ihwxqH+4z8UxyI70wM1z9yCvkWcfz/a3mj48k/Zngc= github.com/rs/zerolog v1.26.1/go.mod h1:/wSSJWX7lVrsOwlbyTRSOJvqRlc+WjWlfes+CiJ+tmc= -github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= -github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ= github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= @@ -317,8 +293,6 @@ github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81P github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/xanzy/go-gitlab v0.52.2 h1:gkgg1z4ON70sphibtD86Bfmt1qV3mZ0pU0CBBCFAEvQ= -github.com/xanzy/go-gitlab v0.52.2/go.mod h1:Q+hQhV508bDPoBijv7YjK/Lvlb4PhVhJdKqXVQrUoAE= github.com/xanzy/go-gitlab v0.55.1 h1:IgX/DS9buV0AUz8fuJPQkdl0fQGfBiAsAHxpun8sNhg= github.com/xanzy/go-gitlab v0.55.1/go.mod h1:F0QEXwmqiBUxCgJm8fE9S+1veX4XC9Z4cfaAbqwk4YM= github.com/xanzy/ssh-agent v0.3.0/go.mod h1:3s9xbODqPuuhK9JV1R321M/FlMZSBvE5aY6eAcqrDh0= @@ -347,8 +321,6 @@ golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20211202192323-5770296d904e h1:MUP6MR3rJ7Gk9LEia0LP2ytiH6MuCfs7qYz+47jGdD8= -golang.org/x/crypto v0.0.0-20211202192323-5770296d904e/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20211215165025-cf75a172585e/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= golang.org/x/crypto v0.0.0-20220214200702-86341886e292 h1:f+lwQ+GtmgoY+A2YaQxlSOnDjXcQ7ZRLWOHbC6HtRqE= golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= @@ -418,8 +390,6 @@ golang.org/x/net v0.0.0-20210326060303-6b1517762897/go.mod h1:uSPa2vr4CLtc/ILN5o golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20211201190559-0a0e4e1bb54c h1:WtYZ93XtWSO5KlOMgPZu7hXY9WhMZpprvlm5VwvAl8c= -golang.org/x/net v0.0.0-20211201190559-0a0e4e1bb54c/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd h1:O7DYs+zxREGLKzKoMQrtrEacpb0ZVXA5rIwylE2Xchk= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/oauth2 v0.0.0-20180227000427-d7d64896b5ff/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -491,9 +461,6 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211031064116-611d5d643895/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211124211545-fe61309f8881 h1:TyHqChC80pFkXWraUUf6RuB5IqFdQieMLwwCJokV2pc= -golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e h1:fLOSk5Q00efkSvAm+4xcoXD+RRmLmmulPn5I3Y9F2EM= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220209214540-3681064d5158 h1:rm+CHSpPEEW2IsXUib1ThaHIjuBVZjxNgSKmBLFfD4c= @@ -506,14 +473,12 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20211116232009-f0f3c7e86c11 h1:GZokNIeuVkl3aZHJchRrr13WCsols02MLUcz1U9is6M= -golang.org/x/time v0.0.0-20211116232009-f0f3c7e86c11/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 h1:vVKdlvoWBphwdxWKrFZEuM0kGgGLxUOYcY4U/2Vjg44= golang.org/x/time v0.0.0-20220210224613-90d013bbcef8/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= diff --git a/gogs/gogs.go b/gogs/gogs.go index dfb4b28..541bef3 100644 --- a/gogs/gogs.go +++ b/gogs/gogs.go @@ -1,53 +1,102 @@ package gogs import ( - "gickup/types" - + "github.com/cooperspencer/gickup/types" "github.com/gogs/go-gogs-client" "github.com/rs/zerolog/log" ) +// Backup TODO. func Backup(r types.Repo, d types.GenRepo, dry bool) { - log.Info().Str("stage", "gogs").Str("url", d.Url).Msgf("mirroring %s to %s", types.Blue(r.Name), d.Url) - gogsclient := gogs.NewClient(d.Url, d.GetToken()) + log.Info(). + Str("stage", "gogs"). + Str("url", d.URL). + Msgf("mirroring %s to %s", types.Blue(r.Name), d.URL) + + gogsclient := gogs.NewClient(d.URL, d.GetToken()) user, err := gogsclient.GetSelfInfo() if err != nil { - log.Fatal().Str("stage", "gogs").Str("url", d.Url).Msg(err.Error()) + log.Fatal(). + Str("stage", "gogs"). + Str("url", d.URL). + Msg(err.Error()) } - if !dry { - repo, err := gogsclient.GetRepo(user.UserName, r.Name) - if err != nil { - opts := gogs.MigrateRepoOption{RepoName: r.Name, UID: int(user.ID), Mirror: true, CloneAddr: r.Url, AuthUsername: r.Token} - if r.Token == "" { - opts = gogs.MigrateRepoOption{RepoName: r.Name, UID: int(user.ID), Mirror: true, CloneAddr: r.Url, AuthUsername: r.Origin.User, AuthPassword: r.Origin.Password} - } - _, err := gogsclient.MigrateRepo(opts) - if err != nil { - log.Fatal().Str("stage", "gogs").Str("url", d.Url).Msg(err.Error()) - } - } else { - if repo.Mirror { - log.Info().Str("stage", "gogs").Str("url", d.Url).Msgf("mirror of %s already exists, syncing instead", types.Blue(r.Name)) - err := gogsclient.MirrorSync(user.UserName, repo.Name) - if err != nil { - log.Fatal().Str("stage", "gogs").Str("url", d.Url).Msg(err.Error()) - } - log.Info().Str("stage", "gogs").Str("url", d.Url).Msgf("successfully synced %s.", types.Blue(r.Name)) + + if dry { + return + } + + repo, err := gogsclient.GetRepo(user.UserName, r.Name) + if err != nil { + opts := gogs.MigrateRepoOption{ + RepoName: r.Name, + UID: int(user.ID), + Mirror: true, + CloneAddr: r.URL, + AuthUsername: r.Token, + } + + if r.Token == "" { + opts = gogs.MigrateRepoOption{ + RepoName: r.Name, + UID: int(user.ID), + Mirror: true, + CloneAddr: r.URL, + AuthUsername: r.Origin.User, + AuthPassword: r.Origin.Password, } } + + _, err := gogsclient.MigrateRepo(opts) + if err != nil { + log.Fatal(). + Str("stage", "gogs"). + Str("url", d.URL). + Msg(err.Error()) + } + + return + } + + if repo.Mirror { + log.Info(). + Str("stage", "gogs"). + Str("url", d.URL). + Msgf("mirror of %s already exists, syncing instead", types.Blue(r.Name)) + + err := gogsclient.MirrorSync(user.UserName, repo.Name) + if err != nil { + log.Fatal(). + Str("stage", "gogs"). + Str("url", d.URL). + Msg(err.Error()) + } + + log.Info(). + Str("stage", "gogs"). + Str("url", d.URL). + Msgf("successfully synced %s.", types.Blue(r.Name)) } } +// Get TODO. func Get(conf *types.Conf) []types.Repo { repos := []types.Repo{} for _, repo := range conf.Source.Gogs { - log.Info().Str("stage", "gogs").Str("url", repo.Url).Msgf("grabbing repositories from %s", repo.User) + log.Info(). + Str("stage", "gogs"). + Str("url", repo.URL). + Msgf("grabbing repositories from %s", repo.User) + token := repo.GetToken() - client := gogs.NewClient(repo.Url, token) + client := gogs.NewClient(repo.URL, token) gogsrepos, err := client.ListUserRepos(repo.User) if err != nil { - log.Fatal().Str("stage", "gogs").Str("url", repo.Url).Msg(err.Error()) + log.Fatal(). + Str("stage", "gogs"). + Str("url", repo.URL). + Msg(err.Error()) } include := types.GetMap(repo.Include) @@ -57,25 +106,67 @@ func Get(conf *types.Conf) []types.Repo { for _, r := range gogsrepos { if include[r.Name] { - repos = append(repos, types.Repo{Name: r.Name, Url: r.CloneURL, SshUrl: r.SSHURL, Token: token, Defaultbranch: r.DefaultBranch, Origin: repo, Owner: r.Owner.UserName, Hoster: types.GetHost(repo.Url)}) + repos = append(repos, types.Repo{ + Name: r.Name, + URL: r.CloneURL, + SSHURL: r.SSHURL, + Token: token, + Defaultbranch: r.DefaultBranch, + Origin: repo, + Owner: r.Owner.UserName, + Hoster: types.GetHost(repo.URL), + }) if repo.Wiki { - repos = append(repos, types.Repo{Name: r.Name + ".wiki", Url: types.DotGitRx.ReplaceAllString(r.CloneURL, ".wiki.git"), SshUrl: types.DotGitRx.ReplaceAllString(r.SSHURL, ".wiki.git"), Token: token, Defaultbranch: r.DefaultBranch, Origin: repo, Owner: r.Owner.UserName, Hoster: types.GetHost(repo.Url)}) + repos = append(repos, types.Repo{ + Name: r.Name + ".wiki", + URL: types.DotGitRx.ReplaceAllString(r.CloneURL, ".wiki.git"), + SSHURL: types.DotGitRx.ReplaceAllString(r.SSHURL, ".wiki.git"), + Token: token, + Defaultbranch: r.DefaultBranch, + Origin: repo, + Owner: r.Owner.UserName, + Hoster: types.GetHost(repo.URL), + }) } + continue } + if exclude[r.Name] { continue } + if len(include) == 0 { - repos = append(repos, types.Repo{Name: r.Name, Url: r.CloneURL, SshUrl: r.SSHURL, Token: token, Defaultbranch: r.DefaultBranch, Origin: repo, Owner: r.Owner.UserName, Hoster: types.GetHost(repo.Url)}) + repos = append(repos, types.Repo{ + Name: r.Name, + URL: r.CloneURL, + SSHURL: r.SSHURL, + Token: token, + Defaultbranch: r.DefaultBranch, + Origin: repo, + Owner: r.Owner.UserName, + Hoster: types.GetHost(repo.URL), + }) if repo.Wiki { - repos = append(repos, types.Repo{Name: r.Name + ".wiki", Url: types.DotGitRx.ReplaceAllString(r.CloneURL, ".wiki.git"), SshUrl: types.DotGitRx.ReplaceAllString(r.SSHURL, ".wiki.git"), Token: token, Defaultbranch: r.DefaultBranch, Origin: repo, Owner: r.Owner.UserName, Hoster: types.GetHost(repo.Url)}) + repos = append(repos, types.Repo{ + Name: r.Name + ".wiki", + URL: types.DotGitRx.ReplaceAllString(r.CloneURL, ".wiki.git"), + SSHURL: types.DotGitRx.ReplaceAllString(r.SSHURL, ".wiki.git"), + Token: token, + Defaultbranch: r.DefaultBranch, + Origin: repo, + Owner: r.Owner.UserName, + Hoster: types.GetHost(repo.URL), + }) } } } orgs, err := client.ListUserOrgs(repo.User) if err != nil { - log.Fatal().Str("stage", "gogs").Str("url", repo.Url).Msg(err.Error()) + log.Fatal(). + Str("stage", "gogs"). + Str("url", repo.URL). + Msg(err.Error()) } orgrepos := []*gogs.Repository{} @@ -88,43 +179,94 @@ func Get(conf *types.Conf) []types.Repo { if includeorgs[org.UserName] { o, err := client.ListOrgRepos(org.UserName) if err != nil { - log.Fatal().Str("stage", "gogs").Str("url", repo.Url).Msg(err.Error()) + log.Fatal(). + Str("stage", "gogs"). + Str("url", repo.URL). + Msg(err.Error()) } + if len(o) == 0 { break } + orgrepos = append(orgrepos, o...) } } else { o, err := client.ListOrgRepos(org.UserName) if err != nil { - log.Fatal().Str("stage", "gogs").Str("url", repo.Url).Msg(err.Error()) + log.Fatal(). + Str("stage", "gogs"). + Str("url", repo.URL). + Msg(err.Error()) } + if len(o) == 0 { break } + orgrepos = append(orgrepos, o...) } } } for _, r := range orgrepos { if include[r.Name] { - repos = append(repos, types.Repo{Name: r.Name, Url: r.CloneURL, SshUrl: r.SSHURL, Token: token, Defaultbranch: r.DefaultBranch, Origin: repo, Owner: r.Owner.UserName, Hoster: types.GetHost(repo.Url)}) + repos = append(repos, types.Repo{ + Name: r.Name, + URL: r.CloneURL, + SSHURL: r.SSHURL, + Token: token, + Defaultbranch: r.DefaultBranch, + Origin: repo, + Owner: r.Owner.UserName, + Hoster: types.GetHost(repo.URL), + }) if repo.Wiki { - repos = append(repos, types.Repo{Name: r.Name + ".wiki", Url: types.DotGitRx.ReplaceAllString(r.CloneURL, ".wiki.git"), SshUrl: types.DotGitRx.ReplaceAllString(r.SSHURL, ".wiki.git"), Token: token, Defaultbranch: r.DefaultBranch, Origin: repo, Owner: r.Owner.UserName, Hoster: types.GetHost(repo.Url)}) + repos = append(repos, types.Repo{ + Name: r.Name + ".wiki", + URL: types.DotGitRx.ReplaceAllString(r.CloneURL, ".wiki.git"), + SSHURL: types.DotGitRx.ReplaceAllString(r.SSHURL, ".wiki.git"), + Token: token, + Defaultbranch: r.DefaultBranch, + Origin: repo, + Owner: r.Owner.UserName, + Hoster: types.GetHost(repo.URL), + }) } + continue } + if exclude[r.Name] { continue } + if len(repo.Include) == 0 { - repos = append(repos, types.Repo{Name: r.Name, Url: r.CloneURL, SshUrl: r.SSHURL, Token: token, Defaultbranch: r.DefaultBranch, Origin: repo, Owner: r.Owner.UserName, Hoster: types.GetHost(repo.Url)}) + repos = append(repos, types.Repo{ + Name: r.Name, + URL: r.CloneURL, + SSHURL: r.SSHURL, + Token: token, + Defaultbranch: r.DefaultBranch, + Origin: repo, + Owner: r.Owner.UserName, + Hoster: types.GetHost(repo.URL), + }) + if repo.Wiki { - repos = append(repos, types.Repo{Name: r.Name + ".wiki", Url: types.DotGitRx.ReplaceAllString(r.CloneURL, ".wiki.git"), SshUrl: types.DotGitRx.ReplaceAllString(r.SSHURL, ".wiki.git"), Token: token, Defaultbranch: r.DefaultBranch, Origin: repo, Owner: r.Owner.UserName, Hoster: types.GetHost(repo.Url)}) + repos = append(repos, types.Repo{ + Name: r.Name + ".wiki", + URL: types.DotGitRx.ReplaceAllString(r.CloneURL, ".wiki.git"), + SSHURL: types.DotGitRx.ReplaceAllString(r.SSHURL, ".wiki.git"), + Token: token, + Defaultbranch: r.DefaultBranch, + Origin: repo, + Owner: r.Owner.UserName, + Hoster: types.GetHost(repo.URL), + }) } } } } + return repos } diff --git a/local/local.go b/local/local.go index 32b73f1..577fb19 100644 --- a/local/local.go +++ b/local/local.go @@ -1,13 +1,13 @@ package local import ( - "gickup/types" "net" "os" "path" "strings" "time" + "github.com/cooperspencer/gickup/types" "github.com/go-git/go-git/v5" "github.com/go-git/go-git/v5/config" "github.com/go-git/go-git/v5/plumbing/transport" @@ -18,94 +18,153 @@ import ( gossh "golang.org/x/crypto/ssh" ) +// Locally TODO. func Locally(repo types.Repo, l types.Local, dry bool) { if l.Structured { repo.Name = path.Join(repo.Hoster, repo.Owner, repo.Name) } + stat, err := os.Stat(l.Path) if os.IsNotExist(err) && !dry { - err := os.MkdirAll(l.Path, 0777) - if err != nil { - log.Fatal().Str("stage", "locally").Str("path", l.Path).Msg(err.Error()) + if err := os.MkdirAll(l.Path, 0o777); err != nil { + log.Fatal(). + Str("stage", "locally"). + Str("path", l.Path). + Msg(err.Error()) } + stat, _ = os.Stat(l.Path) } - if stat != nil { - if stat.IsDir() { - os.Chdir(l.Path) + + if stat != nil && stat.IsDir() { + if err := os.Chdir(l.Path); err != nil { + log.Fatal(). + Str("stage", "locally"). + Str("path", l.Path). + Msg(err.Error()) } } tries := 5 + var auth transport.AuthMethod - if repo.Origin.SSH { + + switch { + case repo.Origin.SSH: if repo.Origin.SSHKey == "" { home := os.Getenv("HOME") repo.Origin.SSHKey = path.Join(home, ".ssh", "id_rsa") } + auth, err = ssh.NewPublicKeysFromFile("git", repo.Origin.SSHKey, "") if err != nil { panic(err) } - } else if repo.Token != "" { + case repo.Token != "": auth = &http.BasicAuth{ Username: "xyz", Password: repo.Token, } - } else if repo.Origin.Username != "" && repo.Origin.Password != "" { + case repo.Origin.Username != "" && repo.Origin.Password != "": auth = &http.BasicAuth{ Username: repo.Origin.Username, Password: repo.Origin.Password, } } + for x := 1; x <= tries; x++ { stat, err := os.Stat(repo.Name) if os.IsNotExist(err) { - log.Info().Str("stage", "locally").Str("path", l.Path).Msgf("cloning %s", types.Green(repo.Name)) + log.Info(). + Str("stage", "locally"). + Str("path", l.Path). + Msgf("cloning %s", types.Green(repo.Name)) err := cloneRepository(repo, auth, dry) - if err != nil { if x == tries { - log.Fatal().Str("stage", "locally").Str("path", l.Path).Str("repo", repo.Name).Msg(err.Error()) - } else { - if strings.Contains(err.Error(), "ERR access denied or repository not exported") { - log.Warn().Str("stage", "locally").Str("path", l.Path).Str("repo", repo.Name).Msgf("%s doesn't exist.", repo.Name) - break - } - if strings.Contains(err.Error(), "remote repository is empty") { - log.Warn().Str("stage", "locally").Str("path", l.Path).Str("repo", repo.Name).Msg(err.Error()) - break - } - log.Warn().Str("stage", "locally").Str("path", l.Path).Msgf("retry %s from %s", types.Red(x), types.Red(tries)) - time.Sleep(5 * time.Second) - continue + log.Fatal(). + Str("stage", "locally"). + Str("path", l.Path). + Str("repo", repo.Name). + Msg(err.Error()) + + break } + + if strings.Contains(err.Error(), "ERR access denied or repository not exported") { + log.Warn(). + Str("stage", "locally"). + Str("path", l.Path). + Str("repo", repo.Name). + Msgf("%s doesn't exist.", repo.Name) + + break + } + + if strings.Contains(err.Error(), "remote repository is empty") { + log.Warn(). + Str("stage", "locally"). + Str("path", l.Path). + Str("repo", repo.Name). + Msg(err.Error()) + + break + } + + log.Warn(). + Str("stage", "locally"). + Str("path", l.Path). + Msgf("retry %s from %s", types.Red(x), types.Red(tries)) + + time.Sleep(5 * time.Second) + + continue } } else { if !stat.IsDir() { - log.Warn().Str("stage", "locally").Str("path", l.Path).Str("repo", repo.Name).Msgf("%s is a file", types.Red(repo.Name)) + log.Warn(). + Str("stage", "locally"). + Str("path", l.Path). + Str("repo", repo.Name). + Msgf("%s is a file", types.Red(repo.Name)) } else { - log.Info().Str("stage", "locally").Str("path", l.Path).Msgf("opening %s locally", types.Green(repo.Name)) + log.Info(). + Str("stage", "locally"). + Str("path", l.Path). + Msgf("opening %s locally", types.Green(repo.Name)) err := updateRepository(repo.Name, auth, dry) - if err != nil { if strings.Contains(err.Error(), "already up-to-date") { - log.Info().Str("stage", "locally").Str("path", l.Path).Msg(err.Error()) + log.Info(). + Str("stage", "locally"). + Str("path", l.Path). + Msg(err.Error()) } else { if x == tries { - log.Fatal().Str("stage", "locally").Str("path", l.Path).Str("repo", repo.Name).Msg(err.Error()) + log.Fatal(). + Str("stage", "locally"). + Str("path", l.Path). + Str("repo", repo.Name). + Msg(err.Error()) } else { os.RemoveAll(repo.Name) - log.Warn().Str("stage", "locally").Str("path", l.Path).Str("repo", repo.Name).Msgf("retry %s from %s", types.Red(x), types.Red(tries)) + log.Warn(). + Str("stage", "locally"). + Str("path", l.Path). + Str("repo", repo.Name). + Msgf("retry %s from %s", types.Red(x), types.Red(tries)) + time.Sleep(5 * time.Second) + continue } } } } } + x = 5 } } @@ -122,55 +181,67 @@ func updateRepository(repoPath string, auth transport.AuthMethod, dry bool) erro } if !dry { - log.Info().Str("stage", "locally").Msgf("pulling %s", types.Green(repoPath)) + log.Info(). + Str("stage", "locally"). + Msgf("pulling %s", types.Green(repoPath)) + err = w.Pull(&git.PullOptions{Auth: auth, RemoteName: "origin", SingleBranch: false}) } + return err } func cloneRepository(repo types.Repo, auth transport.AuthMethod, dry bool) error { - if !dry { - url := repo.Url - if repo.Origin.SSH { - url = repo.SshUrl - site := types.Site{} - err := site.GetValues(url) - if err != nil { - log.Fatal().Str("stage", "locally").Str("repo", repo.Name).Msg(err.Error()) - } + if dry { + return nil + } - sshAuth, err := goph.Key(repo.Origin.SSHKey, "") - if err != nil { - log.Fatal().Str("stage", "locally").Str("repo", repo.Name).Msg(err.Error()) - } + url := repo.URL + if repo.Origin.SSH { + url = repo.SSHURL + site := types.Site{} - err = testSshConnection(site, sshAuth) - if err != nil { - log.Fatal().Str("stage", "locally").Str("repo", repo.Name).Msg(err.Error()) - } - } - - rem := git.NewRemote(nil, &config.RemoteConfig{Name: "origin", URLs: []string{url}}) - _, err := rem.List(&git.ListOptions{Auth: auth}) + err := site.GetValues(url) if err != nil { - return err + log.Fatal().Str("stage", "locally").Str("repo", repo.Name).Msg(err.Error()) } - _, err = git.PlainClone(repo.Name, false, &git.CloneOptions{ - URL: url, - Auth: auth, - SingleBranch: false, - }) + sshAuth, err := goph.Key(repo.Origin.SSHKey, "") + if err != nil { + log.Fatal().Str("stage", "locally").Str("repo", repo.Name).Msg(err.Error()) + } + err = testSSHConnection(site, sshAuth) + if err != nil { + log.Fatal().Str("stage", "locally").Str("repo", repo.Name).Msg(err.Error()) + } + } + + remoteConfig := config.RemoteConfig{ + Name: "origin", + URLs: []string{url}, + } + + rem := git.NewRemote(nil, &remoteConfig) + + _, err := rem.List(&git.ListOptions{Auth: auth}) + if err != nil { return err } - return nil + + _, err = git.PlainClone(repo.Name, false, &git.CloneOptions{ + URL: url, + Auth: auth, + SingleBranch: false, + }) + + return err } -func testSshConnection(site types.Site, sshAuth goph.Auth) error { +func testSSHConnection(site types.Site, sshAuth goph.Auth) error { _, err := goph.NewConn(&goph.Config{ User: site.User, - Addr: site.Url, + Addr: site.URL, Port: uint(site.Port), Auth: sshAuth, Callback: VerifyHost, @@ -179,8 +250,10 @@ func testSshConnection(site types.Site, sshAuth goph.Auth) error { return err } +// VerifyHost TODO. func VerifyHost(host string, remote net.Addr, key gossh.PublicKey) error { - // Got from the example from https://github.com/melbahja/goph/blob/master/examples/goph/main.go + // Got from the example from + // https://github.com/melbahja/goph/blob/master/examples/goph/main.go // // If you want to connect to new hosts. // here your should check new connections public keys @@ -188,7 +261,8 @@ func VerifyHost(host string, remote net.Addr, key gossh.PublicKey) error { // // hostFound: is host in known hosts file. - // err: error if key not in known hosts file OR host in known hosts file but key changed! + // err: error if key not in known hosts file + // OR host in known hosts file but key changed! hostFound, err := goph.CheckKnownHost(host, remote, key, "") // Host in known hosts but key mismatch! // Maybe because of MAN IN THE MIDDLE ATTACK! diff --git a/logger/logger.go b/logger/logger.go index 767d80f..ea36bf0 100644 --- a/logger/logger.go +++ b/logger/logger.go @@ -5,17 +5,21 @@ import ( "os" "path" - "gickup/types" - + "github.com/cooperspencer/gickup/types" "github.com/rs/zerolog" "github.com/rs/zerolog/log" "gopkg.in/natefinch/lumberjack.v2" ) +// NewRollingFile TODO. func NewRollingFile(config types.FileLogging) io.Writer { if config.Dir != "" { - if err := os.MkdirAll(config.Dir, 0744); err != nil { - log.Error().Err(err).Str("path", config.Dir).Msg("can't create log directory") + if err := os.MkdirAll(config.Dir, 0o744); err != nil { + log.Error(). + Err(err). + Str("path", config.Dir). + Msg("can't create log directory") + return nil } } else { @@ -28,6 +32,7 @@ func NewRollingFile(config types.FileLogging) io.Writer { } } +// CreateLogger create an instance of Logger. func CreateLogger(conf types.Logging) zerolog.Logger { var writers []io.Writer @@ -36,6 +41,7 @@ func CreateLogger(conf types.Logging) zerolog.Logger { if conf.FileLogging.File != "" { writers = append(writers, NewRollingFile(conf.FileLogging)) } + mw := io.MultiWriter(writers...) return zerolog.New(mw).With().Timestamp().Logger() diff --git a/main.go b/main.go index c067f1d..ac1c6ba 100644 --- a/main.go +++ b/main.go @@ -9,17 +9,16 @@ import ( "strings" "time" - "gickup/bitbucket" - "gickup/gitea" - "gickup/github" - "gickup/gitlab" - "gickup/gogs" - "gickup/local" - "gickup/logger" - prometheus "gickup/metrics/prometheus" - "gickup/types" - "github.com/alecthomas/kong" + "github.com/cooperspencer/gickup/bitbucket" + "github.com/cooperspencer/gickup/gitea" + "github.com/cooperspencer/gickup/github" + "github.com/cooperspencer/gickup/gitlab" + "github.com/cooperspencer/gickup/gogs" + "github.com/cooperspencer/gickup/local" + "github.com/cooperspencer/gickup/logger" + "github.com/cooperspencer/gickup/metrics/prometheus" + "github.com/cooperspencer/gickup/types" "github.com/robfig/cron/v3" "github.com/rs/zerolog" "github.com/rs/zerolog/log" @@ -27,38 +26,40 @@ import ( ) var cli struct { - Configfile string `arg name:"conf" help:"path to the configfile." default:"conf.yml"` - Version bool `flag name:"version" help:"show version."` - Dry bool `flag name:"dryrun" help:"make a dry-run."` + Configfile string `arg name:"conf" help:"Path to the configfile." default:"conf.yml"` + Version bool `flag name:"version" help:"Show version."` + Dry bool `flag name:"dryrun" help:"Make a dry-run."` Quiet bool `flag name:"quiet" help:"Output only warnings, errors, and fatal messages to stderr log output"` Silent bool `flag name:"silent" help:"Suppress all stderr log output"` } -var ( - version = "unknown" -) +var version = "unknown" -func ReadConfigfile(configfile string) *types.Conf { +func readConfigFile(configfile string) *types.Conf { cfgdata, err := ioutil.ReadFile(configfile) - if err != nil { - log.Fatal().Str("stage", "readconfig").Str("file", configfile).Msgf("Cannot open config file from %s", types.Red(configfile)) + log.Fatal(). + Str("stage", "readconfig"). + Str("file", configfile). + Msgf("Cannot open config file from %s", types.Red(configfile)) } t := types.Conf{} - err = yaml.Unmarshal([]byte(cfgdata), &t) + err = yaml.Unmarshal(cfgdata, &t) if err != nil { - log.Fatal().Str("stage", "readconfig").Str("file", configfile).Msg("Cannot map yml config file to interface, possible syntax error") + log.Fatal(). + Str("stage", "readconfig"). + Str("file", configfile). + Msg("Cannot map yml config file to interface, possible syntax error") } return &t } -func GetUserHome() (string, error) { +func getUserHome() (string, error) { usr, err := user.Current() - if err != nil { return "", err } @@ -66,65 +67,88 @@ func GetUserHome() (string, error) { return usr.HomeDir, nil } -func SubstituteHomeForTildeInPath(path string) string { +func substituteHomeForTildeInPath(path string) string { if !strings.HasPrefix(path, "~") { return path - } else { - if path == "~" { - userHome, err := GetUserHome() - if err != nil { - log.Fatal().Str("stage", "local ~ substitution").Str("path", path).Msg(err.Error()) - } else { - return userHome - } - } else if strings.HasPrefix(path, "~/") { - userHome, err := GetUserHome() - if err != nil { - log.Fatal().Str("stage", "local ~/ substitution").Str("path", path).Msg(err.Error()) - } else { - return filepath.Join(userHome, path[2:]) - } + } + + if path == "~" { + userHome, err := getUserHome() + if err != nil { + log.Fatal(). + Str("stage", "local ~ substitution"). + Str("path", path). + Msg(err.Error()) + } else { + return userHome + } + } + + if strings.HasPrefix(path, "~/") { + userHome, err := getUserHome() + if err != nil { + log.Fatal(). + Str("stage", "local ~/ substitution"). + Str("path", path). + Msg(err.Error()) + } else { + return filepath.Join(userHome, path[2:]) } } // in any other strange case return path } -func Backup(repos []types.Repo, conf *types.Conf) { +func backup(repos []types.Repo, conf *types.Conf) { checkedpath := false + for _, r := range repos { - log.Info().Str("stage", "backup").Msgf("starting backup for %s", r.Url) + log.Info(). + Str("stage", "backup"). + Msgf("starting backup for %s", r.URL) + for i, d := range conf.Destination.Local { if !checkedpath { - d.Path = SubstituteHomeForTildeInPath(d.Path) + d.Path = substituteHomeForTildeInPath(d.Path) + path, err := filepath.Abs(d.Path) if err != nil { - log.Fatal().Str("stage", "locally").Str("path", d.Path).Msg(err.Error()) + log.Fatal(). + Str("stage", "locally"). + Str("path", d.Path). + Msg(err.Error()) } + conf.Destination.Local[i].Path = path checkedpath = true } + local.Locally(r, d, cli.Dry) prometheus.DestinationBackupsComplete.WithLabelValues("local").Inc() } + for _, d := range conf.Destination.Gitea { gitea.Backup(r, d, cli.Dry) prometheus.DestinationBackupsComplete.WithLabelValues("gitea").Inc() } + for _, d := range conf.Destination.Gogs { gogs.Backup(r, d, cli.Dry) prometheus.DestinationBackupsComplete.WithLabelValues("gogs").Inc() } + for _, d := range conf.Destination.Gitlab { gitlab.Backup(r, d, cli.Dry) prometheus.DestinationBackupsComplete.WithLabelValues("gitlab").Inc() } + prometheus.SourceBackupsComplete.WithLabelValues(r.Name).Inc() } } -func RunBackup(conf *types.Conf) { +func runBackup(conf *types.Conf) { log.Info().Msg("Backup run starting") + startTime := time.Now() prometheus.JobsStarted.Inc() @@ -132,27 +156,26 @@ func RunBackup(conf *types.Conf) { // Github repos := github.Get(conf) prometheus.CountReposDiscovered.WithLabelValues("github").Set(float64(len(repos))) - Backup(repos, conf) + backup(repos, conf) // Gitea repos = gitea.Get(conf) prometheus.CountReposDiscovered.WithLabelValues("gitea").Set(float64(len(repos))) - Backup(repos, conf) + backup(repos, conf) // Gogs repos = gogs.Get(conf) prometheus.CountReposDiscovered.WithLabelValues("gogs").Set(float64(len(repos))) - Backup(repos, conf) + backup(repos, conf) // Gitlab repos = gitlab.Get(conf) prometheus.CountReposDiscovered.WithLabelValues("gitlab").Set(float64(len(repos))) - Backup(repos, conf) + backup(repos, conf) - //Bitbucket repos = bitbucket.Get(conf) prometheus.CountReposDiscovered.WithLabelValues("bitbucket").Set(float64(len(repos))) - Backup(repos, conf) + backup(repos, conf) endTime := time.Now() duration := endTime.Sub(startTime) @@ -169,8 +192,9 @@ func RunBackup(conf *types.Conf) { } } -func PlaysForever() { +func playsForever() { wait := make(chan struct{}) + for { <-wait } @@ -178,61 +202,79 @@ func PlaysForever() { func main() { timeformat := "2006-01-02T15:04:05Z07:00" - log.Logger = log.Output(zerolog.ConsoleWriter{Out: os.Stderr, TimeFormat: timeformat}) + log.Logger = log.Output(zerolog.ConsoleWriter{ + Out: os.Stderr, + TimeFormat: timeformat, + }) - kong.Parse(&cli, kong.Name("gickup"), kong.Description("a tool to backup all your favorite repos")) + kong.Parse(&cli, kong.Name("gickup"), + kong.Description("a tool to backup all your favorite repos")) if cli.Version { fmt.Println(version) - } else { - if cli.Quiet { - zerolog.SetGlobalLevel(zerolog.WarnLevel) - } - if cli.Silent { - zerolog.SetGlobalLevel(zerolog.Disabled) - } + return + } - if cli.Dry { - log.Info().Str("dry", "true").Msgf("this is a %s", types.Blue("dry run")) - } + if cli.Quiet { + zerolog.SetGlobalLevel(zerolog.WarnLevel) + } - log.Info().Str("file", cli.Configfile).Msgf("Reading %s", types.Green(cli.Configfile)) + if cli.Silent { + zerolog.SetGlobalLevel(zerolog.Disabled) + } - conf := ReadConfigfile(cli.Configfile) - if conf.Log.Timeformat == "" { - conf.Log.Timeformat = timeformat - } - - log.Logger = logger.CreateLogger(conf.Log) - - // one pair per source-destination - pairs := conf.Source.Count() * conf.Destination.Count() + if cli.Dry { log.Info(). - Int("sources", conf.Source.Count()). - Int("destinations", conf.Destination.Count()). - Int("pairs", pairs). - Msg("Configuration loaded") + Str("dry", "true"). + Msgf("this is a %s", types.Blue("dry run")) + } - if conf.HasValidCronSpec() { - c := cron.New() - logNextRun(conf) + log.Info().Str("file", cli.Configfile). + Msgf("Reading %s", types.Green(cli.Configfile)) - c.AddFunc(conf.Cron, func() { - RunBackup(conf) - }) - c.Start() + conf := readConfigFile(cli.Configfile) + if conf.Log.Timeformat == "" { + conf.Log.Timeformat = timeformat + } - if conf.HasAllPrometheusConf() { - prometheus.CountSourcesConfigured.Add(float64(conf.Source.Count())) - prometheus.CountDestinationsConfigured.Add(float64(conf.Destination.Count())) - prometheus.Serve(conf.Metrics.Prometheus) - } else { - PlaysForever() - } - } else { - RunBackup(conf) + log.Logger = logger.CreateLogger(conf.Log) + + // one pair per source-destination + pairs := conf.Source.Count() * conf.Destination.Count() + log.Info(). + Int("sources", conf.Source.Count()). + Int("destinations", conf.Destination.Count()). + Int("pairs", pairs). + Msg("Configuration loaded") + + if conf.HasValidCronSpec() { + c := cron.New() + + logNextRun(conf) + + _, err := c.AddFunc(conf.Cron, func() { + runBackup(conf) + }) + if err != nil { + log.Fatal(). + Int("sources", conf.Source.Count()). + Int("destinations", conf.Destination.Count()). + Int("pairs", pairs). + Msg(err.Error()) } + + c.Start() + + if conf.HasAllPrometheusConf() { + prometheus.CountSourcesConfigured.Add(float64(conf.Source.Count())) + prometheus.CountDestinationsConfigured.Add(float64(conf.Destination.Count())) + prometheus.Serve(conf.Metrics.Prometheus) + } else { + playsForever() + } + } else { + runBackup(conf) } } diff --git a/main_test.go b/main_test.go index fff9772..b93f98a 100644 --- a/main_test.go +++ b/main_test.go @@ -6,22 +6,26 @@ import ( ) func TestTildeReplacement_NoAction(t *testing.T) { - path := "/boop" - if SubstituteHomeForTildeInPath(path) != path { + t.Parallel() + + if path := "/boop"; substituteHomeForTildeInPath(path) != path { t.Error("Altered path when no alteration was expected") } } func TestTildeReplacement_TildeOnly(t *testing.T) { - path := "~" - if SubstituteHomeForTildeInPath(path) == path { + t.Parallel() + + if path := "~"; substituteHomeForTildeInPath(path) == path { t.Error("Path unaltered when alteration was expected") } } func TestTildeReplacement_TildeDir(t *testing.T) { + t.Parallel() + path := "~/boop" - actual := SubstituteHomeForTildeInPath(path) + actual := substituteHomeForTildeInPath(path) if strings.HasPrefix(actual, "~") { t.Error("Altered path still contains ~") } diff --git a/metrics/prometheus/prometheus.go b/metrics/prometheus/prometheus.go index d036b3d..83b568f 100644 --- a/metrics/prometheus/prometheus.go +++ b/metrics/prometheus/prometheus.go @@ -1,9 +1,9 @@ package prometheus import ( - "gickup/types" "net/http" + "github.com/cooperspencer/gickup/types" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" "github.com/prometheus/client_golang/prometheus/promhttp" @@ -51,8 +51,15 @@ var DestinationBackupsComplete = promauto.NewCounterVec(prometheus.CounterOpts{ }, []string{"destination_type"}) func Serve(conf types.PrometheusConfig) { - log.Info().Str("listenAddr", conf.ListenAddr).Str("endpoint", conf.Endpoint).Msg("Starting Prometheus listener") + log.Info(). + Str("listenAddr", conf.ListenAddr). + Str("endpoint", conf.Endpoint). + Msg("Starting Prometheus listener") http.Handle(conf.Endpoint, promhttp.Handler()) - http.ListenAndServe(conf.ListenAddr, nil) + err := http.ListenAndServe(conf.ListenAddr, nil) + log.Fatal(). + Str("listenAddr", conf.ListenAddr). + Str("endpoint", conf.Endpoint). + Msg(err.Error()) } diff --git a/types/types.go b/types/types.go index fa60fc3..8402e96 100644 --- a/types/types.go +++ b/types/types.go @@ -19,7 +19,7 @@ import ( "github.com/rs/zerolog/log" ) -// Destination +// Destination TODO. type Destination struct { Gitlab []GenRepo `yaml:"gitlab"` Local []Local `yaml:"local"` @@ -28,6 +28,7 @@ type Destination struct { Gogs []GenRepo `yaml:"gogs"` } +// Count TODO. func (dest Destination) Count() int { return len(dest.Gogs) + len(dest.Gitea) + @@ -36,13 +37,13 @@ func (dest Destination) Count() int { len(dest.Gitlab) } -// Local +// Local TODO. type Local struct { Path string `yaml:"path"` Structured bool `yaml:"structured"` } -// Conf +// Conf TODO. type Conf struct { Source Source `yaml:"source"` Destination Destination `yaml:"destination"` @@ -51,32 +52,39 @@ type Conf struct { Metrics Metrics `yaml:"metrics"` } +// PrometheusConfig TODO. type PrometheusConfig struct { ListenAddr string `yaml:"listen_addr"` Endpoint string `yaml:"endpoint"` } +// Metrics TODO. type Metrics struct { Prometheus PrometheusConfig `yaml:"prometheus"` } +// Logging TODO. type Logging struct { Timeformat string `yaml:"timeformat"` FileLogging FileLogging `yaml:"file-logging"` } +// FileLogging TODO. type FileLogging struct { Dir string `yaml:"dir"` File string `yaml:"file"` MaxAge int `yaml:"maxage"` } +// CheckAllValuesOrNone TODO. func CheckAllValuesOrNone(parent string, theMap map[string]string) bool { - missing := false + var missing bool + for key, value := range theMap { if value == "" { - log.Warn().Str("expectedButMissing", key).Msg( - "A configuration value is expected but not present. Ensure all required configuration is present.") + log.Warn().Str("expectedButMissing", key). + Msg("A configuration value is expected but not present. Ensure all required configuration is present.") + missing = true } } @@ -84,33 +92,34 @@ func CheckAllValuesOrNone(parent string, theMap map[string]string) bool { return !missing } +// HasAllPrometheusConf TODO. func (conf Conf) HasAllPrometheusConf() bool { if len(conf.Metrics.Prometheus.ListenAddr) == 0 && len(conf.Metrics.Prometheus.Endpoint) == 0 { return false - } else { - checks := map[string]string{ - "listenaddr": conf.Metrics.Prometheus.ListenAddr, - "endpoint": conf.Metrics.Prometheus.Endpoint, - } - - ok := CheckAllValuesOrNone("prometheus", checks) - - if !ok { - log.Fatal().Str("monitoring", "prometheus").Msg( - "Fix the values in the configuration.") - } - - return ok } + + checks := map[string]string{ + "listenaddr": conf.Metrics.Prometheus.ListenAddr, + "endpoint": conf.Metrics.Prometheus.Endpoint, + } + + ok := CheckAllValuesOrNone("prometheus", checks) + if !ok { + log.Fatal().Str("monitoring", "prometheus").Msg( + "Fix the values in the configuration.") + } + + return ok } +// MissingCronSpec TODO. func (conf Conf) MissingCronSpec() bool { return conf.Cron == "" } +// ParseCronSpec TODO. func ParseCronSpec(spec string) (cron.Schedule, error) { sched, err := cron.ParseStandard(spec) - if err != nil { log.Error().Str("spec", spec).Msg(err.Error()) } @@ -118,18 +127,23 @@ func ParseCronSpec(spec string) (cron.Schedule, error) { return sched, err } +// GetNextRun TODO. func (conf Conf) GetNextRun() (*time.Time, error) { if conf.MissingCronSpec() { return nil, fmt.Errorf("cron unspecified") } + parsedSched, err := ParseCronSpec(conf.Cron) if err != nil { return nil, err } + next := parsedSched.Next(time.Now()) + return &next, nil } +// HasValidCronSpec TODO. func (conf Conf) HasValidCronSpec() bool { if conf.MissingCronSpec() { return false @@ -140,7 +154,7 @@ func (conf Conf) HasValidCronSpec() bool { return err == nil } -// Source +// Source TODO. type Source struct { Gogs []GenRepo `yaml:"gogs"` Gitlab []GenRepo `yaml:"gitlab"` @@ -149,6 +163,7 @@ type Source struct { BitBucket []GenRepo `yaml:"bitbucket"` } +// Count TODO. func (source Source) Count() int { return len(source.Gogs) + len(source.Gitea) + @@ -157,7 +172,7 @@ func (source Source) Count() int { len(source.Gitlab) } -// Generell Repo +// GenRepo Generell Repo. type GenRepo struct { Token string `yaml:"token"` TokenFile string `yaml:"token_file"` @@ -166,7 +181,7 @@ type GenRepo struct { SSHKey string `yaml:"sshkey"` Username string `yaml:"username"` Password string `yaml:"password"` - Url string `yaml:"url"` + URL string `yaml:"url"` Exclude []string `yaml:"exclude"` ExcludeOrgs []string `yaml:"excludeorgs"` Include []string `yaml:"include"` @@ -175,12 +190,12 @@ type GenRepo struct { Starred bool `yaml:"starred"` } +// GetToken TODO. func (grepo GenRepo) GetToken() string { token, err := resolveToken(grepo.Token, grepo.TokenFile) - if err != nil { log.Fatal(). - Str("url", grepo.Url). + Str("url", grepo.URL). Str("tokenfile", grepo.TokenFile). Err(err) } @@ -195,7 +210,6 @@ func resolveToken(tokenString string, tokenFile string) (string, error) { if tokenFile != "" { data, err := os.ReadFile(tokenFile) - if err != nil { return "", err } @@ -206,17 +220,18 @@ func resolveToken(tokenString string, tokenFile string) (string, error) { Msg("Read token file") tokenData := strings.ReplaceAll(string(data), "\n", "") - return tokenData, nil + return tokenData, nil } + return "", fmt.Errorf("no token or tokenfile was specified in config when one was expected") } -// Repo +// Repo TODO. type Repo struct { Name string - Url string - SshUrl string + URL string + SSHURL string Token string Defaultbranch string Origin GenRepo @@ -224,80 +239,111 @@ type Repo struct { Hoster string } -// Site +// Site TODO. type Site struct { - Url string + URL string User string Port int } +// GetHost TODO. func GetHost(url string) string { if strings.Contains(url, "http://") { url = strings.Split(url, "http://")[1] } + if strings.Contains(url, "https://") { url = strings.Split(url, "https://")[1] } + if strings.Contains(url, "/") { url = strings.Split(url, "/")[0] } + return url } +// GetValues TODO. func (s *Site) GetValues(url string) error { if strings.HasPrefix(url, "ssh://") { url = strings.Split(url, "ssh://")[1] userurl := strings.Split(url, "@") s.User = userurl[0] + urlport := strings.Split(userurl[1], ":") - s.Url = urlport[0] + s.URL = urlport[0] + portstring := strings.Split(urlport[1], "/")[0] + port, err := strconv.Atoi(portstring) if err != nil { return err } + s.Port = port - } else { - userurl := strings.Split(url, "@") - s.User = userurl[0] - urlport := strings.Split(userurl[1], ":") - s.Url = urlport[0] - s.Port = 22 + + return nil } + + userurl := strings.Split(url, "@") + s.User = userurl[0] + + urlport := strings.Split(userurl[1], ":") + s.URL = urlport[0] + s.Port = 22 + return nil } var ( - Red = color.FgRed.Render - Green = color.FgGreen.Render - Blue = color.FgBlue.Render + // Red Render message with Red color. + Red = color.FgRed.Render + // Green Render message with Green color. + Green = color.FgGreen.Render + // Blue Render message with Blue color. + Blue = color.FgBlue.Render + // DotGitRx .git regexp. DotGitRx = regexp.MustCompile(`\.git$`) ) +// GetMap TODO. func GetMap(excludes []string) map[string]bool { excludemap := make(map[string]bool) for _, exclude := range excludes { excludemap[exclude] = true } + return excludemap } -func StatRemote(URL, sshURL string, repo GenRepo) bool { - var url string - var auth transport.AuthMethod - var err error +func statRemoteSSH(sshURL string, repo GenRepo) (string, transport.AuthMethod, error) { + url := DotGitRx.ReplaceAllString(sshURL, ".wiki.git") + + if repo.SSHKey == "" { + home := os.Getenv("HOME") + repo.SSHKey = path.Join(home, ".ssh", "id_rsa") + } + + auth, err := ssh.NewPublicKeysFromFile("git", repo.SSHKey, "") + + return url, auth, err +} + +// StatRemote TODO. +func StatRemote(remoteURL, sshURL string, repo GenRepo) bool { + var ( + url string + auth transport.AuthMethod + err error + ) + if repo.SSH { - url = DotGitRx.ReplaceAllString(sshURL, ".wiki.git") - if repo.SSHKey == "" { - home := os.Getenv("HOME") - repo.SSHKey = path.Join(home, ".ssh", "id_rsa") - } - auth, err = ssh.NewPublicKeysFromFile("git", repo.SSHKey, "") + url, auth, err = statRemoteSSH(sshURL, repo) if err != nil { - panic(err) + return false } } else { - url = DotGitRx.ReplaceAllString(URL, ".wiki.git") + url = DotGitRx.ReplaceAllString(remoteURL, ".wiki.git") if repo.Token != "" { auth = &http.BasicAuth{ Username: "xyz", @@ -310,6 +356,13 @@ func StatRemote(URL, sshURL string, repo GenRepo) bool { } } } - _, err = git.NewRemote(nil, &config.RemoteConfig{Name: "origin", URLs: []string{url}}).List(&git.ListOptions{Auth: auth}) + + remoteConfig := config.RemoteConfig{ + Name: "origin", + URLs: []string{url}, + } + + _, err = git.NewRemote(nil, &remoteConfig).List(&git.ListOptions{Auth: auth}) + return err == nil } diff --git a/types/types_test.go b/types/types_test.go index e59ce14..3347504 100644 --- a/types/types_test.go +++ b/types/types_test.go @@ -3,6 +3,8 @@ package types import "testing" func TestConfCronMissing(t *testing.T) { + t.Parallel() + conf := Conf{Cron: ""} if !conf.MissingCronSpec() { @@ -11,6 +13,8 @@ func TestConfCronMissing(t *testing.T) { } func TestParseValidCron(t *testing.T) { + t.Parallel() + conf := Conf{Cron: "0 0 * * *"} if !conf.HasValidCronSpec() { @@ -19,6 +23,8 @@ func TestParseValidCron(t *testing.T) { } func TestParseInvalidCron(t *testing.T) { + t.Parallel() + conf := Conf{Cron: "redshirt"} if conf.HasValidCronSpec() {