chore: add linting (#101)

* chore: adding makefile

* chore: adding lint

* chore: update package name

* lint: run with fix

* chore: update main.go

* chore: update main_test.go

* chore: update types.go

* chore: update logger.go

* chore: update types_test.go

* chore: update prometheus.go

* chore: update local.go

* chore: update gitlab.go

* chore: update gogs.go

* chore: update bitbucket.go

* chore: update gitea.go

* chore: update github.go

* chore: update main*.go

* chore: go mod tidy

* chore: update action to have a correct working-directory
This commit is contained in:
Tom Moulard
2022-03-20 11:25:11 +01:00
committed by GitHub
parent 2a069398ad
commit 646b792ac1
18 changed files with 1276 additions and 435 deletions
+55
View File
@@ -0,0 +1,55 @@
name: Validate lint
on:
pull_request:
branches:
- '*'
env:
GO_VERSION: 1.17
GOLANGCI_LINT_VERSION: v1.45.0
jobs:
lint:
runs-on: ubuntu-20.04
defaults:
run:
working-directory: ${{ github.workspace }}/go/src/github.com/cooperspencer/gickup
steps:
- name: Set up Go ${{ env.GO_VERSION }}
uses: actions/setup-go@v2
with:
go-version: ${{ env.GO_VERSION }}
- name: Check out code
uses: actions/checkout@v2
with:
path: go/src/github.com/cooperspencer/gickup
fetch-depth: 0
- name: Cache Go modules
uses: actions/cache@v2
with:
path: |
~/go/pkg/mod
~/.cache/go-build
key: ${{ runner.os }}-lint-go-${{ hashFiles('**/go.sum') }}
restore-keys: ${{ runner.os }}-lint-go
- name: Setup GOPATH
run: go env -w GOPATH=${{ github.workspace }}/go
- name: go mod tidy
run: |
go mod tidy
git diff --exit-code
- name: golangci-lint
uses: golangci/golangci-lint-action@v2
with:
version: ${{ env.GOLANGCI_LINT_VERSION }}
working-directory: ${{ github.workspace }}/go/src/github.com/cooperspencer/gickup
+2 -1
View File
@@ -2,4 +2,5 @@ bla.conf
gickup.exe
gickup
conf.yml
.idea
.idea
dist/
+75
View File
@@ -0,0 +1,75 @@
linters:
enable-all: true
disable:
- cyclop # duplicate of gocyclo
- exhaustivestruct # not revelent
- gochecknoglobals
- gocognit # too strict
- goerr113 # too strict
- golint # deprecated
- gomnd # too strict
- interfacer # deprecated
- ireturn # not revelent
- maligned # deprecated
- scopelint # deprecated
- tagliatelle # breaking
- testpackage # too strict
- varnamelen # not revelent
- wrapcheck # too strict
- wsl # too strict
- maintidx # kind of duplicate of gocyclo
issues:
exclude-rules:
- path: main.go
text: bad syntax for struct tag pair
- path: main.go # FIXME
text: Function 'main' is too long
- path: (.+)_test.go
linters:
- typecheck
- path: metrics/prometheus/prometheus.go # FIXME
text: counter metrics should have "_total" suffix
- path: local/local.go # FIXME
text: has complex nested blocks
- path: local/local.go # FIXME
text: Function 'Locally' has too many statements
- path: gitlab/gitlab.go # FIXME
text: Function 'Backup' is too long
- path: gitlab/gitlab.go # FIXME
text: Function 'Get' has too many statements
- path: gitlab/gitlab.go # FIXME
text: cyclomatic complexity \d+ of func `Get` is high
- path: gitlab/gitlab.go # FIXME
text: has complex nested blocks
- path: gogs/gogs.go # FIXME
text: has complex nested blocks
- path: gogs/gogs.go # FIXME
text: Function 'Backup' is too long
- path: gogs/gogs.go # FIXME
text: Function 'Get' has too many statements
- path: bitbucket/bitbucket.go # FIXME
text: Function 'Get' is too long
- path: bitbucket/bitbucket.go # FIXME
text: type assertion must be checked
- path: gitea/gitea.go
text: .code.gitea.io/sdk/gitea.ListOrgReposOptions. composite literal uses unkeyed fields
- path: gitea/gitea.go # FIXME
text: has complex nested blocks
- path: gitea/gitea.go # FIXME
text: Function 'Backup' is too long
- path: gitea/gitea.go # FIXME
text: Function 'Get' has too many statements
- path: gitea/gitea.go # FIXME
text: cyclomatic complexity \d+ of func `Get` is high
- path: gitea/gitea.go # FIXME
text: lines are duplicate of `gitea/gitea.go:\d+-\d+`
- path: github/github.go # FIXME
text: Function 'Get' has too many statements
- path: github/github.go # FIXME
text: has complex nested blocks
linters-settings:
forbidigo:
forbid:
- '^print(ln)?$'
+24
View File
@@ -0,0 +1,24 @@
test:
go test ./...
.PHONY: test
dist:
mkdir -p dist
dist/gickup: dist
go build -o dist/gickup ./main.go
build: dist/gickup
.PHONY: build
clean:
$(RM) -r dist
.PHONY: clean
install-tools:
go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.45.0
.PHONY: install-tools
lint:
golangci-lint run
.PHONY: lint
+45 -10
View File
@@ -1,13 +1,14 @@
package bitbucket
import (
"gickup/types"
"net/url"
"github.com/cooperspencer/gickup/types"
"github.com/ktrysmt/go-bitbucket"
"github.com/rs/zerolog/log"
)
// Get TODO.
func Get(conf *types.Conf) []types.Repo {
repos := []types.Repo{}
for _, repo := range conf.Source.BitBucket {
@@ -15,20 +16,31 @@ func Get(conf *types.Conf) []types.Repo {
if repo.User == "" {
repo.User = repo.Username
}
if repo.Url == "" {
repo.Url = bitbucket.DEFAULT_BITBUCKET_API_BASE_URL
if repo.URL == "" {
repo.URL = bitbucket.DEFAULT_BITBUCKET_API_BASE_URL
} else {
bitbucketUrl, err := url.Parse(repo.Url)
bitbucketURL, err := url.Parse(repo.URL)
if err != nil {
log.Fatal().Str("stage", "bitbucket").Str("url", repo.Url).Msg(err.Error())
log.Fatal().
Str("stage", "bitbucket").
Str("url", repo.URL).
Msg(err.Error())
}
client.SetApiBaseURL(*bitbucketUrl)
client.SetApiBaseURL(*bitbucketURL)
}
log.Info().Str("stage", "bitbucket").Str("url", repo.Url).Msgf("grabbing repositories from %s", repo.User)
log.Info().
Str("stage", "bitbucket").
Str("url", repo.URL).
Msgf("grabbing repositories from %s", repo.User)
repositories, err := client.Repositories.ListForAccount(&bitbucket.RepositoriesOptions{Owner: repo.User})
if err != nil {
log.Fatal().Str("stage", "bitbucket").Str("url", repo.Url).Msg(err.Error())
log.Fatal().
Str("stage", "bitbucket").
Str("url", repo.URL).
Msg(err.Error())
}
include := types.GetMap(repo.Include)
@@ -41,17 +53,40 @@ func Get(conf *types.Conf) []types.Repo {
user = r.Owner["nickname"].(string)
}
}
if include[r.Name] {
repos = append(repos, types.Repo{Name: r.Name, Url: r.Links["clone"].([]interface{})[0].(map[string]interface{})["href"].(string), SshUrl: r.Links["clone"].([]interface{})[1].(map[string]interface{})["href"].(string), Token: "", Defaultbranch: r.Mainbranch.Name, Origin: repo, Owner: user, Hoster: types.GetHost(repo.Url)})
repos = append(repos, types.Repo{
Name: r.Name,
URL: r.Links["clone"].([]interface{})[0].(map[string]interface{})["href"].(string),
SSHURL: r.Links["clone"].([]interface{})[1].(map[string]interface{})["href"].(string),
Token: "",
Defaultbranch: r.Mainbranch.Name,
Origin: repo,
Owner: user,
Hoster: types.GetHost(repo.URL),
})
continue
}
if exclude[r.Name] {
continue
}
if len(include) == 0 {
repos = append(repos, types.Repo{Name: r.Name, Url: r.Links["clone"].([]interface{})[0].(map[string]interface{})["href"].(string), SshUrl: r.Links["clone"].([]interface{})[1].(map[string]interface{})["href"].(string), Token: "", Defaultbranch: r.Mainbranch.Name, Origin: repo, Owner: user, Hoster: types.GetHost(repo.Url)})
repos = append(repos, types.Repo{
Name: r.Name,
URL: r.Links["clone"].([]interface{})[0].(map[string]interface{})["href"].(string),
SSHURL: r.Links["clone"].([]interface{})[1].(map[string]interface{})["href"].(string),
Token: "",
Defaultbranch: r.Mainbranch.Name,
Origin: repo,
Owner: user,
Hoster: types.GetHost(repo.URL),
})
}
}
}
return repos
}
+197 -48
View File
@@ -2,75 +2,135 @@ package gitea
import (
"code.gitea.io/sdk/gitea"
"gickup/types"
"github.com/cooperspencer/gickup/types"
"github.com/rs/zerolog/log"
)
// Backup TODO.
func Backup(r types.Repo, d types.GenRepo, dry bool) {
if d.Url == "" {
d.Url = "https://gitea.com/"
if d.URL == "" {
d.URL = "https://gitea.com/"
}
log.Info().Str("stage", "gitea").Str("url", d.Url).Msgf("mirroring %s to %s", types.Blue(r.Name), d.Url)
giteaclient, err := gitea.NewClient(d.Url)
log.Info().
Str("stage", "gitea").
Str("url", d.URL).
Msgf("mirroring %s to %s", types.Blue(r.Name), d.URL)
giteaclient, err := gitea.NewClient(d.URL)
if err != nil {
log.Fatal().Str("stage", "gitea").Str("url", d.Url).Msg(err.Error())
log.Fatal().Str("stage", "gitea").Str("url", d.URL).Msg(err.Error())
}
giteaclient.SetBasicAuth(d.GetToken(), "")
user, _, err := giteaclient.GetMyUserInfo()
if err != nil {
log.Fatal().Str("stage", "gitea").Str("url", d.Url).Msg(err.Error())
log.Fatal().
Str("stage", "gitea").
Str("url", d.URL).
Msg(err.Error())
}
if !dry {
repo, _, err := giteaclient.GetRepo(user.UserName, r.Name)
if err != nil {
opts := gitea.MigrateRepoOption{RepoName: r.Name, RepoOwner: user.UserName, Mirror: true, CloneAddr: r.Url, AuthToken: r.Token}
if r.Token == "" {
opts = gitea.MigrateRepoOption{RepoName: r.Name, RepoOwner: user.UserName, Mirror: true, CloneAddr: r.Url, AuthUsername: r.Origin.User, AuthPassword: r.Origin.Password}
}
_, _, err := giteaclient.MigrateRepo(opts)
if err != nil {
log.Fatal().Str("stage", "gitea").Str("url", d.Url).Msg(err.Error())
}
log.Info().Str("stage", "gitea").Str("url", d.Url).Msgf("mirrored %s to %s", types.Blue(r.Name), d.Url)
} else {
if repo.Mirror {
log.Info().Str("stage", "gitea").Str("url", d.Url).Msgf("mirror of %s already exists, syncing instead", types.Blue(r.Name))
_, err := giteaclient.MirrorSync(user.UserName, repo.Name)
if err != nil {
log.Fatal().Str("stage", "gitea").Str("url", d.Url).Msg(err.Error())
}
log.Info().Str("stage", "gitea").Str("url", d.Url).Msgf("successfully synced %s.", types.Blue(r.Name))
if dry {
return
}
repo, _, err := giteaclient.GetRepo(user.UserName, r.Name)
if err != nil {
opts := gitea.MigrateRepoOption{
RepoName: r.Name,
RepoOwner: user.UserName,
Mirror: true,
CloneAddr: r.URL,
AuthToken: r.Token,
}
if r.Token == "" {
opts = gitea.MigrateRepoOption{
RepoName: r.Name,
RepoOwner: user.UserName,
Mirror: true,
CloneAddr: r.URL,
AuthUsername: r.Origin.User,
AuthPassword: r.Origin.Password,
}
}
_, _, err := giteaclient.MigrateRepo(opts)
if err != nil {
log.Fatal().
Str("stage", "gitea").
Str("url", d.URL).
Msg(err.Error())
}
log.Info().
Str("stage", "gitea").
Str("url", d.URL).
Msgf("mirrored %s to %s", types.Blue(r.Name), d.URL)
return
}
if repo.Mirror {
log.Info().
Str("stage", "gitea").
Str("url", d.URL).
Msgf("mirror of %s already exists, syncing instead", types.Blue(r.Name))
_, err := giteaclient.MirrorSync(user.UserName, repo.Name)
if err != nil {
log.Fatal().
Str("stage", "gitea").
Str("url", d.URL).
Msg(err.Error())
}
log.Info().
Str("stage", "gitea").
Str("url", d.URL).
Msgf("successfully synced %s.", types.Blue(r.Name))
}
}
// Get TODO.
func Get(conf *types.Conf) []types.Repo {
repos := []types.Repo{}
for _, repo := range conf.Source.Gitea {
if repo.Url == "" {
repo.Url = "https://gitea.com"
if repo.URL == "" {
repo.URL = "https://gitea.com"
}
log.Info().Str("stage", "gitea").Str("url", repo.Url).Msgf("grabbing repositories from %s", repo.User)
log.Info().
Str("stage", "gitea").
Str("url", repo.URL).
Msgf("grabbing repositories from %s", repo.User)
opt := gitea.ListReposOptions{}
opt.PageSize = 50
opt.Page = 1
gitearepos := []*gitea.Repository{}
client := &gitea.Client{}
var client *gitea.Client
var err error
token := repo.GetToken()
if token != "" {
client, err = gitea.NewClient(repo.Url, gitea.SetToken(token))
client, err = gitea.NewClient(repo.URL, gitea.SetToken(token))
} else {
client, err = gitea.NewClient(repo.Url)
client, err = gitea.NewClient(repo.URL)
}
for {
if err != nil {
log.Fatal().Str("stage", "gitea").Str("url", repo.Url).Msg(err.Error())
log.Fatal().
Str("stage", "gitea").
Str("url", repo.URL).
Msg(err.Error())
}
repos, _, err := client.ListUserRepos(repo.User, opt)
if err != nil {
log.Fatal().Str("stage", "gitea").Str("url", repo.Url).Msg(err.Error())
log.Fatal().
Str("stage", "gitea").
Str("url", repo.URL).
Msg(err.Error())
}
if len(repos) == 0 {
break
@@ -82,7 +142,10 @@ func Get(conf *types.Conf) []types.Repo {
if repo.Starred {
starredrepos, _, err := client.GetMyStarredRepos()
if err != nil {
log.Fatal().Str("stage", "gitea").Str("url", repo.Url).Msg(err.Error())
log.Fatal().
Str("stage", "gitea").
Str("url", repo.URL).
Msg(err.Error())
}
gitearepos = append(gitearepos, starredrepos...)
}
@@ -94,19 +157,58 @@ func Get(conf *types.Conf) []types.Repo {
for _, r := range gitearepos {
if include[r.Name] {
repos = append(repos, types.Repo{Name: r.Name, Url: r.CloneURL, SshUrl: r.SSHURL, Token: token, Defaultbranch: r.DefaultBranch, Origin: repo, Owner: r.Owner.UserName, Hoster: types.GetHost(repo.Url)})
repos = append(repos, types.Repo{
Name: r.Name,
URL: r.CloneURL,
SSHURL: r.SSHURL,
Token: token,
Defaultbranch: r.DefaultBranch,
Origin: repo,
Owner: r.Owner.UserName,
Hoster: types.GetHost(repo.URL),
})
if r.HasWiki && repo.Wiki && types.StatRemote(r.CloneURL, r.SSHURL, repo) {
repos = append(repos, types.Repo{Name: r.Name + ".wiki", Url: types.DotGitRx.ReplaceAllString(r.CloneURL, ".wiki.git"), SshUrl: types.DotGitRx.ReplaceAllString(r.SSHURL, ".wiki.git"), Token: token, Defaultbranch: r.DefaultBranch, Origin: repo, Owner: r.Owner.UserName, Hoster: types.GetHost(repo.Url)})
repos = append(repos, types.Repo{
Name: r.Name + ".wiki",
URL: types.DotGitRx.ReplaceAllString(r.CloneURL, ".wiki.git"),
SSHURL: types.DotGitRx.ReplaceAllString(r.SSHURL, ".wiki.git"),
Token: token,
Defaultbranch: r.DefaultBranch,
Origin: repo,
Owner: r.Owner.UserName,
Hoster: types.GetHost(repo.URL),
})
}
continue
}
if exclude[r.Name] {
continue
}
if len(repo.Include) == 0 {
repos = append(repos, types.Repo{Name: r.Name, Url: r.CloneURL, SshUrl: r.SSHURL, Token: token, Defaultbranch: r.DefaultBranch, Origin: repo, Owner: r.Owner.UserName, Hoster: types.GetHost(repo.Url)})
repos = append(repos, types.Repo{
Name: r.Name,
URL: r.CloneURL,
SSHURL: r.SSHURL,
Token: token,
Defaultbranch: r.DefaultBranch,
Origin: repo,
Owner: r.Owner.UserName,
Hoster: types.GetHost(repo.URL),
})
if r.HasWiki && repo.Wiki && types.StatRemote(r.CloneURL, r.SSHURL, repo) {
repos = append(repos, types.Repo{Name: r.Name + ".wiki", Url: types.DotGitRx.ReplaceAllString(r.CloneURL, ".wiki.git"), SshUrl: types.DotGitRx.ReplaceAllString(r.SSHURL, ".wiki.git"), Token: token, Defaultbranch: r.DefaultBranch, Origin: repo, Owner: r.Owner.UserName, Hoster: types.GetHost(repo.Url)})
repos = append(repos, types.Repo{
Name: r.Name + ".wiki",
URL: types.DotGitRx.ReplaceAllString(r.CloneURL, ".wiki.git"),
SSHURL: types.DotGitRx.ReplaceAllString(r.SSHURL, ".wiki.git"),
Token: token,
Defaultbranch: r.DefaultBranch,
Origin: repo,
Owner: r.Owner.UserName,
Hoster: types.GetHost(repo.URL),
})
}
}
}
@@ -115,7 +217,10 @@ func Get(conf *types.Conf) []types.Repo {
for {
o, _, err := client.ListUserOrgs(repo.User, gitea.ListOrgsOptions{ListOptions: orgopt})
if err != nil {
log.Fatal().Str("stage", "gitea").Str("url", repo.Url).Msg(err.Error())
log.Fatal().
Str("stage", "gitea").
Str("url", repo.URL).
Msg(err.Error())
}
if len(o) == 0 {
break
@@ -151,30 +256,74 @@ func Get(conf *types.Conf) []types.Repo {
}
for _, r := range orgrepos {
if include[r.Name] {
repos = append(repos, types.Repo{Name: r.Name, Url: r.CloneURL, SshUrl: r.SSHURL, Token: token, Defaultbranch: r.DefaultBranch, Origin: repo, Owner: r.Owner.UserName, Hoster: types.GetHost(repo.Url)})
repos = append(repos, types.Repo{
Name: r.Name,
URL: r.CloneURL,
SSHURL: r.SSHURL,
Token: token,
Defaultbranch: r.DefaultBranch,
Origin: repo,
Owner: r.Owner.UserName,
Hoster: types.GetHost(repo.URL),
})
if r.HasWiki && repo.Wiki && types.StatRemote(r.CloneURL, r.SSHURL, repo) {
repos = append(repos, types.Repo{Name: r.Name + ".wiki", Url: types.DotGitRx.ReplaceAllString(r.CloneURL, ".wiki.git"), SshUrl: types.DotGitRx.ReplaceAllString(r.SSHURL, ".wiki.git"), Token: token, Defaultbranch: r.DefaultBranch, Origin: repo, Owner: r.Owner.UserName, Hoster: types.GetHost(repo.Url)})
repos = append(repos, types.Repo{
Name: r.Name + ".wiki",
URL: types.DotGitRx.ReplaceAllString(r.CloneURL, ".wiki.git"),
SSHURL: types.DotGitRx.ReplaceAllString(r.SSHURL, ".wiki.git"),
Token: token,
Defaultbranch: r.DefaultBranch,
Origin: repo,
Owner: r.Owner.UserName,
Hoster: types.GetHost(repo.URL),
})
}
continue
}
if exclude[r.Name] {
continue
}
if len(repo.Include) == 0 {
repos = append(repos, types.Repo{Name: r.Name, Url: r.CloneURL, SshUrl: r.SSHURL, Token: token, Defaultbranch: r.DefaultBranch, Origin: repo, Owner: r.Owner.UserName, Hoster: types.GetHost(repo.Url)})
repos = append(repos, types.Repo{
Name: r.Name,
URL: r.CloneURL,
SSHURL: r.SSHURL,
Token: token,
Defaultbranch: r.DefaultBranch,
Origin: repo,
Owner: r.Owner.UserName,
Hoster: types.GetHost(repo.URL),
})
if r.HasWiki && repo.Wiki && types.StatRemote(r.CloneURL, r.SSHURL, repo) {
repos = append(repos, types.Repo{Name: r.Name + ".wiki", Url: types.DotGitRx.ReplaceAllString(r.CloneURL, ".wiki.git"), SshUrl: types.DotGitRx.ReplaceAllString(r.SSHURL, ".wiki.git"), Token: token, Defaultbranch: r.DefaultBranch, Origin: repo, Owner: r.Owner.UserName, Hoster: types.GetHost(repo.Url)})
repos = append(repos, types.Repo{
Name: r.Name + ".wiki",
URL: types.DotGitRx.ReplaceAllString(r.CloneURL, ".wiki.git"),
SSHURL: types.DotGitRx.ReplaceAllString(r.SSHURL, ".wiki.git"),
Token: token,
Defaultbranch: r.DefaultBranch,
Origin: repo,
Owner: r.Owner.UserName,
Hoster: types.GetHost(repo.URL),
})
}
}
}
}
return repos
}
func getOrgRepos(client *gitea.Client, org *gitea.Organization, orgopt gitea.ListOptions, repo types.GenRepo) []*gitea.Repository {
o, _, err := client.ListOrgRepos(org.UserName, gitea.ListOrgReposOptions{orgopt})
func getOrgRepos(client *gitea.Client, org *gitea.Organization,
orgopt gitea.ListOptions, repo types.GenRepo,
) []*gitea.Repository {
o, _, err := client.ListOrgRepos(org.UserName,
gitea.ListOrgReposOptions{orgopt})
if err != nil {
log.Fatal().Str("stage", "gitea").Str("url", repo.Url).Msg(err.Error())
log.Fatal().Str("stage", "gitea").Str("url", repo.URL).Msg(err.Error())
}
return o
}
+88 -14
View File
@@ -2,28 +2,51 @@ package github
import (
"context"
"gickup/types"
"github.com/cooperspencer/gickup/types"
"github.com/google/go-github/v41/github"
"github.com/rs/zerolog/log"
"golang.org/x/oauth2"
)
func addWiki(r github.Repository, repo types.GenRepo, token string) types.Repo {
if r.GetHasWiki() && repo.Wiki && types.StatRemote(r.GetCloneURL(), r.GetSSHURL(), repo) {
return types.Repo{Name: *r.Name + ".wiki", Url: types.DotGitRx.ReplaceAllString(r.GetCloneURL(), ".wiki.git"), SshUrl: types.DotGitRx.ReplaceAllString(r.GetSSHURL(), ".wiki.git"), Token: token, Defaultbranch: r.GetDefaultBranch(), Origin: repo, Owner: r.GetOwner().GetLogin(), Hoster: "github.com"}
if !(r.GetHasWiki() && repo.Wiki &&
types.StatRemote(r.GetCloneURL(), r.GetSSHURL(), repo)) {
return types.Repo{}
}
return types.Repo{
Name: *r.Name + ".wiki",
URL: types.DotGitRx.ReplaceAllString(r.GetCloneURL(), ".wiki.git"),
SSHURL: types.DotGitRx.ReplaceAllString(r.GetSSHURL(), ".wiki.git"),
Token: token,
Defaultbranch: r.GetDefaultBranch(),
Origin: repo,
Owner: r.GetOwner().GetLogin(),
Hoster: "github.com",
}
return types.Repo{}
}
// Get TODO.
func Get(conf *types.Conf) []types.Repo {
repos := []types.Repo{}
for _, repo := range conf.Source.Github {
log.Info().Str("stage", "github").Str("url", "https://github.com").Msgf("grabbing the repositories from %s", repo.User)
client := &github.Client{}
opt := &github.RepositoryListOptions{ListOptions: github.ListOptions{PerPage: 50}}
log.Info().
Str("stage", "github").
Str("url", "https://github.com").
Msgf("grabbing the repositories from %s", repo.User)
opt := &github.RepositoryListOptions{
ListOptions: github.ListOptions{
PerPage: 50,
},
}
i := 1
githubrepos := []*github.Repository{}
token := repo.GetToken()
var client *github.Client
if token == "" {
client = github.NewClient(nil)
} else {
@@ -31,23 +54,34 @@ func Get(conf *types.Conf) []types.Repo {
&oauth2.Token{AccessToken: token},
)
tc := oauth2.NewClient(context.TODO(), ts)
client = github.NewClient(tc)
}
if token != "" {
user, _, err := client.Users.Get(context.TODO(), "")
if err != nil {
log.Fatal().Str("stage", "github").Str("url", "https://github.com").Msg(err.Error())
log.Fatal().
Str("stage", "github").
Str("url", "https://github.com").
Msg(err.Error())
}
if repo.User == user.GetLogin() {
repo.User = ""
}
}
for {
opt.Page = i
repos, _, err := client.Repositories.List(context.TODO(), repo.User, opt)
if err != nil {
log.Fatal().Str("stage", "github").Str("url", "https://github.com").Msg(err.Error())
log.Fatal().
Str("stage", "github").
Str("url", "https://github.com").
Msg(err.Error())
}
if len(repos) == 0 {
break
}
@@ -57,12 +91,20 @@ func Get(conf *types.Conf) []types.Repo {
if repo.Starred {
i = 1
opt := &github.ActivityListStarredOptions{ListOptions: github.ListOptions{PerPage: 50}}
opt := &github.ActivityListStarredOptions{
ListOptions: github.ListOptions{
PerPage: 50,
},
}
for {
opt.ListOptions.Page = i
repos, _, err := client.Activity.ListStarred(context.TODO(), repo.User, opt)
if err != nil {
log.Fatal().Str("stage", "github").Str("url", "https://github.com").Msg(err.Error())
log.Fatal().
Str("stage", "github").
Str("url", "https://github.com").
Msg(err.Error())
}
if len(repos) == 0 {
break
@@ -81,30 +123,61 @@ func Get(conf *types.Conf) []types.Repo {
for _, r := range githubrepos {
if include[*r.Name] {
repos = append(repos, types.Repo{Name: r.GetName(), Url: r.GetCloneURL(), SshUrl: r.GetSSHURL(), Token: token, Defaultbranch: r.GetDefaultBranch(), Origin: repo, Owner: r.GetOwner().GetLogin(), Hoster: "github.com"})
repos = append(repos, types.Repo{
Name: r.GetName(),
URL: r.GetCloneURL(),
SSHURL: r.GetSSHURL(),
Token: token,
Defaultbranch: r.GetDefaultBranch(),
Origin: repo,
Owner: r.GetOwner().GetLogin(),
Hoster: "github.com",
})
wiki := addWiki(*r, repo, token)
if wiki.Name != "" {
repos = append(repos, wiki)
}
continue
}
if exclude[*r.Name] {
continue
}
if excludeorgs[r.GetOwner().GetLogin()] {
continue
}
if len(repo.Include) == 0 {
if len(includeorgs) > 0 {
if includeorgs[r.GetOwner().GetLogin()] {
repos = append(repos, types.Repo{Name: r.GetName(), Url: r.GetCloneURL(), SshUrl: r.GetSSHURL(), Token: token, Defaultbranch: r.GetDefaultBranch(), Origin: repo, Owner: r.GetOwner().GetLogin(), Hoster: "github.com"})
repos = append(repos, types.Repo{
Name: r.GetName(),
URL: r.GetCloneURL(),
SSHURL: r.GetSSHURL(),
Token: token,
Defaultbranch: r.GetDefaultBranch(),
Origin: repo,
Owner: r.GetOwner().GetLogin(),
Hoster: "github.com",
})
wiki := addWiki(*r, repo, token)
if wiki.Name != "" {
repos = append(repos, wiki)
}
}
} else {
repos = append(repos, types.Repo{Name: r.GetName(), Url: r.GetCloneURL(), SshUrl: r.GetSSHURL(), Token: token, Defaultbranch: r.GetDefaultBranch(), Origin: repo, Owner: r.GetOwner().GetLogin(), Hoster: "github.com"})
repos = append(repos, types.Repo{
Name: r.GetName(),
URL: r.GetCloneURL(),
SSHURL: r.GetSSHURL(),
Token: token,
Defaultbranch: r.GetDefaultBranch(),
Origin: repo,
Owner: r.GetOwner().GetLogin(),
Hoster: "github.com",
})
wiki := addWiki(*r, repo, token)
if wiki.Name != "" {
repos = append(repos, wiki)
@@ -113,5 +186,6 @@ func Get(conf *types.Conf) []types.Repo {
}
}
}
return repos
}
+190 -60
View File
@@ -2,34 +2,48 @@ package gitlab
import (
"fmt"
"gickup/types"
"path"
"strings"
"github.com/cooperspencer/gickup/types"
"github.com/rs/zerolog/log"
"github.com/xanzy/go-gitlab"
)
// Backup TODO.
func Backup(r types.Repo, d types.GenRepo, dry bool) {
gitlabclient := &gitlab.Client{}
var gitlabclient *gitlab.Client
token := d.GetToken()
var err error
if d.Url == "" {
d.Url = "https://gitlab.com"
if d.URL == "" {
d.URL = "https://gitlab.com"
gitlabclient, err = gitlab.NewClient(token)
} else {
gitlabclient, err = gitlab.NewClient(token, gitlab.WithBaseURL(d.Url))
}
log.Info().Str("stage", "gitlab").Str("url", d.Url).Msgf("mirroring %s to %s", types.Blue(r.Name), d.Url)
if err != nil {
log.Fatal().Str("stage", "gitlab").Str("url", d.Url).Msg(err.Error())
gitlabclient, err = gitlab.NewClient(token, gitlab.WithBaseURL(d.URL))
}
if err != nil {
log.Fatal().
Str("stage", "gitlab").
Str("url", d.URL).
Msg(err.Error())
}
log.Info().
Str("stage", "gitlab").
Str("url", d.URL).
Msgf("mirroring %s to %s", types.Blue(r.Name), d.URL)
True := true
opt := gitlab.ListProjectsOptions{Search: &r.Name, Owned: &True}
opt := gitlab.ListProjectsOptions{
Search: &r.Name,
Owned: &True,
}
projects, _, err := gitlabclient.Projects.ListProjects(&opt)
if err != nil {
log.Fatal().Str("stage", "gitlab").Str("url", d.Url).Msg(err.Error())
log.Fatal().Str("stage", "gitlab").Str("url", d.URL).Msg(err.Error())
}
found := false
@@ -39,42 +53,62 @@ func Backup(r types.Repo, d types.GenRepo, dry bool) {
}
}
if !dry {
if !found {
if r.Token != "" {
splittedurl := strings.Split(r.Url, "//")
r.Url = fmt.Sprintf("%s//%s@%s", splittedurl[0], r.Token, splittedurl[1])
if r.Token == "" {
r.Url = fmt.Sprintf("%s//%s:%s@%s", splittedurl[0], r.Origin.User, r.Origin.Password, splittedurl[1])
}
}
opts := &gitlab.CreateProjectOptions{Mirror: &True, ImportURL: &r.Url, Name: &r.Name}
_, _, err := gitlabclient.Projects.CreateProject(opts)
if err != nil {
log.Fatal().Str("stage", "gitlab").Str("url", d.Url).Msg(err.Error())
}
}
if dry || found {
return
}
if r.Token != "" {
splittedurl := strings.Split(r.URL, "//")
r.URL = fmt.Sprintf("%s//%s:%s@%s",
splittedurl[0], r.Origin.User, r.Origin.Password, splittedurl[1])
}
opts := &gitlab.CreateProjectOptions{
Mirror: &True,
ImportURL: &r.URL,
Name: &r.Name,
}
_, _, err = gitlabclient.Projects.CreateProject(opts)
if err != nil {
log.Fatal().
Str("stage", "gitlab").
Str("url", d.URL).
Msg(err.Error())
}
}
// Get TODO.
func Get(conf *types.Conf) []types.Repo {
repos := []types.Repo{}
for _, repo := range conf.Source.Gitlab {
if repo.Url == "" {
repo.Url = "https://gitlab.com"
if repo.URL == "" {
repo.URL = "https://gitlab.com"
}
log.Info().Str("stage", "gitlab").Str("url", repo.Url).Msgf("grabbing repositories from %s", repo.User)
log.Info().
Str("stage", "gitlab").
Str("url", repo.URL).
Msgf("grabbing repositories from %s", repo.User)
gitlabrepos := []*gitlab.Project{}
gitlabgrouprepos := map[string][]*gitlab.Project{}
token := repo.GetToken()
client, err := gitlab.NewClient(token, gitlab.WithBaseURL(repo.Url))
client, err := gitlab.NewClient(token, gitlab.WithBaseURL(repo.URL))
if err != nil {
log.Fatal().Str("stage", "gitlab").Str("url", repo.Url).Msg(err.Error())
log.Fatal().
Str("stage", "gitlab").
Str("url", repo.URL).
Msg(err.Error())
}
opt := &gitlab.ListProjectsOptions{}
users, _, err := client.Users.ListUsers(&gitlab.ListUsersOptions{Username: &repo.User})
if err != nil {
log.Fatal().Str("stage", "gitlab").Str("url", repo.Url).Msg(err.Error())
log.Fatal().
Str("stage", "gitlab").
Str("url", repo.URL).
Msg(err.Error())
}
opt.PerPage = 50
@@ -85,7 +119,10 @@ func Get(conf *types.Conf) []types.Repo {
opt.Page = i
projects, _, err := client.Projects.ListUserProjects(user.ID, opt)
if err != nil {
log.Fatal().Str("stage", "gitlab").Str("url", repo.Url).Msg(err.Error())
log.Fatal().
Str("stage", "gitlab").
Str("url", repo.URL).
Msg(err.Error())
}
if len(projects) == 0 {
break
@@ -104,7 +141,10 @@ func Get(conf *types.Conf) []types.Repo {
opt.Page = i
projects, _, err := client.Projects.ListUserStarredProjects(user.ID, opt)
if err != nil {
log.Fatal().Str("stage", "gitlab").Str("url", repo.Url).Msg(err.Error())
log.Fatal().
Str("stage", "gitlab").
Str("url", repo.URL).
Msg(err.Error())
}
if len(projects) == 0 {
break
@@ -124,14 +164,32 @@ func Get(conf *types.Conf) []types.Repo {
for _, r := range gitlabrepos {
if include[r.Name] {
if r.RepositoryAccessLevel != gitlab.DisabledAccessControl {
repos = append(repos, types.Repo{Name: r.Path, Url: r.HTTPURLToRepo, SshUrl: r.SSHURLToRepo, Token: token, Defaultbranch: r.DefaultBranch, Origin: repo, Owner: r.Namespace.FullPath, Hoster: types.GetHost(repo.Url)})
repos = append(repos, types.Repo{
Name: r.Path,
URL: r.HTTPURLToRepo,
SSHURL: r.SSHURLToRepo,
Token: token,
Defaultbranch: r.DefaultBranch,
Origin: repo,
Owner: r.Namespace.FullPath,
Hoster: types.GetHost(repo.URL),
})
}
if r.WikiEnabled && repo.Wiki {
if activeWiki(r, client, repo) {
httpUrlToRepo := types.DotGitRx.ReplaceAllString(r.HTTPURLToRepo, ".wiki.git")
sshUrlToRepo := types.DotGitRx.ReplaceAllString(r.SSHURLToRepo, ".wiki.git")
repos = append(repos, types.Repo{Name: r.Path + ".wiki", Url: httpUrlToRepo, SshUrl: sshUrlToRepo, Token: token, Defaultbranch: r.DefaultBranch, Origin: repo, Owner: r.Namespace.FullPath, Hoster: types.GetHost(repo.Url)})
httpURLToRepo := types.DotGitRx.ReplaceAllString(r.HTTPURLToRepo, ".wiki.git")
sshURLToRepo := types.DotGitRx.ReplaceAllString(r.SSHURLToRepo, ".wiki.git")
repos = append(repos, types.Repo{
Name: r.Path + ".wiki",
URL: httpURLToRepo,
SSHURL: sshURLToRepo,
Token: token,
Defaultbranch: r.DefaultBranch,
Origin: repo,
Owner: r.Namespace.FullPath,
Hoster: types.GetHost(repo.URL),
})
}
}
@@ -142,29 +200,57 @@ func Get(conf *types.Conf) []types.Repo {
}
if len(include) == 0 {
if r.RepositoryAccessLevel != gitlab.DisabledAccessControl {
repos = append(repos, types.Repo{Name: r.Path, Url: r.HTTPURLToRepo, SshUrl: r.SSHURLToRepo, Token: token, Defaultbranch: r.DefaultBranch, Origin: repo, Owner: r.Namespace.FullPath, Hoster: types.GetHost(repo.Url)})
repos = append(repos, types.Repo{
Name: r.Path,
URL: r.HTTPURLToRepo,
SSHURL: r.SSHURLToRepo,
Token: token,
Defaultbranch: r.DefaultBranch,
Origin: repo,
Owner: r.Namespace.FullPath,
Hoster: types.GetHost(repo.URL),
})
}
if r.WikiEnabled && repo.Wiki {
if activeWiki(r, client, repo) {
httpUrlToRepo := types.DotGitRx.ReplaceAllString(r.HTTPURLToRepo, ".wiki.git")
sshUrlToRepo := types.DotGitRx.ReplaceAllString(r.SSHURLToRepo, ".wiki.git")
repos = append(repos, types.Repo{Name: r.Path + ".wiki", Url: httpUrlToRepo, SshUrl: sshUrlToRepo, Token: token, Defaultbranch: r.DefaultBranch, Origin: repo, Owner: r.Namespace.FullPath, Hoster: types.GetHost(repo.Url)})
httpURLToRepo := types.DotGitRx.ReplaceAllString(r.HTTPURLToRepo, ".wiki.git")
sshURLToRepo := types.DotGitRx.ReplaceAllString(r.SSHURLToRepo, ".wiki.git")
repos = append(repos, types.Repo{
Name: r.Path + ".wiki",
URL: httpURLToRepo,
SSHURL: sshURLToRepo,
Token: token,
Defaultbranch: r.DefaultBranch,
Origin: repo,
Owner: r.Namespace.FullPath,
Hoster: types.GetHost(repo.URL),
})
}
}
}
}
if token != "" {
groups := []*gitlab.Group{}
i := 1
for {
g, _, err := client.Groups.ListGroups(&gitlab.ListGroupsOptions{ListOptions: gitlab.ListOptions{Page: i, PerPage: 50}})
g, _, err := client.Groups.ListGroups(&gitlab.ListGroupsOptions{
ListOptions: gitlab.ListOptions{
Page: i,
PerPage: 50,
},
})
if err != nil {
log.Fatal().Str("stage", "gitlab").Str("url", repo.Url).Msg(err.Error())
log.Fatal().
Str("stage", "gitlab").
Str("url", repo.URL).Msg(err.Error())
}
if len(g) == 0 {
break
}
groups = append(groups, g...)
i++
}
@@ -177,7 +263,10 @@ func Get(conf *types.Conf) []types.Repo {
for {
projects, _, err := client.Groups.ListGroupProjects(group.ID, gopt)
if err != nil {
log.Fatal().Str("stage", "gitlab").Str("url", repo.Url).Msg(err.Error())
log.Fatal().
Str("stage", "gitlab").
Str("url", repo.URL).
Msg(err.Error())
}
if len(projects) == 0 {
break
@@ -197,16 +286,35 @@ func Get(conf *types.Conf) []types.Repo {
for _, r := range gr {
if include[r.Name] {
if r.RepositoryAccessLevel != gitlab.DisabledAccessControl {
repos = append(repos, types.Repo{Name: r.Path, Url: r.HTTPURLToRepo, SshUrl: r.SSHURLToRepo, Token: token, Defaultbranch: r.DefaultBranch, Origin: repo, Owner: k, Hoster: types.GetHost(repo.Url)})
repos = append(repos, types.Repo{
Name: r.Path,
URL: r.HTTPURLToRepo,
SSHURL: r.SSHURLToRepo,
Token: token,
Defaultbranch: r.DefaultBranch,
Origin: repo,
Owner: k,
Hoster: types.GetHost(repo.URL),
})
}
if r.WikiEnabled && repo.Wiki {
if activeWiki(r, client, repo) {
httpUrlToRepo := types.DotGitRx.ReplaceAllString(r.HTTPURLToRepo, ".wiki.git")
sshUrlToRepo := types.DotGitRx.ReplaceAllString(r.SSHURLToRepo, ".wiki.git")
repos = append(repos, types.Repo{Name: r.Path + ".wiki", Url: httpUrlToRepo, SshUrl: sshUrlToRepo, Token: token, Defaultbranch: r.DefaultBranch, Origin: repo, Owner: k, Hoster: types.GetHost(repo.Url)})
httpURLToRepo := types.DotGitRx.ReplaceAllString(r.HTTPURLToRepo, ".wiki.git")
sshURLToRepo := types.DotGitRx.ReplaceAllString(r.SSHURLToRepo, ".wiki.git")
repos = append(repos, types.Repo{
Name: r.Path + ".wiki",
URL: httpURLToRepo,
SSHURL: sshURLToRepo,
Token: token,
Defaultbranch: r.DefaultBranch,
Origin: repo,
Owner: k,
Hoster: types.GetHost(repo.URL),
})
}
}
continue
}
if exclude[r.Name] {
@@ -218,33 +326,55 @@ func Get(conf *types.Conf) []types.Repo {
if len(include) == 0 {
if len(includeorgs) == 0 || includeorgs[r.Namespace.FullPath] {
if r.RepositoryAccessLevel != gitlab.DisabledAccessControl {
repos = append(repos, types.Repo{Name: r.Path, Url: r.HTTPURLToRepo, SshUrl: r.SSHURLToRepo, Token: token, Defaultbranch: r.DefaultBranch, Origin: repo, Owner: k, Hoster: types.GetHost(repo.Url)})
repos = append(repos, types.Repo{
Name: r.Path,
URL: r.HTTPURLToRepo,
SSHURL: r.SSHURLToRepo,
Token: token,
Defaultbranch: r.DefaultBranch,
Origin: repo,
Owner: k,
Hoster: types.GetHost(repo.URL),
})
}
if r.WikiEnabled && repo.Wiki {
if activeWiki(r, client, repo) {
httpUrlToRepo := types.DotGitRx.ReplaceAllString(r.HTTPURLToRepo, ".wiki.git")
sshUrlToRepo := types.DotGitRx.ReplaceAllString(r.SSHURLToRepo, ".wiki.git")
repos = append(repos, types.Repo{Name: r.Path + ".wiki", Url: httpUrlToRepo, SshUrl: sshUrlToRepo, Token: token, Defaultbranch: r.DefaultBranch, Origin: repo, Owner: k, Hoster: types.GetHost(repo.Url)})
httpURLToRepo := types.DotGitRx.ReplaceAllString(r.HTTPURLToRepo, ".wiki.git")
sshURLToRepo := types.DotGitRx.ReplaceAllString(r.SSHURLToRepo, ".wiki.git")
repos = append(repos, types.Repo{
Name: r.Path + ".wiki",
URL: httpURLToRepo,
SSHURL: sshURLToRepo,
Token: token, Defaultbranch: r.DefaultBranch,
Origin: repo,
Owner: k,
Hoster: types.GetHost(repo.URL),
})
}
}
}
}
}
}
}
}
}
return repos
}
func activeWiki(r *gitlab.Project, client *gitlab.Client, repo types.GenRepo) bool {
wikilistoptions := &gitlab.ListWikisOptions{WithContent: gitlab.Bool(true)}
wikilistoptions := &gitlab.ListWikisOptions{
WithContent: gitlab.Bool(true),
}
wikis, _, err := client.Wikis.ListWikis(r.ID, wikilistoptions)
if err != nil {
log.Warn().Str("stage", "gitlab").Str("url", repo.Url).Msg(err.Error())
log.Warn().
Str("stage", "gitlab").
Str("url", repo.URL).
Msg(err.Error())
}
if len(wikis) > 0 {
return true
}
return false
return len(wikis) > 0
}
+1 -2
View File
@@ -1,4 +1,4 @@
module gickup
module github.com/cooperspencer/gickup
go 1.16
@@ -9,7 +9,6 @@ require (
github.com/alecthomas/kong v0.4.1
github.com/go-git/go-git/v5 v5.4.2
github.com/gogs/go-gogs-client v0.0.0-20210131175652-1d7215cd8d85
github.com/golang/protobuf v1.5.2 // indirect
github.com/google/go-github/v41 v41.0.0
github.com/gookit/color v1.5.0
github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
+2 -37
View File
@@ -31,8 +31,6 @@ cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohl
cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs=
cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
code.gitea.io/gitea-vet v0.2.1/go.mod h1:zcNbT/aJEmivCAhfmkHOlT645KNOf9W2KnkLgFjGGfE=
code.gitea.io/sdk/gitea v0.15.0 h1:tsNhxDM/2N1Ohv1Xq5UWrht/esg0WmtRj4wsHVHriTg=
code.gitea.io/sdk/gitea v0.15.0/go.mod h1:klY2LVI3s3NChzIk/MzMn7G1FHrfU7qd63iSMVoHRBA=
code.gitea.io/sdk/gitea v0.15.1 h1:WJreC7YYuxbn0UDaPuWIe/mtiNKTvLN8MLkaw71yx/M=
code.gitea.io/sdk/gitea v0.15.1/go.mod h1:klY2LVI3s3NChzIk/MzMn7G1FHrfU7qd63iSMVoHRBA=
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
@@ -42,21 +40,16 @@ github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym
github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA=
github.com/Microsoft/go-winio v0.4.16/go.mod h1:XB6nPKklQyQ7GC9LdcBEcBl8PF76WugXOPRXwdLnMv0=
github.com/Microsoft/go-winio v0.5.0/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84=
github.com/Microsoft/go-winio v0.5.1 h1:aPJp2QD7OOrhO5tQXqQoGSJc+DjDtWTGLOmNyAm6FgY=
github.com/Microsoft/go-winio v0.5.1/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84=
github.com/Microsoft/go-winio v0.5.2 h1:a9IhgEQBCUEk6QCdml9CiJGhAws+YwffDHEMp1VMrpA=
github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY=
github.com/ProtonMail/go-crypto v0.0.0-20210428141323-04723f9f07d7/go.mod h1:z4/9nQmJSSwwds7ejkxaJwO37dru3geImFUdJlaLzQo=
github.com/ProtonMail/go-crypto v0.0.0-20211112122917-428f8eabeeb3 h1:XcF0cTDJeiuZ5NU8w7WUDge0HRwwNRmxj/GGk6KSA6g=
github.com/ProtonMail/go-crypto v0.0.0-20211112122917-428f8eabeeb3/go.mod h1:z4/9nQmJSSwwds7ejkxaJwO37dru3geImFUdJlaLzQo=
github.com/ProtonMail/go-crypto v0.0.0-20220113124808-70ae35bab23f h1:J2FzIrXN82q5uyUraeJpLIm7U6PffRwje2ORho5yIik=
github.com/ProtonMail/go-crypto v0.0.0-20220113124808-70ae35bab23f/go.mod h1:z4/9nQmJSSwwds7ejkxaJwO37dru3geImFUdJlaLzQo=
github.com/acomagu/bufpipe v1.0.3 h1:fxAGrHZTgQ9w5QqVItgzwj235/uYZYgbXitB+dLupOk=
github.com/acomagu/bufpipe v1.0.3/go.mod h1:mxdxdup/WdsKVreO5GpW4+M/1CE2sMG4jeGJ2sYmHc4=
github.com/alecthomas/kong v0.2.19 h1:qBDfByO5XgWUXyNB4D6OOhGh5Z1eNOwWayDPQJFNWdc=
github.com/alecthomas/kong v0.2.19/go.mod h1:ka3VZ8GZNPXv9Ov+j4YNLkI8mTuhXyr/0ktSlqIydQQ=
github.com/alecthomas/kong v0.4.1 h1:0sFnMts+ijOiFuSHsMB9MlDi3NGINBkx9KIw1/gcuDw=
github.com/alecthomas/kong v0.4.1/go.mod h1:uzxf/HUh0tj43x1AyJROl3JT7SgsZ5m+icOv1csRhc0=
github.com/alecthomas/repr v0.0.0-20210801044451-80ca428c5142 h1:8Uy0oSf5co/NZXje7U1z8Mpep++QJOldL2hs/sBQf48=
github.com/alecthomas/repr v0.0.0-20210801044451-80ca428c5142/go.mod h1:2kn6fqh/zIyPLmm3ugklbEi5hg5wS435eygvNfaDQL8=
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
@@ -72,7 +65,6 @@ github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+Ce
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY=
github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE=
github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
@@ -163,7 +155,6 @@ github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ=
github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-github/v41 v41.0.0 h1:HseJrM2JFf2vfiZJ8anY2hqBjdfY1Vlj/K27ueww4gg=
github.com/google/go-github/v41 v41.0.0/go.mod h1:XgmCA5H323A9rtgExdTcnDkcqp6S30AVACCBDOonIxg=
github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8=
github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
@@ -190,8 +181,6 @@ github.com/hashicorp/go-retryablehttp v0.6.8/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER
github.com/hashicorp/go-retryablehttp v0.7.0 h1:eu1EI/mbirUgP5C8hVsTNaGZreBDlYiwC1FZWkvQPQ4=
github.com/hashicorp/go-retryablehttp v0.7.0/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY=
github.com/hashicorp/go-version v1.2.1/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
github.com/hashicorp/go-version v1.3.0 h1:McDWVJIU/y+u1BRV06dPaLfLCaT7fUTJLp5r04x7iNw=
github.com/hashicorp/go-version v1.3.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
github.com/hashicorp/go-version v1.4.0 h1:aAQzgqIrRKRa7w75CKpbBxYsmUoPjzVm1W59ca1L0J4=
github.com/hashicorp/go-version v1.4.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
@@ -229,12 +218,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/ktrysmt/go-bitbucket v0.9.32 h1:IVk0m0gdB4OzRRLgxFnqNsfWKPXNdrcvgdpp9BojTpI=
github.com/ktrysmt/go-bitbucket v0.9.32/go.mod h1:FWxy2UK7GlK5b0NSJGc5hPqnssVlkNnsChvyuOf/Xno=
github.com/ktrysmt/go-bitbucket v0.9.36 h1:tXuZarVceStMxkmFxkSDMPfbm/Zff6wd8ctmus4d/s4=
github.com/ktrysmt/go-bitbucket v0.9.36/go.mod h1:FWxy2UK7GlK5b0NSJGc5hPqnssVlkNnsChvyuOf/Xno=
github.com/lib/pq v1.10.4 h1:SO9z7FRPzA03QhHKJrH5BXA6HU1rS4V2nIVrrNC1iYk=
github.com/lib/pq v1.10.4/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
github.com/matryer/is v1.2.0 h1:92UTHpy8CDwaJ08GqLDzhhuixiBUUD1p3AU6PHddz4A=
github.com/matryer/is v1.2.0/go.mod h1:2fLPjFQM9rhQ15aVEtbuwhJinnOqrmgXPNdZsdwlWXA=
github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
@@ -267,7 +252,6 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN
github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
github.com/prometheus/client_golang v1.11.0 h1:HNkLOAEQMIDv/K+04rukrLx6ch7msSRwf3/SASFAGtQ=
github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=
github.com/prometheus/client_golang v1.12.1 h1:ZiaPsmm9uiBeaSMRznKsCDNtPCS0T3JVDGF+06gjBzk=
github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY=
@@ -278,29 +262,21 @@ github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2
github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo=
github.com/prometheus/common v0.26.0 h1:iMAkS2TDoNWnKM+Kopnx/8tnEStIfpYA0ur0xQzzhMQ=
github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc=
github.com/prometheus/common v0.32.1 h1:hWIdL3N2HoUx3B8j3YN9mWor0qhY/NlEKZEaXxuIRh4=
github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls=
github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4=
github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
github.com/prometheus/procfs v0.7.3 h1:4jVXhlkAyzOScmCkXBTOLRLTz8EeU+eyjrwB/EPq0VU=
github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
github.com/robfig/cron/v3 v3.0.0 h1:kQ6Cb7aHOHTSzNVNEhmp8EcWKLb4CbiMW9h9VyIhO4E=
github.com/robfig/cron/v3 v3.0.0/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/rs/xid v1.3.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
github.com/rs/zerolog v1.26.0 h1:ORM4ibhEZeTeQlCojCK2kPz1ogAY4bGs4tD+SaAdGaE=
github.com/rs/zerolog v1.26.0/go.mod h1:yBiM87lvSqX8h0Ww4sdzNSkVYZ8dL2xjZJG1lAuGZEo=
github.com/rs/zerolog v1.26.1 h1:/ihwxqH+4z8UxyI70wM1z9yCvkWcfz/a3mj48k/Zngc=
github.com/rs/zerolog v1.26.1/go.mod h1:/wSSJWX7lVrsOwlbyTRSOJvqRlc+WjWlfes+CiJ+tmc=
github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww=
github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ=
github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
@@ -317,8 +293,6 @@ github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81P
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/xanzy/go-gitlab v0.52.2 h1:gkgg1z4ON70sphibtD86Bfmt1qV3mZ0pU0CBBCFAEvQ=
github.com/xanzy/go-gitlab v0.52.2/go.mod h1:Q+hQhV508bDPoBijv7YjK/Lvlb4PhVhJdKqXVQrUoAE=
github.com/xanzy/go-gitlab v0.55.1 h1:IgX/DS9buV0AUz8fuJPQkdl0fQGfBiAsAHxpun8sNhg=
github.com/xanzy/go-gitlab v0.55.1/go.mod h1:F0QEXwmqiBUxCgJm8fE9S+1veX4XC9Z4cfaAbqwk4YM=
github.com/xanzy/ssh-agent v0.3.0/go.mod h1:3s9xbODqPuuhK9JV1R321M/FlMZSBvE5aY6eAcqrDh0=
@@ -347,8 +321,6 @@ golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm
golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.0.0-20211202192323-5770296d904e h1:MUP6MR3rJ7Gk9LEia0LP2ytiH6MuCfs7qYz+47jGdD8=
golang.org/x/crypto v0.0.0-20211202192323-5770296d904e/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/crypto v0.0.0-20211215165025-cf75a172585e/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8=
golang.org/x/crypto v0.0.0-20220214200702-86341886e292 h1:f+lwQ+GtmgoY+A2YaQxlSOnDjXcQ7ZRLWOHbC6HtRqE=
golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
@@ -418,8 +390,6 @@ golang.org/x/net v0.0.0-20210326060303-6b1517762897/go.mod h1:uSPa2vr4CLtc/ILN5o
golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20211201190559-0a0e4e1bb54c h1:WtYZ93XtWSO5KlOMgPZu7hXY9WhMZpprvlm5VwvAl8c=
golang.org/x/net v0.0.0-20211201190559-0a0e4e1bb54c/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd h1:O7DYs+zxREGLKzKoMQrtrEacpb0ZVXA5rIwylE2Xchk=
golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
golang.org/x/oauth2 v0.0.0-20180227000427-d7d64896b5ff/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
@@ -491,9 +461,6 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211031064116-611d5d643895/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211124211545-fe61309f8881 h1:TyHqChC80pFkXWraUUf6RuB5IqFdQieMLwwCJokV2pc=
golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e h1:fLOSk5Q00efkSvAm+4xcoXD+RRmLmmulPn5I3Y9F2EM=
golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220209214540-3681064d5158 h1:rm+CHSpPEEW2IsXUib1ThaHIjuBVZjxNgSKmBLFfD4c=
@@ -506,14 +473,12 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20211116232009-f0f3c7e86c11 h1:GZokNIeuVkl3aZHJchRrr13WCsols02MLUcz1U9is6M=
golang.org/x/time v0.0.0-20211116232009-f0f3c7e86c11/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 h1:vVKdlvoWBphwdxWKrFZEuM0kGgGLxUOYcY4U/2Vjg44=
golang.org/x/time v0.0.0-20220210224613-90d013bbcef8/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+180 -38
View File
@@ -1,53 +1,102 @@
package gogs
import (
"gickup/types"
"github.com/cooperspencer/gickup/types"
"github.com/gogs/go-gogs-client"
"github.com/rs/zerolog/log"
)
// Backup TODO.
func Backup(r types.Repo, d types.GenRepo, dry bool) {
log.Info().Str("stage", "gogs").Str("url", d.Url).Msgf("mirroring %s to %s", types.Blue(r.Name), d.Url)
gogsclient := gogs.NewClient(d.Url, d.GetToken())
log.Info().
Str("stage", "gogs").
Str("url", d.URL).
Msgf("mirroring %s to %s", types.Blue(r.Name), d.URL)
gogsclient := gogs.NewClient(d.URL, d.GetToken())
user, err := gogsclient.GetSelfInfo()
if err != nil {
log.Fatal().Str("stage", "gogs").Str("url", d.Url).Msg(err.Error())
log.Fatal().
Str("stage", "gogs").
Str("url", d.URL).
Msg(err.Error())
}
if !dry {
repo, err := gogsclient.GetRepo(user.UserName, r.Name)
if err != nil {
opts := gogs.MigrateRepoOption{RepoName: r.Name, UID: int(user.ID), Mirror: true, CloneAddr: r.Url, AuthUsername: r.Token}
if r.Token == "" {
opts = gogs.MigrateRepoOption{RepoName: r.Name, UID: int(user.ID), Mirror: true, CloneAddr: r.Url, AuthUsername: r.Origin.User, AuthPassword: r.Origin.Password}
}
_, err := gogsclient.MigrateRepo(opts)
if err != nil {
log.Fatal().Str("stage", "gogs").Str("url", d.Url).Msg(err.Error())
}
} else {
if repo.Mirror {
log.Info().Str("stage", "gogs").Str("url", d.Url).Msgf("mirror of %s already exists, syncing instead", types.Blue(r.Name))
err := gogsclient.MirrorSync(user.UserName, repo.Name)
if err != nil {
log.Fatal().Str("stage", "gogs").Str("url", d.Url).Msg(err.Error())
}
log.Info().Str("stage", "gogs").Str("url", d.Url).Msgf("successfully synced %s.", types.Blue(r.Name))
if dry {
return
}
repo, err := gogsclient.GetRepo(user.UserName, r.Name)
if err != nil {
opts := gogs.MigrateRepoOption{
RepoName: r.Name,
UID: int(user.ID),
Mirror: true,
CloneAddr: r.URL,
AuthUsername: r.Token,
}
if r.Token == "" {
opts = gogs.MigrateRepoOption{
RepoName: r.Name,
UID: int(user.ID),
Mirror: true,
CloneAddr: r.URL,
AuthUsername: r.Origin.User,
AuthPassword: r.Origin.Password,
}
}
_, err := gogsclient.MigrateRepo(opts)
if err != nil {
log.Fatal().
Str("stage", "gogs").
Str("url", d.URL).
Msg(err.Error())
}
return
}
if repo.Mirror {
log.Info().
Str("stage", "gogs").
Str("url", d.URL).
Msgf("mirror of %s already exists, syncing instead", types.Blue(r.Name))
err := gogsclient.MirrorSync(user.UserName, repo.Name)
if err != nil {
log.Fatal().
Str("stage", "gogs").
Str("url", d.URL).
Msg(err.Error())
}
log.Info().
Str("stage", "gogs").
Str("url", d.URL).
Msgf("successfully synced %s.", types.Blue(r.Name))
}
}
// Get TODO.
func Get(conf *types.Conf) []types.Repo {
repos := []types.Repo{}
for _, repo := range conf.Source.Gogs {
log.Info().Str("stage", "gogs").Str("url", repo.Url).Msgf("grabbing repositories from %s", repo.User)
log.Info().
Str("stage", "gogs").
Str("url", repo.URL).
Msgf("grabbing repositories from %s", repo.User)
token := repo.GetToken()
client := gogs.NewClient(repo.Url, token)
client := gogs.NewClient(repo.URL, token)
gogsrepos, err := client.ListUserRepos(repo.User)
if err != nil {
log.Fatal().Str("stage", "gogs").Str("url", repo.Url).Msg(err.Error())
log.Fatal().
Str("stage", "gogs").
Str("url", repo.URL).
Msg(err.Error())
}
include := types.GetMap(repo.Include)
@@ -57,25 +106,67 @@ func Get(conf *types.Conf) []types.Repo {
for _, r := range gogsrepos {
if include[r.Name] {
repos = append(repos, types.Repo{Name: r.Name, Url: r.CloneURL, SshUrl: r.SSHURL, Token: token, Defaultbranch: r.DefaultBranch, Origin: repo, Owner: r.Owner.UserName, Hoster: types.GetHost(repo.Url)})
repos = append(repos, types.Repo{
Name: r.Name,
URL: r.CloneURL,
SSHURL: r.SSHURL,
Token: token,
Defaultbranch: r.DefaultBranch,
Origin: repo,
Owner: r.Owner.UserName,
Hoster: types.GetHost(repo.URL),
})
if repo.Wiki {
repos = append(repos, types.Repo{Name: r.Name + ".wiki", Url: types.DotGitRx.ReplaceAllString(r.CloneURL, ".wiki.git"), SshUrl: types.DotGitRx.ReplaceAllString(r.SSHURL, ".wiki.git"), Token: token, Defaultbranch: r.DefaultBranch, Origin: repo, Owner: r.Owner.UserName, Hoster: types.GetHost(repo.Url)})
repos = append(repos, types.Repo{
Name: r.Name + ".wiki",
URL: types.DotGitRx.ReplaceAllString(r.CloneURL, ".wiki.git"),
SSHURL: types.DotGitRx.ReplaceAllString(r.SSHURL, ".wiki.git"),
Token: token,
Defaultbranch: r.DefaultBranch,
Origin: repo,
Owner: r.Owner.UserName,
Hoster: types.GetHost(repo.URL),
})
}
continue
}
if exclude[r.Name] {
continue
}
if len(include) == 0 {
repos = append(repos, types.Repo{Name: r.Name, Url: r.CloneURL, SshUrl: r.SSHURL, Token: token, Defaultbranch: r.DefaultBranch, Origin: repo, Owner: r.Owner.UserName, Hoster: types.GetHost(repo.Url)})
repos = append(repos, types.Repo{
Name: r.Name,
URL: r.CloneURL,
SSHURL: r.SSHURL,
Token: token,
Defaultbranch: r.DefaultBranch,
Origin: repo,
Owner: r.Owner.UserName,
Hoster: types.GetHost(repo.URL),
})
if repo.Wiki {
repos = append(repos, types.Repo{Name: r.Name + ".wiki", Url: types.DotGitRx.ReplaceAllString(r.CloneURL, ".wiki.git"), SshUrl: types.DotGitRx.ReplaceAllString(r.SSHURL, ".wiki.git"), Token: token, Defaultbranch: r.DefaultBranch, Origin: repo, Owner: r.Owner.UserName, Hoster: types.GetHost(repo.Url)})
repos = append(repos, types.Repo{
Name: r.Name + ".wiki",
URL: types.DotGitRx.ReplaceAllString(r.CloneURL, ".wiki.git"),
SSHURL: types.DotGitRx.ReplaceAllString(r.SSHURL, ".wiki.git"),
Token: token,
Defaultbranch: r.DefaultBranch,
Origin: repo,
Owner: r.Owner.UserName,
Hoster: types.GetHost(repo.URL),
})
}
}
}
orgs, err := client.ListUserOrgs(repo.User)
if err != nil {
log.Fatal().Str("stage", "gogs").Str("url", repo.Url).Msg(err.Error())
log.Fatal().
Str("stage", "gogs").
Str("url", repo.URL).
Msg(err.Error())
}
orgrepos := []*gogs.Repository{}
@@ -88,43 +179,94 @@ func Get(conf *types.Conf) []types.Repo {
if includeorgs[org.UserName] {
o, err := client.ListOrgRepos(org.UserName)
if err != nil {
log.Fatal().Str("stage", "gogs").Str("url", repo.Url).Msg(err.Error())
log.Fatal().
Str("stage", "gogs").
Str("url", repo.URL).
Msg(err.Error())
}
if len(o) == 0 {
break
}
orgrepos = append(orgrepos, o...)
}
} else {
o, err := client.ListOrgRepos(org.UserName)
if err != nil {
log.Fatal().Str("stage", "gogs").Str("url", repo.Url).Msg(err.Error())
log.Fatal().
Str("stage", "gogs").
Str("url", repo.URL).
Msg(err.Error())
}
if len(o) == 0 {
break
}
orgrepos = append(orgrepos, o...)
}
}
}
for _, r := range orgrepos {
if include[r.Name] {
repos = append(repos, types.Repo{Name: r.Name, Url: r.CloneURL, SshUrl: r.SSHURL, Token: token, Defaultbranch: r.DefaultBranch, Origin: repo, Owner: r.Owner.UserName, Hoster: types.GetHost(repo.Url)})
repos = append(repos, types.Repo{
Name: r.Name,
URL: r.CloneURL,
SSHURL: r.SSHURL,
Token: token,
Defaultbranch: r.DefaultBranch,
Origin: repo,
Owner: r.Owner.UserName,
Hoster: types.GetHost(repo.URL),
})
if repo.Wiki {
repos = append(repos, types.Repo{Name: r.Name + ".wiki", Url: types.DotGitRx.ReplaceAllString(r.CloneURL, ".wiki.git"), SshUrl: types.DotGitRx.ReplaceAllString(r.SSHURL, ".wiki.git"), Token: token, Defaultbranch: r.DefaultBranch, Origin: repo, Owner: r.Owner.UserName, Hoster: types.GetHost(repo.Url)})
repos = append(repos, types.Repo{
Name: r.Name + ".wiki",
URL: types.DotGitRx.ReplaceAllString(r.CloneURL, ".wiki.git"),
SSHURL: types.DotGitRx.ReplaceAllString(r.SSHURL, ".wiki.git"),
Token: token,
Defaultbranch: r.DefaultBranch,
Origin: repo,
Owner: r.Owner.UserName,
Hoster: types.GetHost(repo.URL),
})
}
continue
}
if exclude[r.Name] {
continue
}
if len(repo.Include) == 0 {
repos = append(repos, types.Repo{Name: r.Name, Url: r.CloneURL, SshUrl: r.SSHURL, Token: token, Defaultbranch: r.DefaultBranch, Origin: repo, Owner: r.Owner.UserName, Hoster: types.GetHost(repo.Url)})
repos = append(repos, types.Repo{
Name: r.Name,
URL: r.CloneURL,
SSHURL: r.SSHURL,
Token: token,
Defaultbranch: r.DefaultBranch,
Origin: repo,
Owner: r.Owner.UserName,
Hoster: types.GetHost(repo.URL),
})
if repo.Wiki {
repos = append(repos, types.Repo{Name: r.Name + ".wiki", Url: types.DotGitRx.ReplaceAllString(r.CloneURL, ".wiki.git"), SshUrl: types.DotGitRx.ReplaceAllString(r.SSHURL, ".wiki.git"), Token: token, Defaultbranch: r.DefaultBranch, Origin: repo, Owner: r.Owner.UserName, Hoster: types.GetHost(repo.Url)})
repos = append(repos, types.Repo{
Name: r.Name + ".wiki",
URL: types.DotGitRx.ReplaceAllString(r.CloneURL, ".wiki.git"),
SSHURL: types.DotGitRx.ReplaceAllString(r.SSHURL, ".wiki.git"),
Token: token,
Defaultbranch: r.DefaultBranch,
Origin: repo,
Owner: r.Owner.UserName,
Hoster: types.GetHost(repo.URL),
})
}
}
}
}
return repos
}
+138 -64
View File
@@ -1,13 +1,13 @@
package local
import (
"gickup/types"
"net"
"os"
"path"
"strings"
"time"
"github.com/cooperspencer/gickup/types"
"github.com/go-git/go-git/v5"
"github.com/go-git/go-git/v5/config"
"github.com/go-git/go-git/v5/plumbing/transport"
@@ -18,94 +18,153 @@ import (
gossh "golang.org/x/crypto/ssh"
)
// Locally TODO.
func Locally(repo types.Repo, l types.Local, dry bool) {
if l.Structured {
repo.Name = path.Join(repo.Hoster, repo.Owner, repo.Name)
}
stat, err := os.Stat(l.Path)
if os.IsNotExist(err) && !dry {
err := os.MkdirAll(l.Path, 0777)
if err != nil {
log.Fatal().Str("stage", "locally").Str("path", l.Path).Msg(err.Error())
if err := os.MkdirAll(l.Path, 0o777); err != nil {
log.Fatal().
Str("stage", "locally").
Str("path", l.Path).
Msg(err.Error())
}
stat, _ = os.Stat(l.Path)
}
if stat != nil {
if stat.IsDir() {
os.Chdir(l.Path)
if stat != nil && stat.IsDir() {
if err := os.Chdir(l.Path); err != nil {
log.Fatal().
Str("stage", "locally").
Str("path", l.Path).
Msg(err.Error())
}
}
tries := 5
var auth transport.AuthMethod
if repo.Origin.SSH {
switch {
case repo.Origin.SSH:
if repo.Origin.SSHKey == "" {
home := os.Getenv("HOME")
repo.Origin.SSHKey = path.Join(home, ".ssh", "id_rsa")
}
auth, err = ssh.NewPublicKeysFromFile("git", repo.Origin.SSHKey, "")
if err != nil {
panic(err)
}
} else if repo.Token != "" {
case repo.Token != "":
auth = &http.BasicAuth{
Username: "xyz",
Password: repo.Token,
}
} else if repo.Origin.Username != "" && repo.Origin.Password != "" {
case repo.Origin.Username != "" && repo.Origin.Password != "":
auth = &http.BasicAuth{
Username: repo.Origin.Username,
Password: repo.Origin.Password,
}
}
for x := 1; x <= tries; x++ {
stat, err := os.Stat(repo.Name)
if os.IsNotExist(err) {
log.Info().Str("stage", "locally").Str("path", l.Path).Msgf("cloning %s", types.Green(repo.Name))
log.Info().
Str("stage", "locally").
Str("path", l.Path).
Msgf("cloning %s", types.Green(repo.Name))
err := cloneRepository(repo, auth, dry)
if err != nil {
if x == tries {
log.Fatal().Str("stage", "locally").Str("path", l.Path).Str("repo", repo.Name).Msg(err.Error())
} else {
if strings.Contains(err.Error(), "ERR access denied or repository not exported") {
log.Warn().Str("stage", "locally").Str("path", l.Path).Str("repo", repo.Name).Msgf("%s doesn't exist.", repo.Name)
break
}
if strings.Contains(err.Error(), "remote repository is empty") {
log.Warn().Str("stage", "locally").Str("path", l.Path).Str("repo", repo.Name).Msg(err.Error())
break
}
log.Warn().Str("stage", "locally").Str("path", l.Path).Msgf("retry %s from %s", types.Red(x), types.Red(tries))
time.Sleep(5 * time.Second)
continue
log.Fatal().
Str("stage", "locally").
Str("path", l.Path).
Str("repo", repo.Name).
Msg(err.Error())
break
}
if strings.Contains(err.Error(), "ERR access denied or repository not exported") {
log.Warn().
Str("stage", "locally").
Str("path", l.Path).
Str("repo", repo.Name).
Msgf("%s doesn't exist.", repo.Name)
break
}
if strings.Contains(err.Error(), "remote repository is empty") {
log.Warn().
Str("stage", "locally").
Str("path", l.Path).
Str("repo", repo.Name).
Msg(err.Error())
break
}
log.Warn().
Str("stage", "locally").
Str("path", l.Path).
Msgf("retry %s from %s", types.Red(x), types.Red(tries))
time.Sleep(5 * time.Second)
continue
}
} else {
if !stat.IsDir() {
log.Warn().Str("stage", "locally").Str("path", l.Path).Str("repo", repo.Name).Msgf("%s is a file", types.Red(repo.Name))
log.Warn().
Str("stage", "locally").
Str("path", l.Path).
Str("repo", repo.Name).
Msgf("%s is a file", types.Red(repo.Name))
} else {
log.Info().Str("stage", "locally").Str("path", l.Path).Msgf("opening %s locally", types.Green(repo.Name))
log.Info().
Str("stage", "locally").
Str("path", l.Path).
Msgf("opening %s locally", types.Green(repo.Name))
err := updateRepository(repo.Name, auth, dry)
if err != nil {
if strings.Contains(err.Error(), "already up-to-date") {
log.Info().Str("stage", "locally").Str("path", l.Path).Msg(err.Error())
log.Info().
Str("stage", "locally").
Str("path", l.Path).
Msg(err.Error())
} else {
if x == tries {
log.Fatal().Str("stage", "locally").Str("path", l.Path).Str("repo", repo.Name).Msg(err.Error())
log.Fatal().
Str("stage", "locally").
Str("path", l.Path).
Str("repo", repo.Name).
Msg(err.Error())
} else {
os.RemoveAll(repo.Name)
log.Warn().Str("stage", "locally").Str("path", l.Path).Str("repo", repo.Name).Msgf("retry %s from %s", types.Red(x), types.Red(tries))
log.Warn().
Str("stage", "locally").
Str("path", l.Path).
Str("repo", repo.Name).
Msgf("retry %s from %s", types.Red(x), types.Red(tries))
time.Sleep(5 * time.Second)
continue
}
}
}
}
}
x = 5
}
}
@@ -122,55 +181,67 @@ func updateRepository(repoPath string, auth transport.AuthMethod, dry bool) erro
}
if !dry {
log.Info().Str("stage", "locally").Msgf("pulling %s", types.Green(repoPath))
log.Info().
Str("stage", "locally").
Msgf("pulling %s", types.Green(repoPath))
err = w.Pull(&git.PullOptions{Auth: auth, RemoteName: "origin", SingleBranch: false})
}
return err
}
func cloneRepository(repo types.Repo, auth transport.AuthMethod, dry bool) error {
if !dry {
url := repo.Url
if repo.Origin.SSH {
url = repo.SshUrl
site := types.Site{}
err := site.GetValues(url)
if err != nil {
log.Fatal().Str("stage", "locally").Str("repo", repo.Name).Msg(err.Error())
}
if dry {
return nil
}
sshAuth, err := goph.Key(repo.Origin.SSHKey, "")
if err != nil {
log.Fatal().Str("stage", "locally").Str("repo", repo.Name).Msg(err.Error())
}
url := repo.URL
if repo.Origin.SSH {
url = repo.SSHURL
site := types.Site{}
err = testSshConnection(site, sshAuth)
if err != nil {
log.Fatal().Str("stage", "locally").Str("repo", repo.Name).Msg(err.Error())
}
}
rem := git.NewRemote(nil, &config.RemoteConfig{Name: "origin", URLs: []string{url}})
_, err := rem.List(&git.ListOptions{Auth: auth})
err := site.GetValues(url)
if err != nil {
return err
log.Fatal().Str("stage", "locally").Str("repo", repo.Name).Msg(err.Error())
}
_, err = git.PlainClone(repo.Name, false, &git.CloneOptions{
URL: url,
Auth: auth,
SingleBranch: false,
})
sshAuth, err := goph.Key(repo.Origin.SSHKey, "")
if err != nil {
log.Fatal().Str("stage", "locally").Str("repo", repo.Name).Msg(err.Error())
}
err = testSSHConnection(site, sshAuth)
if err != nil {
log.Fatal().Str("stage", "locally").Str("repo", repo.Name).Msg(err.Error())
}
}
remoteConfig := config.RemoteConfig{
Name: "origin",
URLs: []string{url},
}
rem := git.NewRemote(nil, &remoteConfig)
_, err := rem.List(&git.ListOptions{Auth: auth})
if err != nil {
return err
}
return nil
_, err = git.PlainClone(repo.Name, false, &git.CloneOptions{
URL: url,
Auth: auth,
SingleBranch: false,
})
return err
}
func testSshConnection(site types.Site, sshAuth goph.Auth) error {
func testSSHConnection(site types.Site, sshAuth goph.Auth) error {
_, err := goph.NewConn(&goph.Config{
User: site.User,
Addr: site.Url,
Addr: site.URL,
Port: uint(site.Port),
Auth: sshAuth,
Callback: VerifyHost,
@@ -179,8 +250,10 @@ func testSshConnection(site types.Site, sshAuth goph.Auth) error {
return err
}
// VerifyHost TODO.
func VerifyHost(host string, remote net.Addr, key gossh.PublicKey) error {
// Got from the example from https://github.com/melbahja/goph/blob/master/examples/goph/main.go
// Got from the example from
// https://github.com/melbahja/goph/blob/master/examples/goph/main.go
//
// If you want to connect to new hosts.
// here your should check new connections public keys
@@ -188,7 +261,8 @@ func VerifyHost(host string, remote net.Addr, key gossh.PublicKey) error {
//
// hostFound: is host in known hosts file.
// err: error if key not in known hosts file OR host in known hosts file but key changed!
// err: error if key not in known hosts file
// OR host in known hosts file but key changed!
hostFound, err := goph.CheckKnownHost(host, remote, key, "")
// Host in known hosts but key mismatch!
// Maybe because of MAN IN THE MIDDLE ATTACK!
+10 -4
View File
@@ -5,17 +5,21 @@ import (
"os"
"path"
"gickup/types"
"github.com/cooperspencer/gickup/types"
"github.com/rs/zerolog"
"github.com/rs/zerolog/log"
"gopkg.in/natefinch/lumberjack.v2"
)
// NewRollingFile TODO.
func NewRollingFile(config types.FileLogging) io.Writer {
if config.Dir != "" {
if err := os.MkdirAll(config.Dir, 0744); err != nil {
log.Error().Err(err).Str("path", config.Dir).Msg("can't create log directory")
if err := os.MkdirAll(config.Dir, 0o744); err != nil {
log.Error().
Err(err).
Str("path", config.Dir).
Msg("can't create log directory")
return nil
}
} else {
@@ -28,6 +32,7 @@ func NewRollingFile(config types.FileLogging) io.Writer {
}
}
// CreateLogger create an instance of Logger.
func CreateLogger(conf types.Logging) zerolog.Logger {
var writers []io.Writer
@@ -36,6 +41,7 @@ func CreateLogger(conf types.Logging) zerolog.Logger {
if conf.FileLogging.File != "" {
writers = append(writers, NewRollingFile(conf.FileLogging))
}
mw := io.MultiWriter(writers...)
return zerolog.New(mw).With().Timestamp().Logger()
+135 -93
View File
@@ -9,17 +9,16 @@ import (
"strings"
"time"
"gickup/bitbucket"
"gickup/gitea"
"gickup/github"
"gickup/gitlab"
"gickup/gogs"
"gickup/local"
"gickup/logger"
prometheus "gickup/metrics/prometheus"
"gickup/types"
"github.com/alecthomas/kong"
"github.com/cooperspencer/gickup/bitbucket"
"github.com/cooperspencer/gickup/gitea"
"github.com/cooperspencer/gickup/github"
"github.com/cooperspencer/gickup/gitlab"
"github.com/cooperspencer/gickup/gogs"
"github.com/cooperspencer/gickup/local"
"github.com/cooperspencer/gickup/logger"
"github.com/cooperspencer/gickup/metrics/prometheus"
"github.com/cooperspencer/gickup/types"
"github.com/robfig/cron/v3"
"github.com/rs/zerolog"
"github.com/rs/zerolog/log"
@@ -27,38 +26,40 @@ import (
)
var cli struct {
Configfile string `arg name:"conf" help:"path to the configfile." default:"conf.yml"`
Version bool `flag name:"version" help:"show version."`
Dry bool `flag name:"dryrun" help:"make a dry-run."`
Configfile string `arg name:"conf" help:"Path to the configfile." default:"conf.yml"`
Version bool `flag name:"version" help:"Show version."`
Dry bool `flag name:"dryrun" help:"Make a dry-run."`
Quiet bool `flag name:"quiet" help:"Output only warnings, errors, and fatal messages to stderr log output"`
Silent bool `flag name:"silent" help:"Suppress all stderr log output"`
}
var (
version = "unknown"
)
var version = "unknown"
func ReadConfigfile(configfile string) *types.Conf {
func readConfigFile(configfile string) *types.Conf {
cfgdata, err := ioutil.ReadFile(configfile)
if err != nil {
log.Fatal().Str("stage", "readconfig").Str("file", configfile).Msgf("Cannot open config file from %s", types.Red(configfile))
log.Fatal().
Str("stage", "readconfig").
Str("file", configfile).
Msgf("Cannot open config file from %s", types.Red(configfile))
}
t := types.Conf{}
err = yaml.Unmarshal([]byte(cfgdata), &t)
err = yaml.Unmarshal(cfgdata, &t)
if err != nil {
log.Fatal().Str("stage", "readconfig").Str("file", configfile).Msg("Cannot map yml config file to interface, possible syntax error")
log.Fatal().
Str("stage", "readconfig").
Str("file", configfile).
Msg("Cannot map yml config file to interface, possible syntax error")
}
return &t
}
func GetUserHome() (string, error) {
func getUserHome() (string, error) {
usr, err := user.Current()
if err != nil {
return "", err
}
@@ -66,65 +67,88 @@ func GetUserHome() (string, error) {
return usr.HomeDir, nil
}
func SubstituteHomeForTildeInPath(path string) string {
func substituteHomeForTildeInPath(path string) string {
if !strings.HasPrefix(path, "~") {
return path
} else {
if path == "~" {
userHome, err := GetUserHome()
if err != nil {
log.Fatal().Str("stage", "local ~ substitution").Str("path", path).Msg(err.Error())
} else {
return userHome
}
} else if strings.HasPrefix(path, "~/") {
userHome, err := GetUserHome()
if err != nil {
log.Fatal().Str("stage", "local ~/ substitution").Str("path", path).Msg(err.Error())
} else {
return filepath.Join(userHome, path[2:])
}
}
if path == "~" {
userHome, err := getUserHome()
if err != nil {
log.Fatal().
Str("stage", "local ~ substitution").
Str("path", path).
Msg(err.Error())
} else {
return userHome
}
}
if strings.HasPrefix(path, "~/") {
userHome, err := getUserHome()
if err != nil {
log.Fatal().
Str("stage", "local ~/ substitution").
Str("path", path).
Msg(err.Error())
} else {
return filepath.Join(userHome, path[2:])
}
}
// in any other strange case
return path
}
func Backup(repos []types.Repo, conf *types.Conf) {
func backup(repos []types.Repo, conf *types.Conf) {
checkedpath := false
for _, r := range repos {
log.Info().Str("stage", "backup").Msgf("starting backup for %s", r.Url)
log.Info().
Str("stage", "backup").
Msgf("starting backup for %s", r.URL)
for i, d := range conf.Destination.Local {
if !checkedpath {
d.Path = SubstituteHomeForTildeInPath(d.Path)
d.Path = substituteHomeForTildeInPath(d.Path)
path, err := filepath.Abs(d.Path)
if err != nil {
log.Fatal().Str("stage", "locally").Str("path", d.Path).Msg(err.Error())
log.Fatal().
Str("stage", "locally").
Str("path", d.Path).
Msg(err.Error())
}
conf.Destination.Local[i].Path = path
checkedpath = true
}
local.Locally(r, d, cli.Dry)
prometheus.DestinationBackupsComplete.WithLabelValues("local").Inc()
}
for _, d := range conf.Destination.Gitea {
gitea.Backup(r, d, cli.Dry)
prometheus.DestinationBackupsComplete.WithLabelValues("gitea").Inc()
}
for _, d := range conf.Destination.Gogs {
gogs.Backup(r, d, cli.Dry)
prometheus.DestinationBackupsComplete.WithLabelValues("gogs").Inc()
}
for _, d := range conf.Destination.Gitlab {
gitlab.Backup(r, d, cli.Dry)
prometheus.DestinationBackupsComplete.WithLabelValues("gitlab").Inc()
}
prometheus.SourceBackupsComplete.WithLabelValues(r.Name).Inc()
}
}
func RunBackup(conf *types.Conf) {
func runBackup(conf *types.Conf) {
log.Info().Msg("Backup run starting")
startTime := time.Now()
prometheus.JobsStarted.Inc()
@@ -132,27 +156,26 @@ func RunBackup(conf *types.Conf) {
// Github
repos := github.Get(conf)
prometheus.CountReposDiscovered.WithLabelValues("github").Set(float64(len(repos)))
Backup(repos, conf)
backup(repos, conf)
// Gitea
repos = gitea.Get(conf)
prometheus.CountReposDiscovered.WithLabelValues("gitea").Set(float64(len(repos)))
Backup(repos, conf)
backup(repos, conf)
// Gogs
repos = gogs.Get(conf)
prometheus.CountReposDiscovered.WithLabelValues("gogs").Set(float64(len(repos)))
Backup(repos, conf)
backup(repos, conf)
// Gitlab
repos = gitlab.Get(conf)
prometheus.CountReposDiscovered.WithLabelValues("gitlab").Set(float64(len(repos)))
Backup(repos, conf)
backup(repos, conf)
//Bitbucket
repos = bitbucket.Get(conf)
prometheus.CountReposDiscovered.WithLabelValues("bitbucket").Set(float64(len(repos)))
Backup(repos, conf)
backup(repos, conf)
endTime := time.Now()
duration := endTime.Sub(startTime)
@@ -169,8 +192,9 @@ func RunBackup(conf *types.Conf) {
}
}
func PlaysForever() {
func playsForever() {
wait := make(chan struct{})
for {
<-wait
}
@@ -178,61 +202,79 @@ func PlaysForever() {
func main() {
timeformat := "2006-01-02T15:04:05Z07:00"
log.Logger = log.Output(zerolog.ConsoleWriter{Out: os.Stderr, TimeFormat: timeformat})
log.Logger = log.Output(zerolog.ConsoleWriter{
Out: os.Stderr,
TimeFormat: timeformat,
})
kong.Parse(&cli, kong.Name("gickup"), kong.Description("a tool to backup all your favorite repos"))
kong.Parse(&cli, kong.Name("gickup"),
kong.Description("a tool to backup all your favorite repos"))
if cli.Version {
fmt.Println(version)
} else {
if cli.Quiet {
zerolog.SetGlobalLevel(zerolog.WarnLevel)
}
if cli.Silent {
zerolog.SetGlobalLevel(zerolog.Disabled)
}
return
}
if cli.Dry {
log.Info().Str("dry", "true").Msgf("this is a %s", types.Blue("dry run"))
}
if cli.Quiet {
zerolog.SetGlobalLevel(zerolog.WarnLevel)
}
log.Info().Str("file", cli.Configfile).Msgf("Reading %s", types.Green(cli.Configfile))
if cli.Silent {
zerolog.SetGlobalLevel(zerolog.Disabled)
}
conf := ReadConfigfile(cli.Configfile)
if conf.Log.Timeformat == "" {
conf.Log.Timeformat = timeformat
}
log.Logger = logger.CreateLogger(conf.Log)
// one pair per source-destination
pairs := conf.Source.Count() * conf.Destination.Count()
if cli.Dry {
log.Info().
Int("sources", conf.Source.Count()).
Int("destinations", conf.Destination.Count()).
Int("pairs", pairs).
Msg("Configuration loaded")
Str("dry", "true").
Msgf("this is a %s", types.Blue("dry run"))
}
if conf.HasValidCronSpec() {
c := cron.New()
logNextRun(conf)
log.Info().Str("file", cli.Configfile).
Msgf("Reading %s", types.Green(cli.Configfile))
c.AddFunc(conf.Cron, func() {
RunBackup(conf)
})
c.Start()
conf := readConfigFile(cli.Configfile)
if conf.Log.Timeformat == "" {
conf.Log.Timeformat = timeformat
}
if conf.HasAllPrometheusConf() {
prometheus.CountSourcesConfigured.Add(float64(conf.Source.Count()))
prometheus.CountDestinationsConfigured.Add(float64(conf.Destination.Count()))
prometheus.Serve(conf.Metrics.Prometheus)
} else {
PlaysForever()
}
} else {
RunBackup(conf)
log.Logger = logger.CreateLogger(conf.Log)
// one pair per source-destination
pairs := conf.Source.Count() * conf.Destination.Count()
log.Info().
Int("sources", conf.Source.Count()).
Int("destinations", conf.Destination.Count()).
Int("pairs", pairs).
Msg("Configuration loaded")
if conf.HasValidCronSpec() {
c := cron.New()
logNextRun(conf)
_, err := c.AddFunc(conf.Cron, func() {
runBackup(conf)
})
if err != nil {
log.Fatal().
Int("sources", conf.Source.Count()).
Int("destinations", conf.Destination.Count()).
Int("pairs", pairs).
Msg(err.Error())
}
c.Start()
if conf.HasAllPrometheusConf() {
prometheus.CountSourcesConfigured.Add(float64(conf.Source.Count()))
prometheus.CountDestinationsConfigured.Add(float64(conf.Destination.Count()))
prometheus.Serve(conf.Metrics.Prometheus)
} else {
playsForever()
}
} else {
runBackup(conf)
}
}
+9 -5
View File
@@ -6,22 +6,26 @@ import (
)
func TestTildeReplacement_NoAction(t *testing.T) {
path := "/boop"
if SubstituteHomeForTildeInPath(path) != path {
t.Parallel()
if path := "/boop"; substituteHomeForTildeInPath(path) != path {
t.Error("Altered path when no alteration was expected")
}
}
func TestTildeReplacement_TildeOnly(t *testing.T) {
path := "~"
if SubstituteHomeForTildeInPath(path) == path {
t.Parallel()
if path := "~"; substituteHomeForTildeInPath(path) == path {
t.Error("Path unaltered when alteration was expected")
}
}
func TestTildeReplacement_TildeDir(t *testing.T) {
t.Parallel()
path := "~/boop"
actual := SubstituteHomeForTildeInPath(path)
actual := substituteHomeForTildeInPath(path)
if strings.HasPrefix(actual, "~") {
t.Error("Altered path still contains ~")
}
+10 -3
View File
@@ -1,9 +1,9 @@
package prometheus
import (
"gickup/types"
"net/http"
"github.com/cooperspencer/gickup/types"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
"github.com/prometheus/client_golang/prometheus/promhttp"
@@ -51,8 +51,15 @@ var DestinationBackupsComplete = promauto.NewCounterVec(prometheus.CounterOpts{
}, []string{"destination_type"})
func Serve(conf types.PrometheusConfig) {
log.Info().Str("listenAddr", conf.ListenAddr).Str("endpoint", conf.Endpoint).Msg("Starting Prometheus listener")
log.Info().
Str("listenAddr", conf.ListenAddr).
Str("endpoint", conf.Endpoint).
Msg("Starting Prometheus listener")
http.Handle(conf.Endpoint, promhttp.Handler())
http.ListenAndServe(conf.ListenAddr, nil)
err := http.ListenAndServe(conf.ListenAddr, nil)
log.Fatal().
Str("listenAddr", conf.ListenAddr).
Str("endpoint", conf.Endpoint).
Msg(err.Error())
}
+109 -56
View File
@@ -19,7 +19,7 @@ import (
"github.com/rs/zerolog/log"
)
// Destination
// Destination TODO.
type Destination struct {
Gitlab []GenRepo `yaml:"gitlab"`
Local []Local `yaml:"local"`
@@ -28,6 +28,7 @@ type Destination struct {
Gogs []GenRepo `yaml:"gogs"`
}
// Count TODO.
func (dest Destination) Count() int {
return len(dest.Gogs) +
len(dest.Gitea) +
@@ -36,13 +37,13 @@ func (dest Destination) Count() int {
len(dest.Gitlab)
}
// Local
// Local TODO.
type Local struct {
Path string `yaml:"path"`
Structured bool `yaml:"structured"`
}
// Conf
// Conf TODO.
type Conf struct {
Source Source `yaml:"source"`
Destination Destination `yaml:"destination"`
@@ -51,32 +52,39 @@ type Conf struct {
Metrics Metrics `yaml:"metrics"`
}
// PrometheusConfig TODO.
type PrometheusConfig struct {
ListenAddr string `yaml:"listen_addr"`
Endpoint string `yaml:"endpoint"`
}
// Metrics TODO.
type Metrics struct {
Prometheus PrometheusConfig `yaml:"prometheus"`
}
// Logging TODO.
type Logging struct {
Timeformat string `yaml:"timeformat"`
FileLogging FileLogging `yaml:"file-logging"`
}
// FileLogging TODO.
type FileLogging struct {
Dir string `yaml:"dir"`
File string `yaml:"file"`
MaxAge int `yaml:"maxage"`
}
// CheckAllValuesOrNone TODO.
func CheckAllValuesOrNone(parent string, theMap map[string]string) bool {
missing := false
var missing bool
for key, value := range theMap {
if value == "" {
log.Warn().Str("expectedButMissing", key).Msg(
"A configuration value is expected but not present. Ensure all required configuration is present.")
log.Warn().Str("expectedButMissing", key).
Msg("A configuration value is expected but not present. Ensure all required configuration is present.")
missing = true
}
}
@@ -84,33 +92,34 @@ func CheckAllValuesOrNone(parent string, theMap map[string]string) bool {
return !missing
}
// HasAllPrometheusConf TODO.
func (conf Conf) HasAllPrometheusConf() bool {
if len(conf.Metrics.Prometheus.ListenAddr) == 0 && len(conf.Metrics.Prometheus.Endpoint) == 0 {
return false
} else {
checks := map[string]string{
"listenaddr": conf.Metrics.Prometheus.ListenAddr,
"endpoint": conf.Metrics.Prometheus.Endpoint,
}
ok := CheckAllValuesOrNone("prometheus", checks)
if !ok {
log.Fatal().Str("monitoring", "prometheus").Msg(
"Fix the values in the configuration.")
}
return ok
}
checks := map[string]string{
"listenaddr": conf.Metrics.Prometheus.ListenAddr,
"endpoint": conf.Metrics.Prometheus.Endpoint,
}
ok := CheckAllValuesOrNone("prometheus", checks)
if !ok {
log.Fatal().Str("monitoring", "prometheus").Msg(
"Fix the values in the configuration.")
}
return ok
}
// MissingCronSpec TODO.
func (conf Conf) MissingCronSpec() bool {
return conf.Cron == ""
}
// ParseCronSpec TODO.
func ParseCronSpec(spec string) (cron.Schedule, error) {
sched, err := cron.ParseStandard(spec)
if err != nil {
log.Error().Str("spec", spec).Msg(err.Error())
}
@@ -118,18 +127,23 @@ func ParseCronSpec(spec string) (cron.Schedule, error) {
return sched, err
}
// GetNextRun TODO.
func (conf Conf) GetNextRun() (*time.Time, error) {
if conf.MissingCronSpec() {
return nil, fmt.Errorf("cron unspecified")
}
parsedSched, err := ParseCronSpec(conf.Cron)
if err != nil {
return nil, err
}
next := parsedSched.Next(time.Now())
return &next, nil
}
// HasValidCronSpec TODO.
func (conf Conf) HasValidCronSpec() bool {
if conf.MissingCronSpec() {
return false
@@ -140,7 +154,7 @@ func (conf Conf) HasValidCronSpec() bool {
return err == nil
}
// Source
// Source TODO.
type Source struct {
Gogs []GenRepo `yaml:"gogs"`
Gitlab []GenRepo `yaml:"gitlab"`
@@ -149,6 +163,7 @@ type Source struct {
BitBucket []GenRepo `yaml:"bitbucket"`
}
// Count TODO.
func (source Source) Count() int {
return len(source.Gogs) +
len(source.Gitea) +
@@ -157,7 +172,7 @@ func (source Source) Count() int {
len(source.Gitlab)
}
// Generell Repo
// GenRepo Generell Repo.
type GenRepo struct {
Token string `yaml:"token"`
TokenFile string `yaml:"token_file"`
@@ -166,7 +181,7 @@ type GenRepo struct {
SSHKey string `yaml:"sshkey"`
Username string `yaml:"username"`
Password string `yaml:"password"`
Url string `yaml:"url"`
URL string `yaml:"url"`
Exclude []string `yaml:"exclude"`
ExcludeOrgs []string `yaml:"excludeorgs"`
Include []string `yaml:"include"`
@@ -175,12 +190,12 @@ type GenRepo struct {
Starred bool `yaml:"starred"`
}
// GetToken TODO.
func (grepo GenRepo) GetToken() string {
token, err := resolveToken(grepo.Token, grepo.TokenFile)
if err != nil {
log.Fatal().
Str("url", grepo.Url).
Str("url", grepo.URL).
Str("tokenfile", grepo.TokenFile).
Err(err)
}
@@ -195,7 +210,6 @@ func resolveToken(tokenString string, tokenFile string) (string, error) {
if tokenFile != "" {
data, err := os.ReadFile(tokenFile)
if err != nil {
return "", err
}
@@ -206,17 +220,18 @@ func resolveToken(tokenString string, tokenFile string) (string, error) {
Msg("Read token file")
tokenData := strings.ReplaceAll(string(data), "\n", "")
return tokenData, nil
return tokenData, nil
}
return "", fmt.Errorf("no token or tokenfile was specified in config when one was expected")
}
// Repo
// Repo TODO.
type Repo struct {
Name string
Url string
SshUrl string
URL string
SSHURL string
Token string
Defaultbranch string
Origin GenRepo
@@ -224,80 +239,111 @@ type Repo struct {
Hoster string
}
// Site
// Site TODO.
type Site struct {
Url string
URL string
User string
Port int
}
// GetHost TODO.
func GetHost(url string) string {
if strings.Contains(url, "http://") {
url = strings.Split(url, "http://")[1]
}
if strings.Contains(url, "https://") {
url = strings.Split(url, "https://")[1]
}
if strings.Contains(url, "/") {
url = strings.Split(url, "/")[0]
}
return url
}
// GetValues TODO.
func (s *Site) GetValues(url string) error {
if strings.HasPrefix(url, "ssh://") {
url = strings.Split(url, "ssh://")[1]
userurl := strings.Split(url, "@")
s.User = userurl[0]
urlport := strings.Split(userurl[1], ":")
s.Url = urlport[0]
s.URL = urlport[0]
portstring := strings.Split(urlport[1], "/")[0]
port, err := strconv.Atoi(portstring)
if err != nil {
return err
}
s.Port = port
} else {
userurl := strings.Split(url, "@")
s.User = userurl[0]
urlport := strings.Split(userurl[1], ":")
s.Url = urlport[0]
s.Port = 22
return nil
}
userurl := strings.Split(url, "@")
s.User = userurl[0]
urlport := strings.Split(userurl[1], ":")
s.URL = urlport[0]
s.Port = 22
return nil
}
var (
Red = color.FgRed.Render
Green = color.FgGreen.Render
Blue = color.FgBlue.Render
// Red Render message with Red color.
Red = color.FgRed.Render
// Green Render message with Green color.
Green = color.FgGreen.Render
// Blue Render message with Blue color.
Blue = color.FgBlue.Render
// DotGitRx .git regexp.
DotGitRx = regexp.MustCompile(`\.git$`)
)
// GetMap TODO.
func GetMap(excludes []string) map[string]bool {
excludemap := make(map[string]bool)
for _, exclude := range excludes {
excludemap[exclude] = true
}
return excludemap
}
func StatRemote(URL, sshURL string, repo GenRepo) bool {
var url string
var auth transport.AuthMethod
var err error
func statRemoteSSH(sshURL string, repo GenRepo) (string, transport.AuthMethod, error) {
url := DotGitRx.ReplaceAllString(sshURL, ".wiki.git")
if repo.SSHKey == "" {
home := os.Getenv("HOME")
repo.SSHKey = path.Join(home, ".ssh", "id_rsa")
}
auth, err := ssh.NewPublicKeysFromFile("git", repo.SSHKey, "")
return url, auth, err
}
// StatRemote TODO.
func StatRemote(remoteURL, sshURL string, repo GenRepo) bool {
var (
url string
auth transport.AuthMethod
err error
)
if repo.SSH {
url = DotGitRx.ReplaceAllString(sshURL, ".wiki.git")
if repo.SSHKey == "" {
home := os.Getenv("HOME")
repo.SSHKey = path.Join(home, ".ssh", "id_rsa")
}
auth, err = ssh.NewPublicKeysFromFile("git", repo.SSHKey, "")
url, auth, err = statRemoteSSH(sshURL, repo)
if err != nil {
panic(err)
return false
}
} else {
url = DotGitRx.ReplaceAllString(URL, ".wiki.git")
url = DotGitRx.ReplaceAllString(remoteURL, ".wiki.git")
if repo.Token != "" {
auth = &http.BasicAuth{
Username: "xyz",
@@ -310,6 +356,13 @@ func StatRemote(URL, sshURL string, repo GenRepo) bool {
}
}
}
_, err = git.NewRemote(nil, &config.RemoteConfig{Name: "origin", URLs: []string{url}}).List(&git.ListOptions{Auth: auth})
remoteConfig := config.RemoteConfig{
Name: "origin",
URLs: []string{url},
}
_, err = git.NewRemote(nil, &remoteConfig).List(&git.ListOptions{Auth: auth})
return err == nil
}
+6
View File
@@ -3,6 +3,8 @@ package types
import "testing"
func TestConfCronMissing(t *testing.T) {
t.Parallel()
conf := Conf{Cron: ""}
if !conf.MissingCronSpec() {
@@ -11,6 +13,8 @@ func TestConfCronMissing(t *testing.T) {
}
func TestParseValidCron(t *testing.T) {
t.Parallel()
conf := Conf{Cron: "0 0 * * *"}
if !conf.HasValidCronSpec() {
@@ -19,6 +23,8 @@ func TestParseValidCron(t *testing.T) {
}
func TestParseInvalidCron(t *testing.T) {
t.Parallel()
conf := Conf{Cron: "redshirt"}
if conf.HasValidCronSpec() {