From f2a3abc683ad4b2177b7c7c6160a2c0b4316120a Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sun, 13 Oct 2019 21:23:14 +0800 Subject: [PATCH] Move migrating repository from frontend to backend (#6200) * move migrating to backend * add loading image when migrating and fix tests * fix format * fix lint * add redis task queue support and improve docs * add redis vendor * fix vet * add database migrations and fix app.ini sample * add comments for task section on app.ini.sample * Update models/migrations/v84.go Co-Authored-By: lunny * Update models/repo.go Co-Authored-By: lunny * move migrating to backend * add loading image when migrating and fix tests * fix fmt * add redis task queue support and improve docs * fix fixtures * fix fixtures * fix duplicate function on index.js * fix tests * rename repository statuses * check if repository is being create when SSH request * fix lint * fix template * some improvements * fix template * unified migrate options * fix lint * fix loading page * refactor * When gitea restart, don't restart the running tasks because we may have servel gitea instances, that may break the migration * fix js * Update models/repo.go Co-Authored-By: guillep2k <18600385+guillep2k@users.noreply.github.com> * Update docs/content/doc/advanced/config-cheat-sheet.en-us.md Co-Authored-By: guillep2k <18600385+guillep2k@users.noreply.github.com> * fix tests * rename ErrTaskIsNotExist to ErrTaskDoesNotExist * delete release after add one on tests to make it run happy * fix tests * fix tests * improve codes * fix lint * fix lint * fix migrations --- .gitignore | 1 + custom/conf/app.ini.sample | 9 + .../doc/advanced/config-cheat-sheet.en-us.md | 7 + .../doc/advanced/config-cheat-sheet.zh-cn.md | 7 + models/fixtures/repository.yml | 43 +++- models/migrations/migrations.go | 2 + models/migrations/v99.go | 34 +++ models/models.go | 1 + models/repo.go | 85 ++++--- models/task.go | 240 ++++++++++++++++++ modules/context/repo.go | 35 +-- modules/migrations/base/options.go | 21 +- modules/migrations/gitea.go | 36 ++- modules/migrations/gitea_test.go | 7 +- modules/migrations/github.go | 4 +- modules/migrations/migrate.go | 12 +- modules/setting/setting.go | 1 + modules/setting/task.go | 25 ++ modules/structs/repo.go | 16 +- modules/structs/task.go | 34 +++ modules/task/migrate.go | 120 +++++++++ modules/task/queue.go | 14 + modules/task/queue_channel.go | 48 ++++ modules/task/queue_redis.go | 130 ++++++++++ modules/task/task.go | 66 +++++ options/locale/locale_en-US.ini | 2 + public/img/loading.png | Bin 0 -> 18713 bytes public/js/index.js | 36 +++ routers/api/v1/repo/repo.go | 4 +- routers/init.go | 4 + routers/private/serv.go | 9 + routers/repo/repo.go | 103 +++++--- routers/repo/view.go | 30 +++ routers/routes/routes.go | 2 + services/mirror/mirror_test.go | 29 ++- templates/repo/header.tmpl | 166 ++++++------ templates/repo/migrating.tmpl | 31 +++ 37 files changed, 1192 insertions(+), 222 deletions(-) create mode 100644 models/migrations/v99.go create mode 100644 models/task.go create mode 100644 modules/setting/task.go create mode 100644 modules/structs/task.go create mode 100644 modules/task/migrate.go create mode 100644 modules/task/queue.go create mode 100644 modules/task/queue_channel.go create mode 100644 modules/task/queue_redis.go create mode 100644 modules/task/task.go create mode 100644 public/img/loading.png create mode 100644 templates/repo/migrating.tmpl diff --git a/.gitignore b/.gitignore index fa6cbb454b..773b4726c0 100644 --- a/.gitignore +++ b/.gitignore @@ -77,3 +77,4 @@ prime/ *.snap *.snap-build *_source.tar.bz2 +.DS_Store \ No newline at end of file diff --git a/custom/conf/app.ini.sample b/custom/conf/app.ini.sample index 9bfddc97e8..dd14089d2b 100644 --- a/custom/conf/app.ini.sample +++ b/custom/conf/app.ini.sample @@ -808,3 +808,12 @@ IS_INPUT_FILE = false ENABLED = false ; If you want to add authorization, specify a token here TOKEN = + +[task] +; Task queue type, could be `channel` or `redis`. +QUEUE_TYPE = channel +; Task queue length, available only when `QUEUE_TYPE` is `channel`. +QUEUE_LENGTH = 1000 +; Task queue connction string, available only when `QUEUE_TYPE` is `redis`. +; If there is a password of redis, use `addrs=127.0.0.1:6379 password=123 db=0`. +QUEUE_CONN_STR = "addrs=127.0.0.1:6379 db=0" \ No newline at end of file diff --git a/docs/content/doc/advanced/config-cheat-sheet.en-us.md b/docs/content/doc/advanced/config-cheat-sheet.en-us.md index 198cff6f04..ed34be032b 100644 --- a/docs/content/doc/advanced/config-cheat-sheet.en-us.md +++ b/docs/content/doc/advanced/config-cheat-sheet.en-us.md @@ -514,9 +514,16 @@ Two special environment variables are passed to the render command: - `GITEA_PREFIX_RAW`, which contains the current URL prefix in the `raw` path tree. To be used as prefix for image paths. ## Time (`time`) + - `FORMAT`: Time format to diplay on UI. i.e. RFC1123 or 2006-01-02 15:04:05 - `DEFAULT_UI_LOCATION`: Default location of time on the UI, so that we can display correct user's time on UI. i.e. Shanghai/Asia +## Task (`task`) + +- `QUEUE_TYPE`: **channel**: Task queue type, could be `channel` or `redis`. +- `QUEUE_LENGTH`: **1000**: Task queue length, available only when `QUEUE_TYPE` is `channel`. +- `QUEUE_CONN_STR`: **addrs=127.0.0.1:6379 db=0**: Task queue connection string, available only when `QUEUE_TYPE` is `redis`. If there redis needs a password, use `addrs=127.0.0.1:6379 password=123 db=0`. + ## Other (`other`) - `SHOW_FOOTER_BRANDING`: **false**: Show Gitea branding in the footer. diff --git a/docs/content/doc/advanced/config-cheat-sheet.zh-cn.md b/docs/content/doc/advanced/config-cheat-sheet.zh-cn.md index 541d66f4e9..01ba821a47 100644 --- a/docs/content/doc/advanced/config-cheat-sheet.zh-cn.md +++ b/docs/content/doc/advanced/config-cheat-sheet.zh-cn.md @@ -241,9 +241,16 @@ IS_INPUT_FILE = false - IS_INPUT_FILE: 输入方式是最后一个参数为文件路径还是从标准输入读取。 ## Time (`time`) + - `FORMAT`: 显示在界面上的时间格式。比如： RFC1123 或者 2006-01-02 15:04:05 - `DEFAULT_UI_LOCATION`: 默认显示在界面上的时区，默认为本地时区。比如： Asia/Shanghai +## Task (`task`) + +- `QUEUE_TYPE`: **channel**: 任务队列类型，可以为 `channel` 或 `redis`。 +- `QUEUE_LENGTH`: **1000**: 任务队列长度，当 `QUEUE_TYPE` 为 `channel` 时有效。 +- `QUEUE_CONN_STR`: **addrs=127.0.0.1:6379 db=0**: 任务队列连接字符串，当 `QUEUE_TYPE` 为 `redis` 时有效。如果redis有密码，则可以 `addrs=127.0.0.1:6379 password=123 db=0`。 + ## Other (`other`) - `SHOW_FOOTER_BRANDING`: 为真则在页面底部显示Gitea的字样。 diff --git a/models/fixtures/repository.yml b/models/fixtures/repository.yml index 2e38c5e1dd..cf7d24c6cd 100644 --- a/models/fixtures/repository.yml +++ b/models/fixtures/repository.yml @@ -11,6 +11,7 @@ num_milestones: 3 num_closed_milestones: 1 num_watches: 3 + status: 0 - id: 2 @@ -24,6 +25,7 @@ num_closed_pulls: 0 num_stars: 1 close_issues_via_commit_in_any_branch: true + status: 0 - id: 3 @@ -36,6 +38,7 @@ num_pulls: 0 num_closed_pulls: 0 num_watches: 0 + status: 0 - id: 4 @@ -48,6 +51,7 @@ num_pulls: 0 num_closed_pulls: 0 num_stars: 1 + status: 0 - id: 5 @@ -61,6 +65,7 @@ num_closed_pulls: 0 num_watches: 0 is_mirror: true + status: 0 - id: 6 @@ -73,6 +78,7 @@ num_pulls: 0 num_closed_pulls: 0 is_mirror: false + status: 0 - id: 7 @@ -85,6 +91,7 @@ num_pulls: 0 num_closed_pulls: 0 is_mirror: false + status: 0 - id: 8 @@ -97,6 +104,7 @@ num_pulls: 0 num_closed_pulls: 0 is_mirror: false + status: 0 - id: 9 @@ -109,6 +117,7 @@ num_pulls: 0 num_closed_pulls: 0 is_mirror: false + status: 0 - id: 10 @@ -122,6 +131,7 @@ num_closed_pulls: 0 is_mirror: false num_forks: 1 + status: 0 - id: 11 @@ -135,6 +145,7 @@ num_pulls: 0 num_closed_pulls: 0 is_mirror: false + status: 0 - id: 12 @@ -147,6 +158,7 @@ num_pulls: 0 num_closed_pulls: 0 is_mirror: false + status: 0 - id: 13 @@ -159,6 +171,7 @@ num_pulls: 0 num_closed_pulls: 0 is_mirror: false + status: 0 - id: 14 @@ -172,6 +185,7 @@ num_pulls: 0 num_closed_pulls: 0 is_mirror: false + status: 0 - id: 15 @@ -179,6 +193,7 @@ lower_name: repo15 name: repo15 is_empty: true + status: 0 - id: 16 @@ -191,6 +206,7 @@ num_pulls: 0 num_closed_pulls: 0 num_watches: 0 + status: 0 - id: 17 @@ -205,6 +221,7 @@ num_watches: 0 is_mirror: false is_fork: false + status: 0 - id: 18 @@ -218,6 +235,7 @@ num_closed_pulls: 0 is_mirror: false is_fork: false + status: 0 - id: 19 @@ -231,6 +249,7 @@ num_closed_pulls: 0 is_mirror: false is_fork: false + status: 0 - id: 20 @@ -244,6 +263,7 @@ num_closed_pulls: 0 is_mirror: false is_fork: false + status: 0 - id: 21 @@ -257,6 +277,7 @@ num_closed_pulls: 0 is_mirror: false is_fork: false + status: 0 - id: 22 @@ -270,6 +291,7 @@ num_closed_pulls: 0 is_mirror: false is_fork: false + status: 0 - id: 23 @@ -283,6 +305,7 @@ num_closed_pulls: 0 is_mirror: false is_fork: false + status: 0 - id: 24 @@ -296,6 +319,7 @@ num_closed_pulls: 0 is_mirror: false is_fork: false + status: 0 - id: 25 @@ -310,6 +334,7 @@ num_watches: 0 is_mirror: true is_fork: false + status: 0 - id: 26 @@ -324,6 +349,7 @@ num_watches: 0 is_mirror: true is_fork: false + status: 0 - id: 27 @@ -339,6 +365,7 @@ is_mirror: true num_forks: 1 is_fork: false + status: 0 - id: 28 @@ -354,6 +381,7 @@ is_mirror: true num_forks: 1 is_fork: false + status: 0 - id: 29 @@ -368,6 +396,7 @@ num_closed_pulls: 0 is_mirror: false is_fork: true + status: 0 - id: 30 @@ -382,6 +411,7 @@ num_closed_pulls: 0 is_mirror: false is_fork: true + status: 0 - id: 31 @@ -392,6 +422,7 @@ num_forks: 0 num_issues: 0 is_mirror: false + status: 0 - id: 32 # org public repo @@ -403,6 +434,7 @@ num_forks: 0 num_issues: 0 is_mirror: false + status: 0 - id: 33 @@ -410,6 +442,7 @@ lower_name: utf8 name: utf8 is_private: false + status: 0 - id: 34 @@ -421,6 +454,7 @@ num_forks: 0 num_issues: 0 is_mirror: false + status: 0 - id: 35 @@ -432,6 +466,7 @@ num_forks: 0 num_issues: 0 is_mirror: false + status: 0 - id: 36 @@ -443,6 +478,7 @@ num_forks: 0 num_issues: 0 is_mirror: false + status: 0 - id: 37 @@ -454,6 +490,7 @@ num_forks: 0 num_issues: 0 is_mirror: false + status: 0 - id: 38 @@ -465,6 +502,7 @@ num_forks: 0 num_issues: 0 is_mirror: false + status: 0 - id: 39 @@ -476,6 +514,7 @@ num_forks: 0 num_issues: 0 is_mirror: false + status: 0 - id: 40 @@ -487,6 +526,7 @@ num_forks: 0 num_issues: 0 is_mirror: false + status: 0 - id: 41 @@ -519,4 +559,5 @@ num_stars: 0 num_forks: 0 num_issues: 0 - is_mirror: false \ No newline at end of file + is_mirror: false + status: 0 diff --git a/models/migrations/migrations.go b/models/migrations/migrations.go index e14437a04b..ef5cd377a6 100644 --- a/models/migrations/migrations.go +++ b/models/migrations/migrations.go @@ -252,6 +252,8 @@ var migrations = []Migration{ NewMigration("add repo_admin_change_team_access to user", addRepoAdminChangeTeamAccessColumnForUser), // v98 -> v99 NewMigration("add original author name and id on migrated release", addOriginalAuthorOnMigratedReleases), + // v99 -> v100 + NewMigration("add task table and status column for repository table", addTaskTable), } // Migrate database to current version diff --git a/models/migrations/v99.go b/models/migrations/v99.go new file mode 100644 index 0000000000..3eb287af6c --- /dev/null +++ b/models/migrations/v99.go @@ -0,0 +1,34 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package migrations + +import ( + "code.gitea.io/gitea/modules/structs" + "code.gitea.io/gitea/modules/timeutil" + + "github.com/go-xorm/xorm" +) + +func addTaskTable(x *xorm.Engine) error { + type Task struct { + ID int64 + DoerID int64 `xorm:"index"` // operator + OwnerID int64 `xorm:"index"` // repo owner id, when creating, the repoID maybe zero + RepoID int64 `xorm:"index"` + Type structs.TaskType + Status structs.TaskStatus `xorm:"index"` + StartTime timeutil.TimeStamp + EndTime timeutil.TimeStamp + PayloadContent string `xorm:"TEXT"` + Errors string `xorm:"TEXT"` // if task failed, saved the error reason + Created timeutil.TimeStamp `xorm:"created"` + } + + type Repository struct { + Status int `xorm:"NOT NULL DEFAULT 0"` + } + + return x.Sync2(new(Task), new(Repository)) +} diff --git a/models/models.go b/models/models.go index e802a35a77..ea550cb839 100644 --- a/models/models.go +++ b/models/models.go @@ -112,6 +112,7 @@ func init() { new(OAuth2Application), new(OAuth2AuthorizationCode), new(OAuth2Grant), + new(Task), ) gonicNames := []string{"SSL", "UID"} diff --git a/models/repo.go b/models/repo.go index 8db527477b..23b1c2ef52 100644 --- a/models/repo.go +++ b/models/repo.go @@ -126,6 +126,15 @@ func NewRepoContext() { RemoveAllWithNotice("Clean up repository temporary data", filepath.Join(setting.AppDataPath, "tmp")) } +// RepositoryStatus defines the status of repository +type RepositoryStatus int + +// all kinds of RepositoryStatus +const ( + RepositoryReady RepositoryStatus = iota // a normal repository + RepositoryBeingMigrated // repository is migrating +) + // Repository represents a git repository. type Repository struct { ID int64 `xorm:"pk autoincr"` @@ -156,9 +165,9 @@ type Repository struct { IsPrivate bool `xorm:"INDEX"` IsEmpty bool `xorm:"INDEX"` IsArchived bool `xorm:"INDEX"` - - IsMirror bool `xorm:"INDEX"` - *Mirror `xorm:"-"` + IsMirror bool `xorm:"INDEX"` + *Mirror `xorm:"-"` + Status RepositoryStatus `xorm:"NOT NULL DEFAULT 0"` ExternalMetas map[string]string `xorm:"-"` Units []*RepoUnit `xorm:"-"` @@ -197,6 +206,16 @@ func (repo *Repository) ColorFormat(s fmt.State) { repo.Name) } +// IsBeingMigrated indicates that repository is being migtated +func (repo *Repository) IsBeingMigrated() bool { + return repo.Status == RepositoryBeingMigrated +} + +// IsBeingCreated indicates that repository is being migrated or forked +func (repo *Repository) IsBeingCreated() bool { + return repo.IsBeingMigrated() +} + // AfterLoad is invoked from XORM after setting the values of all fields of this object. func (repo *Repository) AfterLoad() { // FIXME: use models migration to solve all at once. @@ -884,18 +903,6 @@ func (repo *Repository) CloneLink() (cl *CloneLink) { return repo.cloneLink(x, false) } -// MigrateRepoOptions contains the repository migrate options -type MigrateRepoOptions struct { - Name string - Description string - OriginalURL string - IsPrivate bool - IsMirror bool - RemoteAddr string - Wiki bool // include wiki repository - SyncReleasesWithTags bool // sync releases from tags -} - /* GitHub, GitLab, Gogs: *.wiki.git BitBucket: *.git/wiki @@ -915,20 +922,28 @@ func wikiRemoteURL(remote string) string { return "" } -// MigrateRepository migrates an existing repository from other project hosting. -func MigrateRepository(doer, u *User, opts MigrateRepoOptions) (*Repository, error) { - repo, err := CreateRepository(doer, u, CreateRepoOptions{ - Name: opts.Name, - Description: opts.Description, - OriginalURL: opts.OriginalURL, - IsPrivate: opts.IsPrivate, - IsMirror: opts.IsMirror, - }) - if err != nil { - return nil, err +// CheckCreateRepository check if could created a repository +func CheckCreateRepository(doer, u *User, name string) error { + if !doer.CanCreateRepo() { + return ErrReachLimitOfRepo{u.MaxRepoCreation} } - repoPath := RepoPath(u.Name, opts.Name) + if err := IsUsableRepoName(name); err != nil { + return err + } + + has, err := isRepositoryExist(x, u, name) + if err != nil { + return fmt.Errorf("IsRepositoryExist: %v", err) + } else if has { + return ErrRepoAlreadyExist{u.Name, name} + } + return nil +} + +// MigrateRepositoryGitData starts migrating git related data after created migrating repository +func MigrateRepositoryGitData(doer, u *User, repo *Repository, opts api.MigrateRepoOption) (*Repository, error) { + repoPath := RepoPath(u.Name, opts.RepoName) if u.IsOrganization() { t, err := u.GetOwnerTeam() @@ -942,11 +957,12 @@ func MigrateRepository(doer, u *User, opts MigrateRepoOptions) (*Repository, err migrateTimeout := time.Duration(setting.Git.Timeout.Migrate) * time.Second - if err := os.RemoveAll(repoPath); err != nil { + var err error + if err = os.RemoveAll(repoPath); err != nil { return repo, fmt.Errorf("Failed to remove %s: %v", repoPath, err) } - if err = git.Clone(opts.RemoteAddr, repoPath, git.CloneRepoOptions{ + if err = git.Clone(opts.CloneAddr, repoPath, git.CloneRepoOptions{ Mirror: true, Quiet: true, Timeout: migrateTimeout, @@ -955,8 +971,8 @@ func MigrateRepository(doer, u *User, opts MigrateRepoOptions) (*Repository, err } if opts.Wiki { - wikiPath := WikiPath(u.Name, opts.Name) - wikiRemotePath := wikiRemoteURL(opts.RemoteAddr) + wikiPath := WikiPath(u.Name, opts.RepoName) + wikiRemotePath := wikiRemoteURL(opts.CloneAddr) if len(wikiRemotePath) > 0 { if err := os.RemoveAll(wikiPath); err != nil { return repo, fmt.Errorf("Failed to remove %s: %v", wikiPath, err) @@ -986,7 +1002,7 @@ func MigrateRepository(doer, u *User, opts MigrateRepoOptions) (*Repository, err return repo, fmt.Errorf("git.IsEmpty: %v", err) } - if opts.SyncReleasesWithTags && !repo.IsEmpty { + if !opts.Releases && !repo.IsEmpty { // Try to get HEAD branch and set it as default branch. headBranch, err := gitRepo.GetHEADBranch() if err != nil { @@ -1005,7 +1021,7 @@ func MigrateRepository(doer, u *User, opts MigrateRepoOptions) (*Repository, err log.Error("Failed to update size for repository: %v", err) } - if opts.IsMirror { + if opts.Mirror { if _, err = x.InsertOne(&Mirror{ RepoID: repo.ID, Interval: setting.Mirror.DefaultInterval, @@ -1143,6 +1159,7 @@ type CreateRepoOptions struct { IsPrivate bool IsMirror bool AutoInit bool + Status RepositoryStatus } func getRepoInitFile(tp, name string) ([]byte, error) { @@ -1410,6 +1427,7 @@ func CreateRepository(doer, u *User, opts CreateRepoOptions) (_ *Repository, err IsPrivate: opts.IsPrivate, IsFsckEnabled: !opts.IsMirror, CloseIssuesViaCommitInAnyBranch: setting.Repository.DefaultCloseIssuesViaCommitsInAnyBranch, + Status: opts.Status, } sess := x.NewSession() @@ -1856,6 +1874,7 @@ func DeleteRepository(doer *User, uid, repoID int64) error { &CommitStatus{RepoID: repoID}, &RepoIndexerStatus{RepoID: repoID}, &Comment{RefRepoID: repoID}, + &Task{RepoID: repoID}, ); err != nil { return fmt.Errorf("deleteBeans: %v", err) } diff --git a/models/task.go b/models/task.go new file mode 100644 index 0000000000..cb878d387c --- /dev/null +++ b/models/task.go @@ -0,0 +1,240 @@ +// Copyright 2019 Gitea. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package models + +import ( + "encoding/json" + "fmt" + + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/migrations/base" + "code.gitea.io/gitea/modules/structs" + "code.gitea.io/gitea/modules/timeutil" + + "xorm.io/builder" +) + +// Task represents a task +type Task struct { + ID int64 + DoerID int64 `xorm:"index"` // operator + Doer *User `xorm:"-"` + OwnerID int64 `xorm:"index"` // repo owner id, when creating, the repoID maybe zero + Owner *User `xorm:"-"` + RepoID int64 `xorm:"index"` + Repo *Repository `xorm:"-"` + Type structs.TaskType + Status structs.TaskStatus `xorm:"index"` + StartTime timeutil.TimeStamp + EndTime timeutil.TimeStamp + PayloadContent string `xorm:"TEXT"` + Errors string `xorm:"TEXT"` // if task failed, saved the error reason + Created timeutil.TimeStamp `xorm:"created"` +} + +// LoadRepo loads repository of the task +func (task *Task) LoadRepo() error { + return task.loadRepo(x) +} + +func (task *Task) loadRepo(e Engine) error { + if task.Repo != nil { + return nil + } + var repo Repository + has, err := e.ID(task.RepoID).Get(&repo) + if err != nil { + return err + } else if !has { + return ErrRepoNotExist{ + ID: task.RepoID, + } + } + task.Repo = &repo + return nil +} + +// LoadDoer loads do user +func (task *Task) LoadDoer() error { + if task.Doer != nil { + return nil + } + + var doer User + has, err := x.ID(task.DoerID).Get(&doer) + if err != nil { + return err + } else if !has { + return ErrUserNotExist{ + UID: task.DoerID, + } + } + task.Doer = &doer + + return nil +} + +// LoadOwner loads owner user +func (task *Task) LoadOwner() error { + if task.Owner != nil { + return nil + } + + var owner User + has, err := x.ID(task.OwnerID).Get(&owner) + if err != nil { + return err + } else if !has { + return ErrUserNotExist{ + UID: task.OwnerID, + } + } + task.Owner = &owner + + return nil +} + +// UpdateCols updates some columns +func (task *Task) UpdateCols(cols ...string) error { + _, err := x.ID(task.ID).Cols(cols...).Update(task) + return err +} + +// MigrateConfig returns task config when migrate repository +func (task *Task) MigrateConfig() (*structs.MigrateRepoOption, error) { + if task.Type == structs.TaskTypeMigrateRepo { + var opts structs.MigrateRepoOption + err := json.Unmarshal([]byte(task.PayloadContent), &opts) + if err != nil { + return nil, err + } + return &opts, nil + } + return nil, fmt.Errorf("Task type is %s, not Migrate Repo", task.Type.Name()) +} + +// ErrTaskDoesNotExist represents a "TaskDoesNotExist" kind of error. +type ErrTaskDoesNotExist struct { + ID int64 + RepoID int64 + Type structs.TaskType +} + +// IsErrTaskDoesNotExist checks if an error is a ErrTaskIsNotExist. +func IsErrTaskDoesNotExist(err error) bool { + _, ok := err.(ErrTaskDoesNotExist) + return ok +} + +func (err ErrTaskDoesNotExist) Error() string { + return fmt.Sprintf("task is not exist [id: %d, repo_id: %d, type: %d]", + err.ID, err.RepoID, err.Type) +} + +// GetMigratingTask returns the migrating task by repo's id +func GetMigratingTask(repoID int64) (*Task, error) { + var task = Task{ + RepoID: repoID, + Type: structs.TaskTypeMigrateRepo, + } + has, err := x.Get(&task) + if err != nil { + return nil, err + } else if !has { + return nil, ErrTaskDoesNotExist{0, repoID, task.Type} + } + return &task, nil +} + +// FindTaskOptions find all tasks +type FindTaskOptions struct { + Status int +} + +// ToConds generates conditions for database operation. +func (opts FindTaskOptions) ToConds() builder.Cond { + var cond = builder.NewCond() + if opts.Status >= 0 { + cond = cond.And(builder.Eq{"status": opts.Status}) + } + return cond +} + +// FindTasks find all tasks +func FindTasks(opts FindTaskOptions) ([]*Task, error) { + var tasks = make([]*Task, 0, 10) + err := x.Where(opts.ToConds()).Find(&tasks) + return tasks, err +} + +func createTask(e Engine, task *Task) error { + _, err := e.Insert(task) + return err +} + +// CreateMigrateTask creates a migrate task +func CreateMigrateTask(doer, u *User, opts base.MigrateOptions) (*Task, error) { + bs, err := json.Marshal(&opts) + if err != nil { + return nil, err + } + + var task = Task{ + DoerID: doer.ID, + OwnerID: u.ID, + Type: structs.TaskTypeMigrateRepo, + Status: structs.TaskStatusQueue, + PayloadContent: string(bs), + } + + if err := createTask(x, &task); err != nil { + return nil, err + } + + repo, err := CreateRepository(doer, u, CreateRepoOptions{ + Name: opts.RepoName, + Description: opts.Description, + OriginalURL: opts.CloneAddr, + IsPrivate: opts.Private, + IsMirror: opts.Mirror, + Status: RepositoryBeingMigrated, + }) + if err != nil { + task.EndTime = timeutil.TimeStampNow() + task.Status = structs.TaskStatusFailed + err2 := task.UpdateCols("end_time", "status") + if err2 != nil { + log.Error("UpdateCols Failed: %v", err2.Error()) + } + return nil, err + } + + task.RepoID = repo.ID + if err = task.UpdateCols("repo_id"); err != nil { + return nil, err + } + + return &task, nil +} + +// FinishMigrateTask updates database when migrate task finished +func FinishMigrateTask(task *Task) error { + task.Status = structs.TaskStatusFinished + task.EndTime = timeutil.TimeStampNow() + sess := x.NewSession() + defer sess.Close() + if err := sess.Begin(); err != nil { + return err + } + if _, err := sess.ID(task.ID).Cols("status", "end_time").Update(task); err != nil { + return err + } + task.Repo.Status = RepositoryReady + if _, err := sess.ID(task.RepoID).Cols("status").Update(task.Repo); err != nil { + return err + } + + return sess.Commit() +} diff --git a/modules/context/repo.go b/modules/context/repo.go index 3caf583f83..f4af19a0e8 100644 --- a/modules/context/repo.go +++ b/modules/context/repo.go @@ -146,6 +146,9 @@ func (r *Repository) FileExists(path string, branch string) (bool, error) { // GetEditorconfig returns the .editorconfig definition if found in the // HEAD of the default repo branch. func (r *Repository) GetEditorconfig() (*editorconfig.Editorconfig, error) { + if r.GitRepo == nil { + return nil, nil + } commit, err := r.GitRepo.GetBranchCommit(r.Repository.DefaultBranch) if err != nil { return nil, err @@ -358,12 +361,6 @@ func RepoAssignment() macaron.Handler { return } - gitRepo, err := git.OpenRepository(models.RepoPath(userName, repoName)) - if err != nil { - ctx.ServerError("RepoAssignment Invalid repo "+models.RepoPath(userName, repoName), err) - return - } - ctx.Repo.GitRepo = gitRepo ctx.Repo.RepoLink = repo.Link() ctx.Data["RepoLink"] = ctx.Repo.RepoLink ctx.Data["RepoRelPath"] = ctx.Repo.Owner.Name + "/" + ctx.Repo.Repository.Name @@ -373,13 +370,6 @@ func RepoAssignment() macaron.Handler { ctx.Data["RepoExternalIssuesLink"] = unit.ExternalTrackerConfig().ExternalTrackerURL } - tags, err := ctx.Repo.GitRepo.GetTags() - if err != nil { - ctx.ServerError("GetTags", err) - return - } - ctx.Data["Tags"] = tags - count, err := models.GetReleaseCountByRepoID(ctx.Repo.Repository.ID, models.FindReleasesOptions{ IncludeDrafts: false, IncludeTags: true, @@ -425,12 +415,25 @@ func RepoAssignment() macaron.Handler { } // repo is empty and display enable - if ctx.Repo.Repository.IsEmpty { + if ctx.Repo.Repository.IsEmpty || ctx.Repo.Repository.IsBeingCreated() { ctx.Data["BranchName"] = ctx.Repo.Repository.DefaultBranch return } - ctx.Data["TagName"] = ctx.Repo.TagName + gitRepo, err := git.OpenRepository(models.RepoPath(userName, repoName)) + if err != nil { + ctx.ServerError("RepoAssignment Invalid repo "+models.RepoPath(userName, repoName), err) + return + } + ctx.Repo.GitRepo = gitRepo + + tags, err := ctx.Repo.GitRepo.GetTags() + if err != nil { + ctx.ServerError("GetTags", err) + return + } + ctx.Data["Tags"] = tags + brs, err := ctx.Repo.GitRepo.GetBranches() if err != nil { ctx.ServerError("GetBranches", err) @@ -439,6 +442,8 @@ func RepoAssignment() macaron.Handler { ctx.Data["Branches"] = brs ctx.Data["BranchesCount"] = len(brs) + ctx.Data["TagName"] = ctx.Repo.TagName + // If not branch selected, try default one. // If default branch doesn't exists, fall back to some other branch. if len(ctx.Repo.BranchName) == 0 { diff --git a/modules/migrations/base/options.go b/modules/migrations/base/options.go index ba7fdc6815..2d180b61d9 100644 --- a/modules/migrations/base/options.go +++ b/modules/migrations/base/options.go @@ -5,22 +5,7 @@ package base -// MigrateOptions defines the way a repository gets migrated -type MigrateOptions struct { - RemoteURL string - AuthUsername string - AuthPassword string - Name string - Description string - OriginalURL string +import "code.gitea.io/gitea/modules/structs" - Wiki bool - Issues bool - Milestones bool - Labels bool - Releases bool - Comments bool - PullRequests bool - Private bool - Mirror bool -} +// MigrateOptions defines the way a repository gets migrated +type MigrateOptions = structs.MigrateRepoOption diff --git a/modules/migrations/gitea.go b/modules/migrations/gitea.go index 1edac47a6e..ab3b0b9f69 100644 --- a/modules/migrations/gitea.go +++ b/modules/migrations/gitea.go @@ -22,6 +22,7 @@ import ( "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/migrations/base" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/structs" "code.gitea.io/gitea/modules/timeutil" gouuid "github.com/satori/go.uuid" @@ -90,16 +91,33 @@ func (g *GiteaLocalUploader) CreateRepo(repo *base.Repository, opts base.Migrate remoteAddr = u.String() } - r, err := models.MigrateRepository(g.doer, owner, models.MigrateRepoOptions{ - Name: g.repoName, - Description: repo.Description, - OriginalURL: repo.OriginalURL, - IsMirror: repo.IsMirror, - RemoteAddr: remoteAddr, - IsPrivate: repo.IsPrivate, - Wiki: opts.Wiki, - SyncReleasesWithTags: !opts.Releases, // if didn't get releases, then sync them from tags + var r *models.Repository + if opts.MigrateToRepoID <= 0 { + r, err = models.CreateRepository(g.doer, owner, models.CreateRepoOptions{ + Name: g.repoName, + Description: repo.Description, + OriginalURL: repo.OriginalURL, + IsPrivate: opts.Private, + IsMirror: opts.Mirror, + Status: models.RepositoryBeingMigrated, + }) + } else { + r, err = models.GetRepositoryByID(opts.MigrateToRepoID) + } + if err != nil { + return err + } + + r, err = models.MigrateRepositoryGitData(g.doer, owner, r, structs.MigrateRepoOption{ + RepoName: g.repoName, + Description: repo.Description, + Mirror: repo.IsMirror, + CloneAddr: remoteAddr, + Private: repo.IsPrivate, + Wiki: opts.Wiki, + Releases: opts.Releases, // if didn't get releases, then sync them from tags }) + g.repo = r if err != nil { return err diff --git a/modules/migrations/gitea_test.go b/modules/migrations/gitea_test.go index 88a3a6d218..73c119a15d 100644 --- a/modules/migrations/gitea_test.go +++ b/modules/migrations/gitea_test.go @@ -10,6 +10,7 @@ import ( "time" "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/structs" "code.gitea.io/gitea/modules/util" "github.com/stretchr/testify/assert" @@ -29,9 +30,9 @@ func TestGiteaUploadRepo(t *testing.T) { uploader = NewGiteaLocalUploader(user, user.Name, repoName) ) - err := migrateRepository(downloader, uploader, MigrateOptions{ - RemoteURL: "https://github.com/go-xorm/builder", - Name: repoName, + err := migrateRepository(downloader, uploader, structs.MigrateRepoOption{ + CloneAddr: "https://github.com/go-xorm/builder", + RepoName: repoName, AuthUsername: "", Wiki: true, diff --git a/modules/migrations/github.go b/modules/migrations/github.go index 754f98941c..1c5d96c03d 100644 --- a/modules/migrations/github.go +++ b/modules/migrations/github.go @@ -34,7 +34,7 @@ type GithubDownloaderV3Factory struct { // Match returns ture if the migration remote URL matched this downloader factory func (f *GithubDownloaderV3Factory) Match(opts base.MigrateOptions) (bool, error) { - u, err := url.Parse(opts.RemoteURL) + u, err := url.Parse(opts.CloneAddr) if err != nil { return false, err } @@ -44,7 +44,7 @@ func (f *GithubDownloaderV3Factory) Match(opts base.MigrateOptions) (bool, error // New returns a Downloader related to this factory according MigrateOptions func (f *GithubDownloaderV3Factory) New(opts base.MigrateOptions) (base.Downloader, error) { - u, err := url.Parse(opts.RemoteURL) + u, err := url.Parse(opts.CloneAddr) if err != nil { return nil, err } diff --git a/modules/migrations/migrate.go b/modules/migrations/migrate.go index 27782cb940..3f5c0d1118 100644 --- a/modules/migrations/migrate.go +++ b/modules/migrations/migrate.go @@ -6,6 +6,8 @@ package migrations import ( + "fmt" + "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/migrations/base" @@ -27,7 +29,7 @@ func RegisterDownloaderFactory(factory base.DownloaderFactory) { func MigrateRepository(doer *models.User, ownerName string, opts base.MigrateOptions) (*models.Repository, error) { var ( downloader base.Downloader - uploader = NewGiteaLocalUploader(doer, ownerName, opts.Name) + uploader = NewGiteaLocalUploader(doer, ownerName, opts.RepoName) ) for _, factory := range factories { @@ -50,14 +52,18 @@ func MigrateRepository(doer *models.User, ownerName string, opts base.MigrateOpt opts.Comments = false opts.Issues = false opts.PullRequests = false - downloader = NewPlainGitDownloader(ownerName, opts.Name, opts.RemoteURL) - log.Trace("Will migrate from git: %s", opts.RemoteURL) + downloader = NewPlainGitDownloader(ownerName, opts.RepoName, opts.CloneAddr) + log.Trace("Will migrate from git: %s", opts.CloneAddr) } if err := migrateRepository(downloader, uploader, opts); err != nil { if err1 := uploader.Rollback(); err1 != nil { log.Error("rollback failed: %v", err1) } + + if err2 := models.CreateRepositoryNotice(fmt.Sprintf("Migrate repository from %s failed: %v", opts.CloneAddr, err)); err2 != nil { + log.Error("create respotiry notice failed: ", err2) + } return nil, err } diff --git a/modules/setting/setting.go b/modules/setting/setting.go index 5e476854b2..8c61bdbb77 100644 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -1043,4 +1043,5 @@ func NewServices() { newNotifyMailService() newWebhookService() newIndexerService() + newTaskService() } diff --git a/modules/setting/task.go b/modules/setting/task.go new file mode 100644 index 0000000000..97704d4a4d --- /dev/null +++ b/modules/setting/task.go @@ -0,0 +1,25 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package setting + +var ( + // Task settings + Task = struct { + QueueType string + QueueLength int + QueueConnStr string + }{ + QueueType: ChannelQueueType, + QueueLength: 1000, + QueueConnStr: "addrs=127.0.0.1:6379 db=0", + } +) + +func newTaskService() { + sec := Cfg.Section("task") + Task.QueueType = sec.Key("QUEUE_TYPE").MustString(ChannelQueueType) + Task.QueueLength = sec.Key("QUEUE_LENGTH").MustInt(1000) + Task.QueueConnStr = sec.Key("QUEUE_CONN_STR").MustString("addrs=127.0.0.1:6379 db=0") +} diff --git a/modules/structs/repo.go b/modules/structs/repo.go index 87396d6ce9..57f1768a0b 100644 --- a/modules/structs/repo.go +++ b/modules/structs/repo.go @@ -162,8 +162,16 @@ type MigrateRepoOption struct { // required: true UID int `json:"uid" binding:"Required"` // required: true - RepoName string `json:"repo_name" binding:"Required"` - Mirror bool `json:"mirror"` - Private bool `json:"private"` - Description string `json:"description"` + RepoName string `json:"repo_name" binding:"Required"` + Mirror bool `json:"mirror"` + Private bool `json:"private"` + Description string `json:"description"` + Wiki bool + Issues bool + Milestones bool + Labels bool + Releases bool + Comments bool + PullRequests bool + MigrateToRepoID int64 } diff --git a/modules/structs/task.go b/modules/structs/task.go new file mode 100644 index 0000000000..e83d0437ce --- /dev/null +++ b/modules/structs/task.go @@ -0,0 +1,34 @@ +// Copyright 2019 Gitea. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package structs + +// TaskType defines task type +type TaskType int + +// all kinds of task types +const ( + TaskTypeMigrateRepo TaskType = iota // migrate repository from external or local disk +) + +// Name returns the task type name +func (taskType TaskType) Name() string { + switch taskType { + case TaskTypeMigrateRepo: + return "Migrate Repository" + } + return "" +} + +// TaskStatus defines task status +type TaskStatus int + +// enumerate all the kinds of task status +const ( + TaskStatusQueue TaskStatus = iota // 0 task is queue + TaskStatusRunning // 1 task is running + TaskStatusStopped // 2 task is stopped + TaskStatusFailed // 3 task is failed + TaskStatusFinished // 4 task is finished +) diff --git a/modules/task/migrate.go b/modules/task/migrate.go new file mode 100644 index 0000000000..5d15a506d7 --- /dev/null +++ b/modules/task/migrate.go @@ -0,0 +1,120 @@ +// Copyright 2019 Gitea. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package task + +import ( + "bytes" + "errors" + "fmt" + "strings" + + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/migrations" + "code.gitea.io/gitea/modules/notification" + "code.gitea.io/gitea/modules/structs" + "code.gitea.io/gitea/modules/timeutil" + "code.gitea.io/gitea/modules/util" +) + +func handleCreateError(owner *models.User, err error, name string) error { + switch { + case models.IsErrReachLimitOfRepo(err): + return fmt.Errorf("You have already reached your limit of %d repositories", owner.MaxCreationLimit()) + case models.IsErrRepoAlreadyExist(err): + return errors.New("The repository name is already used") + case models.IsErrNameReserved(err): + return fmt.Errorf("The repository name '%s' is reserved", err.(models.ErrNameReserved).Name) + case models.IsErrNamePatternNotAllowed(err): + return fmt.Errorf("The pattern '%s' is not allowed in a repository name", err.(models.ErrNamePatternNotAllowed).Pattern) + default: + return err + } +} + +func runMigrateTask(t *models.Task) (err error) { + defer func() { + if e := recover(); e != nil { + var buf bytes.Buffer + fmt.Fprintf(&buf, "Handler crashed with error: %v", log.Stack(2)) + + err = errors.New(buf.String()) + } + + if err == nil { + err = models.FinishMigrateTask(t) + if err == nil { + notification.NotifyMigrateRepository(t.Doer, t.Owner, t.Repo) + return + } + + log.Error("FinishMigrateTask failed: %s", err.Error()) + } + + t.EndTime = timeutil.TimeStampNow() + t.Status = structs.TaskStatusFailed + t.Errors = err.Error() + if err := t.UpdateCols("status", "errors", "end_time"); err != nil { + log.Error("Task UpdateCols failed: %s", err.Error()) + } + + if t.Repo != nil { + if errDelete := models.DeleteRepository(t.Doer, t.OwnerID, t.Repo.ID); errDelete != nil { + log.Error("DeleteRepository: %v", errDelete) + } + } + }() + + if err := t.LoadRepo(); err != nil { + return err + } + + // if repository is ready, then just finsih the task + if t.Repo.Status == models.RepositoryReady { + return nil + } + + if err := t.LoadDoer(); err != nil { + return err + } + if err := t.LoadOwner(); err != nil { + return err + } + t.StartTime = timeutil.TimeStampNow() + t.Status = structs.TaskStatusRunning + if err := t.UpdateCols("start_time", "status"); err != nil { + return err + } + + var opts *structs.MigrateRepoOption + opts, err = t.MigrateConfig() + if err != nil { + return err + } + + opts.MigrateToRepoID = t.RepoID + repo, err := migrations.MigrateRepository(t.Doer, t.Owner.Name, *opts) + if err == nil { + notification.NotifyMigrateRepository(t.Doer, t.Owner, repo) + + log.Trace("Repository migrated [%d]: %s/%s", repo.ID, t.Owner.Name, repo.Name) + return nil + } + + if models.IsErrRepoAlreadyExist(err) { + return errors.New("The repository name is already used") + } + + // remoteAddr may contain credentials, so we sanitize it + err = util.URLSanitizedError(err, opts.CloneAddr) + if strings.Contains(err.Error(), "Authentication failed") || + strings.Contains(err.Error(), "could not read Username") { + return fmt.Errorf("Authentication failed: %v", err.Error()) + } else if strings.Contains(err.Error(), "fatal:") { + return fmt.Errorf("Migration failed: %v", err.Error()) + } + + return handleCreateError(t.Owner, err, "MigratePost") +} diff --git a/modules/task/queue.go b/modules/task/queue.go new file mode 100644 index 0000000000..ddee0b3d46 --- /dev/null +++ b/modules/task/queue.go @@ -0,0 +1,14 @@ +// Copyright 2019 Gitea. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package task + +import "code.gitea.io/gitea/models" + +// Queue defines an interface to run task queue +type Queue interface { + Run() error + Push(*models.Task) error + Stop() +} diff --git a/modules/task/queue_channel.go b/modules/task/queue_channel.go new file mode 100644 index 0000000000..da541f4755 --- /dev/null +++ b/modules/task/queue_channel.go @@ -0,0 +1,48 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package task + +import ( + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/log" +) + +var ( + _ Queue = &ChannelQueue{} +) + +// ChannelQueue implements +type ChannelQueue struct { + queue chan *models.Task +} + +// NewChannelQueue create a memory channel queue +func NewChannelQueue(queueLen int) *ChannelQueue { + return &ChannelQueue{ + queue: make(chan *models.Task, queueLen), + } +} + +// Run starts to run the queue +func (c *ChannelQueue) Run() error { + for task := range c.queue { + err := Run(task) + if err != nil { + log.Error("Run task failed: %s", err.Error()) + } + } + return nil +} + +// Push will push the task ID to queue +func (c *ChannelQueue) Push(task *models.Task) error { + c.queue <- task + return nil +} + +// Stop stop the queue +func (c *ChannelQueue) Stop() { + close(c.queue) +} diff --git a/modules/task/queue_redis.go b/modules/task/queue_redis.go new file mode 100644 index 0000000000..127de0cdbf --- /dev/null +++ b/modules/task/queue_redis.go @@ -0,0 +1,130 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package task + +import ( + "encoding/json" + "errors" + "strconv" + "strings" + "time" + + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/log" + + "github.com/go-redis/redis" +) + +var ( + _ Queue = &RedisQueue{} +) + +type redisClient interface { + RPush(key string, args ...interface{}) *redis.IntCmd + LPop(key string) *redis.StringCmd + Ping() *redis.StatusCmd +} + +// RedisQueue redis queue +type RedisQueue struct { + client redisClient + queueName string + closeChan chan bool +} + +func parseConnStr(connStr string) (addrs, password string, dbIdx int, err error) { + fields := strings.Fields(connStr) + for _, f := range fields { + items := strings.SplitN(f, "=", 2) + if len(items) < 2 { + continue + } + switch strings.ToLower(items[0]) { + case "addrs": + addrs = items[1] + case "password": + password = items[1] + case "db": + dbIdx, err = strconv.Atoi(items[1]) + if err != nil { + return + } + } + } + return +} + +// NewRedisQueue creates single redis or cluster redis queue +func NewRedisQueue(addrs string, password string, dbIdx int) (*RedisQueue, error) { + dbs := strings.Split(addrs, ",") + var queue = RedisQueue{ + queueName: "task_queue", + closeChan: make(chan bool), + } + if len(dbs) == 0 { + return nil, errors.New("no redis host found") + } else if len(dbs) == 1 { + queue.client = redis.NewClient(&redis.Options{ + Addr: strings.TrimSpace(dbs[0]), // use default Addr + Password: password, // no password set + DB: dbIdx, // use default DB + }) + } else { + // cluster will ignore db + queue.client = redis.NewClusterClient(&redis.ClusterOptions{ + Addrs: dbs, + Password: password, + }) + } + if err := queue.client.Ping().Err(); err != nil { + return nil, err + } + return &queue, nil +} + +// Run starts to run the queue +func (r *RedisQueue) Run() error { + for { + select { + case <-r.closeChan: + return nil + case <-time.After(time.Millisecond * 100): + } + + bs, err := r.client.LPop(r.queueName).Bytes() + if err != nil { + if err != redis.Nil { + log.Error("LPop failed: %v", err) + } + time.Sleep(time.Millisecond * 100) + continue + } + + var task models.Task + err = json.Unmarshal(bs, &task) + if err != nil { + log.Error("Unmarshal task failed: %s", err.Error()) + } else { + err = Run(&task) + if err != nil { + log.Error("Run task failed: %s", err.Error()) + } + } + } +} + +// Push implements Queue +func (r *RedisQueue) Push(task *models.Task) error { + bs, err := json.Marshal(task) + if err != nil { + return err + } + return r.client.RPush(r.queueName, bs).Err() +} + +// Stop stop the queue +func (r *RedisQueue) Stop() { + r.closeChan <- true +} diff --git a/modules/task/task.go b/modules/task/task.go new file mode 100644 index 0000000000..64744afe7a --- /dev/null +++ b/modules/task/task.go @@ -0,0 +1,66 @@ +// Copyright 2019 Gitea. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package task + +import ( + "fmt" + + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/migrations/base" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/structs" +) + +// taskQueue is a global queue of tasks +var taskQueue Queue + +// Run a task +func Run(t *models.Task) error { + switch t.Type { + case structs.TaskTypeMigrateRepo: + return runMigrateTask(t) + default: + return fmt.Errorf("Unknow task type: %d", t.Type) + } +} + +// Init will start the service to get all unfinished tasks and run them +func Init() error { + switch setting.Task.QueueType { + case setting.ChannelQueueType: + taskQueue = NewChannelQueue(setting.Task.QueueLength) + case setting.RedisQueueType: + var err error + addrs, pass, idx, err := parseConnStr(setting.Task.QueueConnStr) + if err != nil { + return err + } + taskQueue, err = NewRedisQueue(addrs, pass, idx) + if err != nil { + return err + } + default: + return fmt.Errorf("Unsupported task queue type: %v", setting.Task.QueueType) + } + + go func() { + if err := taskQueue.Run(); err != nil { + log.Error("taskQueue.Run end failed: %v", err) + } + }() + + return nil +} + +// MigrateRepository add migration repository to task +func MigrateRepository(doer, u *models.User, opts base.MigrateOptions) error { + task, err := models.CreateMigrateTask(doer, u, opts) + if err != nil { + return err + } + + return taskQueue.Push(task) +} diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index ca09b6120d..e6c5839a64 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -633,6 +633,8 @@ migrate.lfs_mirror_unsupported = Mirroring LFS objects is not supported - use 'g migrate.migrate_items_options = When migrating from github, input a username and migration options will be displayed. migrated_from = Migrated from %[2]s migrated_from_fake = Migrated From %[1]s +migrate.migrating = Migrating from %s ... +migrate.migrating_failed = Migrating from %s failed. mirror_from = mirror of forked_from = forked from diff --git a/public/img/loading.png b/public/img/loading.png new file mode 100644 index 0000000000000000000000000000000000000000..aac702cfd6d010abb22f4ebd8f011b606075ef62 GIT binary patch literal 18713 zcmeAS@N?(olHy`uVBq!ia0y~yVDw>NU^vIY%)r2qwEr|nf+I0G1SH79z`(#zb2jY) z0|Ud20G|+71_lPtX_|#=LeiIerYv;{nr-YeL)UGpTILGxyjA|;b4{Y=Tf{7|N?Y!e zyu{gmre4;npqK^LzB6SKz>uR@VP28Q_wLLOyp7ruGGqY9&uR7H^ z>tMm5TdPVo#6A6RZ|}7Qn=edVcVR)>uHpqJCzYB!tzckaP$&uV3ubuEtM0%5`Zq_Z z`OWDCrW>LvPHwm`X<>JL&$QXQw#BTIT=(uY0|SFp8YGMuU;zjUD==ncU}0dGA1HKz zfq{RTr;B4q#jQ8DZuiYL5NJr0E;dnb+#(*hP4iVx*w&P-{1?90ufNrr!V$@np>H6% z|69`1m^I%nTL>%UWIN7!c0K#ZWER26Eypi@t(sanaapFSQQ5X@fg+opa`X!{I(18L z&y~KK->T{3nvncIvQbw!Q$nm(^!A*_CmsA|Yqth*tk}pu@9@2AK6?Gx#=EBP(Y&;3 zRm%y!CD~_OQoXv>_Q$;8>`6LTIZ5RPdqVxKg4rpOcV&AQzG@SAw||vguF{>%4DTJX z_f|}+EZlwf&;_1?ZJ!w~uW2<8{;Bp_qbmRQ4E_v*+aHql{mU0MZ8~XH=AU`t?h?zG zmg@oqAO42RTosR=XP%na)4u7=>PyqKgq=gB#8M|e7oXkpX2s1lv%UG`Gbb;yS}N$U zPUh4c{z)#bGj6Q)5!AoO9Nyt^rSMPQ>fcM=zST8<<@<4czVrRh-CM5KzTal1rX{`O zu%+4Y|7{FMrMx;-eoiV%Sod`)=dy(4`Z}J!ZJ`f?tE{qbJ0HkVT{`P&!JcoSDMi~B zt`<~d3TknU;oGFYM2x>`g78xxhE=x>uO9yY=j(wG`MZ3&{b?s+r))X*{e82+?zsk2 zix;yCbzW**x%=;DhRv4`+>Fsw_xfH^{`)!eGA0q@X}pKD)M}IN&9;8t*fMpt>)aV7 z9?IW5BW)}n{aaVuZ(#bNVXw-M&w}6U=TVLdr~ZthO~)-G*U7&7d*7t?V9(9qizhx=yuQPK=B4R`mbJ^ner`Co_Xy9) zPXVv*@cXcGE;eF$@vCp^x!Qv+Zd?1m?{8jP%x~z;u=nk@HD(LhU*EAmlk>GPwCc}{ zW6iuz}_1BR(K51MF#EuCq zO_*P5Yp6b1lF{lc3 z#Tl~742-u$lqK=a*!uWU)XnM7KfUZ*razmPjE{tZdR+2{-APer?|m(=bs#> z9{=KhACNUy*N1)QpS@!Lw-?JNJmY)xK5^awrzw{Jfl&<73=Bj%(BcA_TS)WVG`>eVu*2@Q7 zT2|&eB%1hKbUBo@?_8JC?K9dcU)a;$Jy5e)G@D=kl1fa<%lg!AS-(#VYCB%~AKW9k zfLSp#ddiwd%<5{b>;}h$d=yV7PQ2$7Wq8crLU8&_-7`Wl8V7u~9s8Qb?o{Qu|I+JK z2RmK9-V#f@Qc!VRL1%W<*L#gOrvJ8(Zt@7py>x|ZS@Zt4S`l2|XGYn6;hDT@OZ55E zx=xF-@|Y&i6?fFUyXALbVN1x8rmQ=)IvWG|3RLATZ0bBwX|(OKQ_{|bVoNj4K2N(Z zV`cZi^P-DLZ)B}Rk(-%C<@Y{oO$MK@2I_~F+}vzHjpt zdK>pk5|iaFU*THbT-_I5miFYzu6Ko(mMlFrr-&NdK>a?ifjq}AT(wncQ zCCs@KXR)I+<>DUkgm0N=+;7RdTxWjtKl~pV{x$J<+80 z?~1GV4b>j6$z0&s8EvZG-nvUlA|-C>u{HJy>-ImeVeor)TAeV0R|HgkUysy=q|H@`a5(hnO_H%-vGI73%+*2nog1)mtjgd5kr{b_d6 z>D?vSD&4YUcRG%U|9$jicKps8PZ`X!W|YU&up6XpTvhhn#lkyPbnl}x2V+-u&3#;y zAaGo)$I{*DNK0>K<78pBPt1JLpY;q*=AS&aCeyxIVRgjU*3B&w7u_hlYH}jKGq%c; zlY4UbOl5nquT7T^xyLj7_56;lv;{I3j>g$*CmEo6Xn+doW z#;aav2wK@5r*%2D{}@M3d=4j{0PDa1`C2Bq%N4KWpM4jgUVHpn@0IE1*7DZIyH)wT zo~>~VNbZ%iWOw&AP2{?^{zOl!kGPGf+#{CI_=M*x8RcR%Egmr|1UoEBG=DRrq|axg z|DNwG({BfxCUOa{Kl#M>(v-6W?#q(-8)arco*mxzgJBlmxoUIq-Wxg*il<6G9Tsz& z{jWOi?)igrUL=JVDwnS75b5)|p1Wl7Z8gUl2D4Y*&l^kodDkP(s~drU=`*xx1e? zz2q+}+xpis@SW0`;$%NAwZ!0l9~&K2f1?A-Sr*xSc)sh9O^3nxjsNf3_}E-m5!P3g z?pWCDu4nhvk$>Yd18>1+ulL_EJk1|yZCq%-VbA)O+`Bx#sy<)Z!+dCsHphAIY4aC7 zIO~|8QX$~n{+{XP!^3l(WcjjJE&0WF<7(SU&sm(Adt`Jax>gy?eRg=w{k;+G#w&l# z;mS|QI$V90hok)O_ahhP_S|)Oe7(l+?;p#H!UAeJCTBSQ1Wj2R)8za5UUKoH%XJbp zbIKa$wu`4RYVH5T`@>VN!Im}I(roTe6Ei9QWwr(O;u#HHGE%(SWiP&HoSc6~QtP)# zic)M;ansi3o3;DZWpD0c4!Iw5g5}Va%|)I4Qm?gqpZ-0PVKMKEa`M)elMXX3*)mJ$ zN1mU~?RF?l*=c&1##Xhfvw{U@EnvRI*!lWr(VrP6Zdz0Hrn0Yi^}&V3dGVS1r|#b3 ze8)I@Mo{*ePYj}Ve4S#a4)w^etJ{2j{Uyq1F8hiC#%80?#A8L8Z8uF;&gWdRi#cS6 z%W>-ulVcv*_Wgfwa+&FA>GLZ&zD4Z6*}Eo2ZkeI?dRzNd%po(1wZeBDGU4wHn;}u1 zZ2i2FAyJPbs`6l>S?BJ>;*~oVUYQkA+Q4t!+C~C-o=&{Cq2-r1xCh|GU@26HKP~L?1j?$p3JjrdYsJ6(2tb zPfyhoM_%}?xv)}PM$q=)xv)JFijApDTiXpj?v|I>E5B{um95369&DZdfhGB(rNFYd z37v*gGlb`^Tr~IdvA1Q78pk(gusKy-RS;fm|0D6u%31c$W&fQ&%#*=9d5I}|)Q1=C z7vH!n_nrD?j`kF{$7$b>u5FPl-D4ZO(aUNVw9E?g+He<#ta{CmxLlNED=!zO1nzxGZ~d}s0R`@83?yZ%O|oZajY z>~DQsW$jJN=JrEpI-?%eo%?C?QdsELnREPpj%M!z-{!`tpP9B}ezD#1@6GWox0VKU zy()NDrpNPOhHd?wFvFAVU58lS9d};9zxhOz@)Wtkud1r{k9-Ydp4CTgIjr-xa9421 zw%Zq-wtp`)>adIWEc~tGsqxyCS8o4p`&uu2)b-zVp}8&*uMW*Aolgo5gZ_-_?#S^#L>K&5kHj%!t zQ`Ab@->q=rb{8g{x{JrY%o5XkR581H!b*eXOl1byO!k)>UaqrAJ_YO&fri8_l>XL_X=I+TQBK5<3sE1dCq%x6;waq<$hP&4_`RS?i49>1t z%<)w>rsevir$;t6oWAOsxbLKokVkOyxt53>J;f6We{pQS|MPppr6VP3TlXH4w6`}p z9kWI+eNMjZWS7(S26wjadE0&2ywGsJjO||K#4Z!jmuDtd7H#_QUGI!vI?K{d$20B^ z8UL^!VyVuY@^a38`Eb5(mrRaiTm83BEVB|eEQz-=Heg?(6WwMunRXE?F8h+L?Q)lrFr{B6= z{M9!@`{T{Si&h>&tKKqh-jrVF16wZrejz*C zDLu?qt=o0e`b`1+Wl_nurv7eVovZjVWtpXlu0p4Y73 z*m<8VXSY^%wp`j@7`yay=|%G&+qTf>9QvZ zbSye5@|z*|%leO>&+&VH-)|Q@%cJMoHJ8(K=WS@3%$UZ;q(8?o-}v8w<;>^zak4(y z-?LNdc&X?B*`eD-PAmN6`Ml3Ziv>il%^k!2gUcZuA*@~Y@o z>yAz3d-b{(Jzdqx^hLxjzre?gStCDx&GF{6hcXO5oJ_YWMq02MyqWveRqC|C)#Dr9 zh+KWV+2RS`IfaH5aeBf_Pm0ZXX`cQy*kofkkrQgFfBm9G^U3FGJQtTpYD?VX3$)x#Q z_El}4j?=Qv1{R?Y&0Yuo-oKcmpP-@A5*mFcYyGd)x0ZcJH|sd* z_^!5ZU1$BKx-s{6+v3IlrTQ+el(zH7Iqv@NQR?}avMpMES@++f-7~pk zb{v?xzd0oN@b8;Ehx5*7*QGwrzTLL4WqqE+w%FCI@%yX(#+mb+JwN~U8C~08<|@e# z9KZFx1m67eK$vk_$&?w=ev+w2*Su)47Tw_&Cj4eQ zU-%^M-vVT{#mp6rrz}5N0Y2<*IeIy zGiOsYQ)}GGg)$j4pL`DGjCK7H`bPTsHk&qWx9>gXpIf8%9q*X&gy)yAeoE=?yAL;9 zfBpTpoPNU9J^GRN_k``gF3opSo-6W^zM;;-rAsBJziEr&lMX+;d}pU?;I-2|0UW0S zcDZ=PnJ&{jtgSO?PvM#UE)hQOb95VK8r+sMdKA*<{`1oNNc}A?%XQsnaBk^Nob6#2 zCgZvN&GE`u8^)}=v37p$0YxWo&tTYGb0+Cw^2Qe~I&Brlr%G-vc((b>tj%G2rntMf z|F|mnS)H*_WzG(f?MV?%+4~dXcziliQ+|A^R}@Z5t)3ZV<#Mz#jBzQ)gAW#IA6ep+ zPk-3hnK5naJQ>6K0+od-OS%5}_VmZ+>&yxcvY54X*OD{c7hHS(S6zx~iZ46Vvu?>h z&yTa3+KVp7dPcVF>@EE2=@S!pvCD4Zo*Z5Vi_}8Sx$ZkYsa;JzR_)azb6Sj1;nErg zCHGL-&yFlMHhJ(^Nl`fr+V_45{Hj3>!W4XP? z{P<3$84}Cazt<>TyJqPg>-GOE!V}LwDW9UY>%^34Py0W=GTg{CSu!_6cJA|2#d~rV z94cIQ$f$pNh;#bYyvTj+yNgu~zF%=NYG<&q@7uYuZszI@>7TfLRkEk=YTr{=-1vV< zf7#s5#V;+FeZTS0SWexBX~XIolc|SGI3`T$Ir+0%(`otC{8?TJa_0;V7j@fM8m7rV z2zX!-o^<}n_sJTk**v_wj9kjEe5tSB;=#TvgmZJ#I>XoB#ee3XEL*%xd1jvGhZW9- zXQUj8=1-b&eSXj??SkAT63_M?^kFzqR9h135#FC%_rmt`y*^I{kK>Npi&od?Ex9o- zlS#AAgk@QO-gRc9pTcMOcbwsTB*`-Akg8vlhQ5E}>M~UWe#tcU1s8%g2j-bwXm&ci zXOV62J;s+}6O>NxYX0<@GghM|GEitUTfi5`?YsK8inhF3e=GEWjh!hMH=FQ*D%0s) zaT!`IE2Aff#0dXfVX^s>_!O@JnCtZI1x_$3R z;hQwG%+0PVGnQ>#A@cF>{)=x+o@#~K%~4%x`^{svLx@P7%|7e+b7@&N&%*4d^z^LW zlh6El^5mEOH-wH<}+TO@TBuq zcAuFN7Oj?fFU=Vgou56kTRyq!((XkQHK!l0bO=#Dm%g#KVdc4XJZED2ILhpI^u;bO z_O$7k`IND-*!axsIMwsfvUX?E^UNHcf8jf6|KH8(d&11kpJXlw9&q{0zt6h<*ZIU% zvtsib&YawnRrF)O_4ftmQX);{Oq)4(f3A*Cu$oktRo8V$hCzS-{Fn-cU|YY}vlH*V zwfXaL@6C5|%U*5Hj+rtmt1Q$!GApFq`RB8wWf$1PZ#CXr&VJV6(%~5^qYuB;`RrJ- zgGa^uzvudCmY3WA2F&=zxuU#wf$anpA0OeRD-&F*Sd15!v>dj1dG(~j-OE#4R33H< zwO*b+OEb2uN%FpDgk_w+*szl!UQH?O=k+J8Sr ztI_0^zK+2m!^XbEQ&XE#k7$ber5ik~`C?ss{e}1WsPq#l)#3YC=Cl|psRr+ey*ktW z_|q1~Ee}4_sjYjsGf8O$XlkLHTXN3mWEn+=a zq}o-MXLbN)m^;_+iO9n5PhCi(DBs-H0_;J?YyN~@EW&u?Em&F$42 zeE5Af>)Gd;uQ}B7&Ums;UeO;O`;bN3Xwy=O54{SJuN7H@&lFeBXfA(zpMTN;=}pek z##iTW>7P7Nw&V59O&;ou0c(_!LMI=t@L7C!r(enx_Ma-c(>9l?ao%3VI_=uAH3ddD zXYsq9{NQQ1(W5x^s8`z;rsn9g-yD7=h~3v(u|jX25mUs$b-7ZLbClCKW*KQpR|Iqm z%v}35tL-U|KyT~$j}rycT+Y8@@hz}uFF$X3?WfO9)7vbJfwR|aF-^;z!=toNK-BT+ zp{IK;aFo~c3fTSm)ad58IaB$_!`=4%+g7u!=GErBkuY5^JICjM=aQF)as@PhYI46S zy`~CAHFy3TD*5*q;Ob4PG{N58|&t8>eAc6(&Bz*OVdw*CIn|0=x$mvcaeF@>oGs>HU73Ml^XkO4cZ^r>v_q%*aXC(XMo8tHIIk#IR9%5*U zEzXwPy6@c2BQq*~-0Ln_J1zEMiyRxv?q`9YYiAo=)QLT^`$%(WM!RtatNOiI>A)`5 zmf63oPd#&Le(2o862|j<4fC9UO^1DL(%yYJ;#k-sW!yjUL|4`=Ge3=+YxAxy{&FX< z@JR8NT`WgkC9DohI-cY^Dp(|b{(@s+%as1^ivB90TiQRJ?`?g%jF%;Fw!|q_PT$ow zE~~T_gT|$L;&V>gFfjf5KYP}MCBag!Zhn2o;(R`O>4{Lkb9c(q!yDpG%E+smZdJ)> zt`>iyf2758h1~>6;hAbTp2;2c)VrMFxzMQd#TVIfgO`tm>_7NPnH`pLJeE4CcGnx* zg|-=e4|2Ev@lboxsXM#S_(SU11^4XIy?R@%-S-%9?iO}Y&{-aq^Gstxs?^a$-UWBCGCs}zEcFce`Y7c7A}j0g`MN%bYI+^y6Kj`wB)PoTo@vK+N$)g z{*(k0KmEW3$tk%no>+%k^mxoK5z2WMJ)^}uF)5aJ9^+EK=NpQ=%(i~q&iCm`;`eX& z()Y`1FMiEX^IW{><~qT8iHbEjDO0o7^0n}}e_wK8qBjTAj8BJp3=?@~*uM;HSRW|$ z-YoY^Wy3$iecf@v5#rXzm##MG^f~tZRHWrj-=xcP&X+rt-jKF)wOyLp;(48MM$Vk{ zJrjDLPElXQRX^zV-Z~9=} z<~uLw?ic16()WTN-c>vh5;0@p%cGocpZQ5mo@v1~IYuod=+KhS|CZ&fx^kA~(q_(L zxy^dTNr#tyF8yQeZYHhfK11Su@I&r%of8b+zPGubwKR>X?481yBMvi;Sc()!CIB(Q&y5sVb^FQ@Z@tn~M2vn|9)%MX+?Y+qFIVY&>9CtB0 zheJ8rtH2Gp-(+0*gg(8n+1Ft_)vqs$x46~8UgWxw=gy3yKjJ!rCb{o-pSrDK?&tRC zat=dAuf?6pDu)ieVLfYb$z^$rAj<|*%bQ$ZD{BvJo4V|Q>x{(<6!VTnY1&TST72Tn zvA7wtpDYqyRxsV$sPfD1t`}1_7RMAN>jxP;=Qo@zq^3GkuVYucUaFSZ`NG^Eryn0% z>Jhc*z(SwuFUMyo?XI^zerZF`FVFlVnK%2b zU8a>W=!s8D$(!K$%5z5U*>0c0bZf3d75Upa(>l}kB|Y_=!GG}Pg^hAQnZ&fEt&9`8 zYStf0`J|NC^R~Z?;h1U3CY40J?JiC$6ARq#T>f#Q>s(QyRDFa|XYtPJ<)0P^JelZs z?~Jx%VjaJW!glsaFYcJ#{MPb&@~^!sTJAocnz^X>?B$sX1>sG1->M!>r29d(-hSbd=4bl1NtWI5*xqn*D^y3Wz&nI8Ld0CY6^6|bPhcxGBN|m*1 zzu#*0v=)$ksq!#V@o=_Knn2C>O;>m}a?YLnT1ai}>JR?U|Er%-keIk|_obN^r#%1M zvhtSlne-MPZNHv#GV7QAu*^F8@`n7}W2=t{_uMYqbAnlJ?-7NphQGX<}{=(+vHN@tb8I+I^}76}{WUB16Tqj25Hmvh%md(_5r z*M9EFkCQij{3`nNOq1XGlC-DKp6Y(~o%yqBL-73ikaN3yB^eYB8Bdr|V*cs)g0l+S zH5vDxpT0V5*^J}<46A+^T_{rizc=WU(1m9nbumk}%dfQhe^ZIMrn+$U+V-s6UH`)0 z)iLJ0ns)8U{$1A1*IF7CdeWTVWL*m5t$6+RZ0f=P|9^Mg(7L)RkmskJyd{g1UjNy1 z=gt{fJ{JC|pt4`fYEuxulbZ0ed0V%gWm>r|Cu3=@Qn=IZb3r$j`F^l8n7mc;L*UGX zS;jI(^1`zhhO2Gg`tntfA=l4cch*i=VsyI6|Iuguxs%Q|CCdDd7M*l+;%%nYr)E_O z&9q>ibZO?b_J}WB^M8p&NIiXQzU#HWCnv|j!+HG|mSnQ0GwiMUm}5|Cy(j&RhSug% z5&NZ;v)$4;70mda&Z~}i9?g5k-9XYIZ>^br`^FcS@|!PzwET0;uynndR{e_mzZ2Yh z>ficqOWS?myW#sZOO8iO$J39vK6vu&53f>v?yOx^O7~kz^fs*yc4I~wykf>ac=d9N z_FD!9p@q=FE0&5|Z{~8BUowzryUq0U+>L|-2e=ha8buWZU07(K|C#6ie|w{avhtgk zWrj}UsqME;x8Bv*|Gxa>bd#varzT!k4=zX$?`{#vy`4LshwX3P-pDMIqSWKR)?bmy z7QL=kq24qz;0r5f$J<@4POC4ky_)s2M)SGFagXbQr%b(mrfpW6ve`&GeD#f%mK!_N zCZDYE`H)n(JA$?G2k*A!FZ@M!mfmi?mQZl-ZR@?-xM!tviuA5$n0TG7+8bpn&ttW= z?Dp2m5HZf{%}cp{_NZ|4%UB-%Xdsk0JKIap>EE9^+u3(>rp4zw@0jsg@HR*O0pY`v z=J~5HzrPx{_Z0X4SqTXbx5j9m|N1I)@AdavqSRXtNj_KWVcK}&TWf1ahuZ#F! zf68Hh)SvD+^kL2Q>Z`Ag9%XG06Nr1PC8Ss}`=bSS)77kvJNEwF6_b~jm$v!+>#u)n z-rd(XJbZV`vOOCN#mk=!%gjFAp-PaZRw6pU)>r>rly$okPi?2AXzN*#X z;*^`XuFQZ_&2fb&yK0GcLT+YZ%p7rcwGPe{p)@B4-QIU;Jl{ois*egRjghE7&lcP% zA>qDAp+d8Mb>)(+_FX1-9hrhS?Tu=DzdZjW#+Ci-cF~l8#yp!ZC5ssT7%rDRQ~M!d zee>yP<2eBr=W;FZY5IB9`Il?C!9=x-qS@Y&({dbUDkz=WS@!c+VwJR8iuS_bLy>$} zXFXtexqK)8W%nqT_lE1donlhGHJ+!elV8S|*mt38Yu!zOothC&mgWkXQVf@WpUP1` zXO!;TvD)O>(-?an=0h@-=KH;7?M%{|7I5xs`%Xqyj*F(p&c<(?^CrS}a#0?W&-@3M z=YM``cRD@z9B)Hsz=2=(R|~8ApKjKCuJ**oMSRT%i8Mu9Z;j`_5|x+{H|#lnZ~L>v9Sn86&8w!-1`e}l!WPwmbaojfE|ruU_NC#Pkw$tB%VzAD=(>8!KB9ZBvEzsg4*Uw9dk6>4xbFPYZNvKDvfB^^h%;Q zZAWh4L6><3=XIvO@~K{&$>j5E>dsE@lbmXns%NB*z5Z`-^j%8z?T!6QkFAODn!2UG zz4ye-hx;tcuSH~?*(9|rI7PZR;nn4?(rw+@H|JT{Z45eb&F-QJgPJSj>!@opMby+D z8eRHv?*z+?M-RfBrh9J4T^jhmyshB)G~>g?H+Y+lZ_-%5tm)?Alsq1v*iYM@8)q{Z z2Ty7KbnCgB+JQNplMgT7^zk-}_}mbKVzAI1gP?6un|?Vbe4VUw{Mw(4sr@`BW_o{Y zXwYu2Jm+cKUX*v}=a~Y=Cth7YCBA8ey!zv3X}`%}Irm4I3A(MFpAUF%wc7vxcw&%9 z2kV4AGu)l|K8HVN`!ch{uRvec=j*GQOfDbaMcHyE7>rNQ&bF(jIpN$<2EVIPl3Uq6O*EH%Yx2~>_vFlb zn~)-8Z4X9ghT1<2oy!GhH)pWKsLUvLHf#-~` zzLKkw@8}cy^75_Na{p43-3HG#Olmv)_KU&icHde1Gz^RO?v|<(urT!$DV{U^P|vP! zUkv6R-uAX`k>UPWUVTeHT|vI-(+_%vyIO{?uD!69yVX#?-MQ~`5VLQ@OzS46P`2WZ zLS5%eQ)G2)_89Ju;cZ=a&u9JDEAl(uEK#ey)bzlsZHeaWQki=*@|9+E)YmO%R{zTR zy3K%d+MyC%WvjC#$7-bORvw&Na68QTz?5^h&U(zs?3H}D#J@-QU86Aj_A`F7P9AgN zmz-SBXL0c|N1jH{qs!C66q@{dKnho~s$Y?R(W_GUZi)Xa_TBXsTMbPgF6nr)u2EXW zHYf1(78~~k%Z;Ah=6E*!q6fR5g|Eaa&3c}WguM9P-C0@5mnJ+-%{XIm?(p=o74`w= z{_mMLy-c0^tby8@epRa@ygU1J>u>Qt&ib_^^RV+2|J*a>e~N7_9==O^YRbXFw`bK= zt0Qg`>UE!&Jhzqjd^=+ayZV{?N}-05r@aGgPrF@^N|3uX7FsmYL1D*YnUu z#oA+5m%84R;~UNI1xtRmoiD^WEi3uZyJLw@bRXJtHkwX2GHdzD%w6^MuAWO<)mb=~ z#&G-H_#dcp_D$qwe(h7t(w`q+6)3UXBmMFqXS=1t@{`xS=48hNda=7IMEuJ*z^QAk zV%;DqEVy^swBveDj@7UmBqpw7RJYm8GI2gY?_5>^+pbo&jnumV6RMHJll-N^&yK>_zSMr|Caj(&qzK$ zVbTJ{O|5wWjJ5y6qb6ON+iMoXetSZh`z*E4sSfeyzwX`l&)U|nFoa2YFZ<>2=FVNl zCU;W9`#v)|yV75F=xy1Bkkw|f&4qEN6vYnhS69^Db=hs|${OjH zGw*r|SEqpX$4YJ3HullKIs~clH^kcFE`7;VeGK&p7%2kxhL?PV8q7&Eft1vt~(x z&qm)li~s9AxfOTU&%V4zUGl_omYDukx7SJ)^CstIn@)Yymv&#tN5pOE+mm8Uwc9Ok zdgq7VU1w|0uPr4krMu+L1(A(!4W3VaduYWa<*WlYwT|9&l|Q%2FkMyi*E(K(aSUinx_ubJY zewZ|W(^OH$R%uy~a< zFN0@J^IkJq!DAv`&qIVZt&@MbylBl5%~cmKOIR?QnD)vnkSbcr9FlRwy{xCR?IQQ` zS?6xtd$z}=a7q3di_Vr?j9Mm-o^P9VC_^Ij%FBNX_Z>gkZIryOMJM3UjFSDqN5ss9 zKXD!o35xCY+n|1q;dqc~ujT*W3{&?{R>`_}rO$BVk=v|GvYw?(uRdBifA)=4TTewe zDJi~ES|VS)JLaco*O{!WIZvOh=_wAdJm;=u9o-tT%Jq0y@6XTLU1wHhl{H+Bn^Ro$ zUBP1Fu{XC(o_!DxOcC06FUj(p<5r9HWq;Mnb>O__nKl7p!OP*{kDraNf z{Bus6yYxTVr%!k?+B_$@Cf6Nb_2k~WKOzg>TKJIt+)H@a|gw9 zJK21Ewl18zTJWV}&&l5&_H0)^*zIE6ZTjNCgFiJPZXO?()YUK~Cn=s)Ie)G#oo&k; zjlG>S<}ns(iY&HK%s!JCG5yQtvzx0g?qWUF#H4w#{qo9NFK5~Ao2R~SgT>E-N3S1w z<9>L7xUG0JztNJJ-`;I+H>?frsVOi_n)xfpE2n>RiqDz4oRt@A=0$#q^b-lt*Jp3t zuruKOhJUgemTUzXx((5KI+DU?w{P!D-MaVuTz~!jsUF3v7`lDktzKtdpEzOWk&?d! z))4~ET0NQht=DZXrG5W%xcyw`>s^kDZ5G$94J>2MPBslc5pbwqSI*{X#6#OTQoYYs zKCk^()A+TRSvq#=yQdS4yN~tkPk#ISL)G7}v#&9GZCST8DR6zd=ec(0dD53l`<4}d z+j`4x=7#-i*o>!U6@O3NVSe;Ye7%wXr9zVfvTW`F%G##g_E~Dy7Rn3DlCSKGn{!9y zrOvEhO5%s7UOOJpzVT(nwi)+ZerV4=(z(<6R26UXCtl}!{P{)tVj|qRub%Z4yOc5? zbJ44p>b>k*WX$mrbQFK@!K~|rE`-z?2hTNJP}XGKY1>{Xj=utM1ko|5e0{RiY7hO z{vpb+eIL`>hP@?y8o%PC8e0D}<(ix`nLT-D#-7kE>ilcdT_pEt+I1*Ir1|u^EwjyD zHF;{#k|S$0#1=$e3X3|K_2<-mcdK0-*+N23bn|08Hhf7gI>s4ly6m}3{Dx(+a}IlW zZOpvb+adW}*?igKJr{1stuNEPnKkEspZz=g7+YJ#WU;N!4s7o>eztw*|CQ3l`{%JY zu4@!NrhU0S!RPo(MfOaOQ?&)(=1(&$jBTx)^MAgf>~kJV5ApOCUg`f!4UKo6aQtNv zE}GEionW``B>xpxJ}V8sydyJuw$8D7$y+pmbI#?ji9Ma)ZL?Hac5!y{JbzJe?2W}& zA*Q<)e!q|9SeRaxPALht`1`XXr|cES`qHa@JB&`MoSR|wdwIUk^_BAjc1a(dIpvYh z{FuFdcTCa+)0JwK9q$!qt(0r?-E1L#y1hp8Ov?Oi!QQ4ZJhRgZyK?8mRQvUn@5)-G zCivpl+lrXm?`8|EzPjYjzYUyo0*mIpKOLbe_)X|Fo8@b(ucCS{Ox7hC8c#NH6k4Bm zEmFcPmt*x#w}@+-j{NaH`^f90`@&i5m3sQ}X7bIi=A1P2td}ui-+Jyz{g+7_-34BJ z)xEww>$|&y@rsvo{*;?P-*iZU$K_&_&szSP#?^Z1W{m&;|IWJPBo-RradxlVpC;!t zqvW=q3q^l;dt6+-$f19p+1e%Y$L^eM$h{moqoDr!_LU2Avwn5Hb}l(!mZ>MtH2YQA zgGq@FxmwQ8os)t;&O4czyJ=Uy#uz?{9X?6mYsLd%ACheE;rP`(I}lUwLUXL7lHubmBz2BZtfWsv47HUVzoASrSRe6^MY+kYaZBguKpJimhkz%p~mYsyB~ZJNy`t~{HsYz_Ip~$ z#~1(Y^FK|EUnp~Hk*v?c=LySpsNL`43a`84Z~?T!3bwC`75k7DOI_r21_mKJ=#Um0 zVo2+<)51HF1_{=2D?CqRILbIX9^WIo{QrOXnKSa@H!jOut|M66CpXt``T2YL@p-8$ zG}gU0w~{@-GK_PjRiE*Tl2^+Yhh6`Ed+DU^f0Ko`Us>~A#_5A>i-F^xwh7TMzwY^0 za3{xXck2EpyWBIcm)?z9`F8s5r^^;?tXLm1WuWQ{(j3hGcF3o449DZ_d2Ayua@)ICOfWmr783xL(F5PQ^>g zf3ikn}6>5Te7+DsoL8|zu&BrS^nus=kd$dQs(n*wWpu1yWMy2 zplIY{o9b&){cV5ueShNFpB?R)Zm8q=r~c7?k7GaWeJ=n0_`T76Lgis!wh$q%mfJ0N zw!|_dE3LGebFe|$&NkabO5NsY?7JWATW7ugH}lY%sVaMHDlho`V7IcietwC~CR@Nv z@p<~YrVYFH#jVe`tG$)CKhJFT>HOVy-(C$Z*?p9!zo>1~!y{_yn$L24n?+3Ac1=Ag zTDHIOp7nms!x9Q}uAUMMsj^Y`w`!`Zx3IC7?{#}z=zZbolxY9tdE5Pd`Aw~3Nv z{GS}>f2$}=Ta)`j#_ox?^mdvh>Pkv8_WrDKm3a3oFp_<8=CcaX3kuS+eyDfkH(Bfr zw^odh+810CALes@@8ql9GgL1#&)0m}<9(YyTI8*3liD8E#y{KqB?QAtw`%#iiiqJz#0=6p%FeBcuEY3+tHrrhUaOw4N!$8z7EGqr``&9As+1u8rLDSGV^+O$MZ z&^U750-d?0jsXoHCOs^k;>#SOb>K&|;mlba%nMuI2_JE>-_do`wcJSDDQL>C9>>Hb za~-eMH5Z)qNHaOz7hrj^WrMGT4c`K5-wOTWc-sssgNJ<&#eyDP>-1Ub_H4O<0sr3V zk~11FWt%x@)wr*^;^1O!?w1miCQz5^vQ@}RaTiy6SzhDWmNOa#OnZ;6vU>j_ne*)( zy_s$kO7FSvUvTZTpm)aeXY9V*r)(C?pV542${gG2W?PG^Pup;q8)Pso)jC$ZVQK#J zXQ%IUrMxqEm$v(l_pP10M#@e87won!IlX_`+}jGq5gUuQw7*t~`DgmpW>dl$o)@z! zGx+&qp5;FeJ88nUH&JxT6296$>$9Glo#5I2yLfSt;Pir`kMYZ50^XfmGf_$J;gdtF zjLZ$v4!H<3+V-Tkrk=aGwQy#b%H%k;)NNd*^Ka&ui3|AT_1@m%tft1V@#&^brH<>3 zr_F3<7r%aXU&zQ-qo~*8jN(UDzV+?88PCmnW|puU{k3tmo36Cl@9&zcd8cnnnr$i8 zo-}hp>ojx8xijyjZ%*~mH=beKv+=sy{PI_fIIz%nQ9+$Dx)xr8oUw!=*OE;5S>+!+*-; zUC?+bJwM}C;Io*tGX3K_+Do>~_}ICO*XZ$Y!&h;3$CtE}m>AysJ6+RHoxSMTx;IZs zmmD?UXPa|Wa)OO{ev(lz+vc_{J3_y0-Ed~Hw;J!dNc-#)Dm$7*Zfsqg*?LA)y3^Tn z5zn%ImTl*Ib6!0A^k~{@86(cJ06D+j^SwF}#!qeEm>o2pdWP-P;Tu1m>Pk=g>3ipV z&Z?AiDN|p6>UuC3EE?XI?ga-7)oN%+>l+92eG{%l}_E zGwGb2i^k%a(*x}Ou$L@Qj-Gv4yD2(r=I1{~ce6fD?oRf8#`pNs;gnltuE&^WO1u5h znd!&-G$mpWvuva}|8}1_Qw5CzPnHDwMe#=l$X&HD{&Pf2qu=NCw{~Uj(=j=`|7RMl zUn{qs+4UoT*wmdhH}t)qaizSHkg4EhQOn#BQ!q`DSE+iEX1d#~yu5JsuBL*IkLIyn z&pbUtGcUJvV%nv7)2}73X;hM0?ejWjhPRr{uKBu~dndSNFVLu&m?U`2>gAy+J~xb+ zW@Z{>Dmv`vE>D5QFnBP{Mt~HnODXTo^T(x69 zH_yEObg^M)nSHSTtYZAY{=X0J zizR}I%9_TH-P)Urf`i{|lee4UD03$K)7htwU$=zc&f=c=>grC>YT3%>OgHr}@;i1+ zWqTYp(NQ>UmRi%#pgoLM@)y;=$Um|2n#oda#xrZ8-V7!|A7!hB{|^K^%)0GXbw^9* zyu9g*V2)?|&K=Kqd!b&X)ai=_H8Mi%-mZ);l3nX~z1a(#uK?Wt*q(=V9{9E0d^v zrysS-ww6!$pvh9orzx`p7YHlIE5o zVXK!>cO_jDK6uT+4FdRmRtI-&<)bF zuA64fm?5*W`KO(vkEMCk9*u^u^pEPyUrPB}-#woz^Y`Z4veusWPv++4Up5eFoorT{ zuFK4l?3AIn@wM6#U1^bP8%x)){`+sQ6d68uTXgZ`W7|8A{cM_{=37;=?tS;$AD;w$ zX7Z}NIyWKll;aUgqbk?rJLlq({p1hNslOB~V5Bqi#PjAW{IB<(ijb+i*P39YW)gXj z#n)8q=mn8@`Mu0noaVfbFsnW@S^3uzF%x#53sWvTbT8vLt2Z;`F5l0{)xNhA=DdGm zFzM49&pA==oV zVLSS_z{up8h9JAhSvlu9!Osm}im0TX5wN&gQnEm%Wv>Zuc2n9~*Lcw2)ymRGH?8(| zT8F*1c+%Vm%ZGK0gG*Uvl})xjmzlhXb&^(_h0liWmu=SVjMZ}<|8P8dX|r0! zFWFv;bhBksrf+VIIQquQPc`2-)SZn>Ip=@Inpr<|%0iXrZVUfYDhq^W=9c)ws%2!yVn?7@TJ7@RZV}kn@y)-(o zgj;!Ucn({qWxQo(;k@1U{HDE}uEtBlL@#$#pA`|m-8%7~a@Ks|Ii~7wqaPn!*vq+7 zyLgo(>*WJGXH0qc#E@z3u}J$o!NPOJJH8&AktO~yV^xY;wi}P|;?k>vFHHQ_Uo6)0 zKmV|f+bsE9;j39M`mZ#tW_J4h*(^7t&*6N_$BhCYr>f=ay1%(R&T|E ze>2Nw?s;rk@gTDL;@x@M<2kogYZ<5+hAP!PzpQa)|5P`b@|Q1dXT11*@6v*~dNNf_ z7U{eB6kb?%$O!G6qkT$I`t-6kd)xc=|1Ve%I@Qq3Q z<@T!8Kji%v=}%1`nMBXV%yXS{>`0Go_Z$_?=bfLq3r}!|1Sp);5dWF@@6f|$omICg zGm=(R&fpHY(6Z+B;t7fBy#Y(7CqMjRc*UfsQA*wqM;Z z{&&=>8#{gVuAj>g)QNqwvRFdcHSPK>=d-)ol|1#&#U?)Dd8iwq);BxFXqiyYF_(!e z*ZU>;<<4P#n_uQ~M zc5Rw3v-&=Uw_SgB$~=!M+@at;S!CwV11Gb!`{$_o%?(*1{ycHQwet=qp8vXQ-E-3Z zz?4JLSaaomX?sh2}h1Iq`h&fhDItYG$5MFe&Q&e#B_O zwZgmK1>WuaY`HhB>#KqKuagc%(!$1k##!5P<^L^t6!YZ7vy80^uiWi8&2vo0@z>3O zqc4xzBnAZkt#pZ-;`9BkMz^8&#JX*j+vF>bIk2bw`usspcO}C!UBl{RGqp?oo~8ZS zoEKCBE2hjl{4)G)$yvLie6{qNv-~1|T)!uWO?_4<^HF;d=Oy3Sd|T}J#rt09A6=nt zuH1C`ceGB1`OmyhY}bE$=&KA}rEg`r`(ZWzC*f6w*IHgP#9N&zkF00)>SdIP@^}zt zd{650p~IQFdRyo8zvW+lPI!{sliz=}6BDf7@Gb39ZSP)w{NSXBphTZG**Lw>)pr-4 z`(1W-TkiijxkonleeWgM&D3%6vhQnre(u`c3tvCYDlG_|-S_!pa{CSY<+l~0?u!_P z8fBh&f8)31o_k-;x1RXeAfy{E=xW;&d2N@*>b^bu?m2&2SiPLRc-5w_@5&d>^xoqh zmbl8~LeQrNGIp=-F7FBd8YnKhaJ$8=gH6JFt;KxxlDpX~?r8{I*&^Yu~=bk35M z&ht6OQ!Dbu^`$CXp^nAzXxHB@r)St3oY3(zTXV9pSmmnK>3qS-hOztqAJdA_?+l%M z#IwYe$z*BOCR6dWo|mHeXKu{$c%fs_yf7$e-pLdf@!G0u$9&dwMongaUQ;vejLGpC z9YqqCH=bHxUaK#nr+sc>#J@wu=4nN;#_y$-DsD%_E7jO(r!Bhv_uu&gPmleKmoVO0 z-n&yx`&|0GQ`4=S_)c{is}vZOK0Rt~Kil$!QUUh?O()~n%IjMFtL*d`o=syrcY@{g z{nq~OZD06m1(^N9lDpO)?eE?eqRuooRYqR&)=$f=rn#AI00#FHvAL)%AMns_W**-QzdrY{=8T+R~rU&0k)3e3^Od@S>pJ>*XuO&YPWi2viN<4)XHx%Ica z|L=B~?-p;^v11vN>-s+wPWq$%({i`8ywf7_*DF^fAAWFUgRgF_$(e88q=Pp*dl)Os zxV`TC=QIn2m%VS)&*uDET=y{}KMr%s-*V%w�&7nmUbbSoD08{And^H7>p8Zv(rkK33$B=+WxQvz zGfkG`qsh0&BDTgaZeHF0+&Yc3v8zycxtHgHwTu5(=vT9S5|m%=CDyp}b=l3094uO? zGIO<_%Pcrj+qqpvqT}WVE7zsxI7QSx>DWiS;aIULggs~8q9PrwhG(ZWOHQ9(!^F68 zPbXjViIqD(I6qCa61``1+rA{IRCg1j#yU&pBfIyuD?f=Z*{9|s!Sm?Dqa5x(*4I|6 id{Agf`MF>1KI7ZC!YvalW`p+sF?hQAxvX 0) { diff --git a/routers/api/v1/repo/repo.go b/routers/api/v1/repo/repo.go index d8b06862a5..08c0635bc3 100644 --- a/routers/api/v1/repo/repo.go +++ b/routers/api/v1/repo/repo.go @@ -398,8 +398,8 @@ func Migrate(ctx *context.APIContext, form auth.MigrateRepoForm) { } var opts = migrations.MigrateOptions{ - RemoteURL: remoteAddr, - Name: form.RepoName, + CloneAddr: remoteAddr, + RepoName: form.RepoName, Description: form.Description, Private: form.Private || setting.Repository.ForcePrivate, Mirror: form.Mirror, diff --git a/routers/init.go b/routers/init.go index 1efddcfaa6..c37bbeb6b0 100644 --- a/routers/init.go +++ b/routers/init.go @@ -20,6 +20,7 @@ import ( "code.gitea.io/gitea/modules/markup/external" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/ssh" + "code.gitea.io/gitea/modules/task" "code.gitea.io/gitea/services/mailer" mirror_service "code.gitea.io/gitea/services/mirror" @@ -102,6 +103,9 @@ func GlobalInit() { mirror_service.InitSyncMirrors() models.InitDeliverHooks() models.InitTestPullRequests() + if err := task.Init(); err != nil { + log.Fatal("Failed to initialize task scheduler: %v", err) + } } if setting.EnableSQLite3 { log.Info("SQLite3 Supported") diff --git a/routers/private/serv.go b/routers/private/serv.go index 71c0f6ea2c..c4508b4cb5 100644 --- a/routers/private/serv.go +++ b/routers/private/serv.go @@ -119,6 +119,15 @@ func ServCommand(ctx *macaron.Context) { repo.OwnerName = ownerName results.RepoID = repo.ID + if repo.IsBeingCreated() { + ctx.JSON(http.StatusInternalServerError, map[string]interface{}{ + "results": results, + "type": "InternalServerError", + "err": "Repository is being created, you could retry after it finished", + }) + return + } + // We can shortcut at this point if the repo is a mirror if mode > models.AccessModeRead && repo.IsMirror { ctx.JSON(http.StatusUnauthorized, map[string]interface{}{ diff --git a/routers/repo/repo.go b/routers/repo/repo.go index b67384d721..bfd0c771b0 100644 --- a/routers/repo/repo.go +++ b/routers/repo/repo.go @@ -19,6 +19,7 @@ import ( "code.gitea.io/gitea/modules/migrations" "code.gitea.io/gitea/modules/notification" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/task" "code.gitea.io/gitea/modules/util" "github.com/unknwon/com" @@ -133,8 +134,6 @@ func Create(ctx *context.Context) { func handleCreateError(ctx *context.Context, owner *models.User, err error, name string, tpl base.TplName, form interface{}) { switch { - case migrations.IsRateLimitError(err): - ctx.RenderWithErr(ctx.Tr("form.visit_rate_limit"), tpl, form) case models.IsErrReachLimitOfRepo(err): ctx.RenderWithErr(ctx.Tr("repo.form.reach_limit_of_creation", owner.MaxCreationLimit()), tpl, form) case models.IsErrRepoAlreadyExist(err): @@ -221,6 +220,40 @@ func Migrate(ctx *context.Context) { ctx.HTML(200, tplMigrate) } +func handleMigrateError(ctx *context.Context, owner *models.User, err error, name string, tpl base.TplName, form *auth.MigrateRepoForm) { + switch { + case migrations.IsRateLimitError(err): + ctx.RenderWithErr(ctx.Tr("form.visit_rate_limit"), tpl, form) + case migrations.IsTwoFactorAuthError(err): + ctx.RenderWithErr(ctx.Tr("form.2fa_auth_required"), tpl, form) + case models.IsErrReachLimitOfRepo(err): + ctx.RenderWithErr(ctx.Tr("repo.form.reach_limit_of_creation", owner.MaxCreationLimit()), tpl, form) + case models.IsErrRepoAlreadyExist(err): + ctx.Data["Err_RepoName"] = true + ctx.RenderWithErr(ctx.Tr("form.repo_name_been_taken"), tpl, form) + case models.IsErrNameReserved(err): + ctx.Data["Err_RepoName"] = true + ctx.RenderWithErr(ctx.Tr("repo.form.name_reserved", err.(models.ErrNameReserved).Name), tpl, form) + case models.IsErrNamePatternNotAllowed(err): + ctx.Data["Err_RepoName"] = true + ctx.RenderWithErr(ctx.Tr("repo.form.name_pattern_not_allowed", err.(models.ErrNamePatternNotAllowed).Pattern), tpl, form) + default: + remoteAddr, _ := form.ParseRemoteAddr(owner) + err = util.URLSanitizedError(err, remoteAddr) + if strings.Contains(err.Error(), "Authentication failed") || + strings.Contains(err.Error(), "Bad credentials") || + strings.Contains(err.Error(), "could not read Username") { + ctx.Data["Err_Auth"] = true + ctx.RenderWithErr(ctx.Tr("form.auth_failed", err.Error()), tpl, form) + } else if strings.Contains(err.Error(), "fatal:") { + ctx.Data["Err_CloneAddr"] = true + ctx.RenderWithErr(ctx.Tr("repo.migrate.failed", err.Error()), tpl, form) + } else { + ctx.ServerError(name, err) + } + } +} + // MigratePost response for migrating from external git repository func MigratePost(ctx *context.Context, form auth.MigrateRepoForm) { ctx.Data["Title"] = ctx.Tr("new_migrate") @@ -258,8 +291,8 @@ func MigratePost(ctx *context.Context, form auth.MigrateRepoForm) { } var opts = migrations.MigrateOptions{ - RemoteURL: remoteAddr, - Name: form.RepoName, + CloneAddr: remoteAddr, + RepoName: form.RepoName, Description: form.Description, Private: form.Private || setting.Repository.ForcePrivate, Mirror: form.Mirror, @@ -282,47 +315,19 @@ func MigratePost(ctx *context.Context, form auth.MigrateRepoForm) { opts.Releases = false } - repo, err := migrations.MigrateRepository(ctx.User, ctxUser.Name, opts) - if err == nil { - notification.NotifyCreateRepository(ctx.User, ctxUser, repo) - - log.Trace("Repository migrated [%d]: %s/%s successfully", repo.ID, ctxUser.Name, form.RepoName) - ctx.Redirect(setting.AppSubURL + "/" + ctxUser.Name + "/" + form.RepoName) + err = models.CheckCreateRepository(ctx.User, ctxUser, opts.RepoName) + if err != nil { + handleMigrateError(ctx, ctxUser, err, "MigratePost", tplMigrate, &form) return } - switch { - case models.IsErrReachLimitOfRepo(err): - ctx.RenderWithErr(ctx.Tr("repo.form.reach_limit_of_creation", ctxUser.MaxCreationLimit()), tplMigrate, &form) - case models.IsErrNameReserved(err): - ctx.Data["Err_RepoName"] = true - ctx.RenderWithErr(ctx.Tr("repo.form.name_reserved", err.(models.ErrNameReserved).Name), tplMigrate, &form) - case models.IsErrRepoAlreadyExist(err): - ctx.Data["Err_RepoName"] = true - ctx.RenderWithErr(ctx.Tr("form.repo_name_been_taken"), tplMigrate, &form) - case models.IsErrNamePatternNotAllowed(err): - ctx.Data["Err_RepoName"] = true - ctx.RenderWithErr(ctx.Tr("repo.form.name_pattern_not_allowed", err.(models.ErrNamePatternNotAllowed).Pattern), tplMigrate, &form) - case migrations.IsRateLimitError(err): - ctx.RenderWithErr(ctx.Tr("form.visit_rate_limit"), tplMigrate, &form) - case migrations.IsTwoFactorAuthError(err): - ctx.Data["Err_Auth"] = true - ctx.RenderWithErr(ctx.Tr("form.2fa_auth_required"), tplMigrate, &form) - default: - // remoteAddr may contain credentials, so we sanitize it - err = util.URLSanitizedError(err, remoteAddr) - if strings.Contains(err.Error(), "Authentication failed") || - strings.Contains(err.Error(), "Bad credentials") || - strings.Contains(err.Error(), "could not read Username") { - ctx.Data["Err_Auth"] = true - ctx.RenderWithErr(ctx.Tr("form.auth_failed", err.Error()), tplMigrate, &form) - } else if strings.Contains(err.Error(), "fatal:") { - ctx.Data["Err_CloneAddr"] = true - ctx.RenderWithErr(ctx.Tr("repo.migrate.failed", err.Error()), tplMigrate, &form) - } else { - ctx.ServerError("MigratePost", err) - } + err = task.MigrateRepository(ctx.User, ctxUser, opts) + if err == nil { + ctx.Redirect(setting.AppSubURL + "/" + ctxUser.Name + "/" + opts.RepoName) + return } + + handleMigrateError(ctx, ctxUser, err, "MigratePost", tplMigrate, &form) } // Action response for actions to a repository @@ -460,3 +465,19 @@ func Download(ctx *context.Context) { ctx.ServeFile(archivePath, ctx.Repo.Repository.Name+"-"+refName+ext) } + +// Status returns repository's status +func Status(ctx *context.Context) { + task, err := models.GetMigratingTask(ctx.Repo.Repository.ID) + if err != nil { + ctx.JSON(500, map[string]interface{}{ + "err": err, + }) + return + } + + ctx.JSON(200, map[string]interface{}{ + "status": ctx.Repo.Repository.Status, + "err": task.Errors, + }) +} diff --git a/routers/repo/view.go b/routers/repo/view.go index 1967b511ca..c4e6a69220 100644 --- a/routers/repo/view.go +++ b/routers/repo/view.go @@ -11,6 +11,7 @@ import ( "fmt" gotemplate "html/template" "io/ioutil" + "net/url" "path" "strings" @@ -31,6 +32,7 @@ const ( tplRepoHome base.TplName = "repo/home" tplWatchers base.TplName = "repo/watchers" tplForks base.TplName = "repo/forks" + tplMigrating base.TplName = "repo/migrating" ) func renderDirectory(ctx *context.Context, treeLink string) { @@ -356,9 +358,37 @@ func renderFile(ctx *context.Context, entry *git.TreeEntry, treeLink, rawLink st } } +func safeURL(address string) string { + u, err := url.Parse(address) + if err != nil { + return address + } + u.User = nil + return u.String() +} + // Home render repository home page func Home(ctx *context.Context) { if len(ctx.Repo.Units) > 0 { + if ctx.Repo.Repository.IsBeingCreated() { + task, err := models.GetMigratingTask(ctx.Repo.Repository.ID) + if err != nil { + ctx.ServerError("models.GetMigratingTask", err) + return + } + cfg, err := task.MigrateConfig() + if err != nil { + ctx.ServerError("task.MigrateConfig", err) + return + } + + ctx.Data["Repo"] = ctx.Repo + ctx.Data["MigrateTask"] = task + ctx.Data["CloneAddr"] = safeURL(cfg.CloneAddr) + ctx.HTML(200, tplMigrating) + return + } + var firstUnit *models.Unit for _, repoUnit := range ctx.Repo.Units { if repoUnit.Type == models.UnitTypeCode { diff --git a/routers/routes/routes.go b/routers/routes/routes.go index 11f2029226..8dfcdb9c9b 100644 --- a/routers/routes/routes.go +++ b/routers/routes/routes.go @@ -845,6 +845,8 @@ func RegisterRoutes(m *macaron.Macaron) { m.Get("/archive/*", repo.MustBeNotEmpty, reqRepoCodeReader, repo.Download) + m.Get("/status", reqRepoCodeReader, repo.Status) + m.Group("/branches", func() { m.Get("", repo.Branches) }, repo.MustBeNotEmpty, context.RepoRef(), reqRepoCodeReader) diff --git a/services/mirror/mirror_test.go b/services/mirror/mirror_test.go index 76bd4c72f7..9ad11b7265 100644 --- a/services/mirror/mirror_test.go +++ b/services/mirror/mirror_test.go @@ -10,6 +10,7 @@ import ( "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/structs" release_service "code.gitea.io/gitea/services/release" "github.com/stretchr/testify/assert" @@ -26,16 +27,26 @@ func TestRelease_MirrorDelete(t *testing.T) { repo := models.AssertExistsAndLoadBean(t, &models.Repository{ID: 1}).(*models.Repository) repoPath := models.RepoPath(user.Name, repo.Name) - migrationOptions := models.MigrateRepoOptions{ - Name: "test_mirror", - Description: "Test mirror", - IsPrivate: false, - IsMirror: true, - RemoteAddr: repoPath, - Wiki: true, - SyncReleasesWithTags: true, + opts := structs.MigrateRepoOption{ + RepoName: "test_mirror", + Description: "Test mirror", + Private: false, + Mirror: true, + CloneAddr: repoPath, + Wiki: true, + Releases: false, } - mirror, err := models.MigrateRepository(user, user, migrationOptions) + + mirrorRepo, err := models.CreateRepository(user, user, models.CreateRepoOptions{ + Name: opts.RepoName, + Description: opts.Description, + IsPrivate: opts.Private, + IsMirror: opts.Mirror, + Status: models.RepositoryBeingMigrated, + }) + assert.NoError(t, err) + + mirror, err := models.MigrateRepositoryGitData(user, user, mirrorRepo, opts) assert.NoError(t, err) gitRepo, err := git.OpenRepository(repoPath) diff --git a/templates/repo/header.tmpl b/templates/repo/header.tmpl index fc7f1b660c..9fb3e32899 100644 --- a/templates/repo/header.tmpl +++ b/templates/repo/header.tmpl @@ -16,93 +16,95 @@ {{if .IsMirror}}
{{$.i18n.Tr "repo.mirror_from"}} {{MirrorAddress $.Mirror}}
{{end}} {{if .IsFork}}
{{$.i18n.Tr "repo.forked_from"}} {{SubStr .BaseRepo.RelLink 1 -1}}
{{end}} -
-
- - {{if $.IsWatchingRepo}}{{$.i18n.Tr "repo.unwatch"}}{{else}}{{$.i18n.Tr "repo.watch"}}{{end}} - - - {{.NumWatches}} - -
-
- - {{if $.IsStaringRepo}}{{$.i18n.Tr "repo.unstar"}}{{else}}{{$.i18n.Tr "repo.star"}}{{end}} - - - {{.NumStars}} - -
- {{if and (not .IsEmpty) ($.Permission.CanRead $.UnitTypeCode)}} -
- - {{$.i18n.Tr "repo.fork"}} + {{if not .IsBeingCreated}} +
+
+ + {{if $.IsWatchingRepo}}{{$.i18n.Tr "repo.unwatch"}}{{else}}{{$.i18n.Tr "repo.watch"}}{{end}} - - {{.NumForks}} + + {{.NumWatches}} + +
+
+ + {{if $.IsStaringRepo}}{{$.i18n.Tr "repo.unstar"}}{{else}}{{$.i18n.Tr "repo.star"}}{{end}} + + + {{.NumStars}} + +
+ {{if and (not .IsEmpty) ($.Permission.CanRead $.UnitTypeCode)}} +
+ + {{$.i18n.Tr "repo.fork"}} + + + {{.NumForks}} + +
+ {{end}} +
+ {{end}} +
+
+{{end}} +
+ {{if not .Repository.IsBeingCreated}} +
+ {{if .Permission.CanRead $.UnitTypeCode}} + + {{.i18n.Tr "repo.code"}} + + {{end}} + + {{if .Permission.CanRead $.UnitTypeIssues}} + + {{.i18n.Tr "repo.issues"}} {{.Repository.NumOpenIssues}} + + {{end}} + + {{if .Permission.CanRead $.UnitTypeExternalTracker}} + + {{.i18n.Tr "repo.issues"}} + + {{end}} + + {{if and .Repository.CanEnablePulls (.Permission.CanRead $.UnitTypePullRequests)}} + + {{.i18n.Tr "repo.pulls"}} {{.Repository.NumOpenPulls}} + + {{end}} + + {{if and (.Permission.CanRead $.UnitTypeReleases) (not .IsEmptyRepo) }} + + {{.i18n.Tr "repo.releases"}} {{.Repository.NumReleases}} + + {{end}} + + {{if or (.Permission.CanRead $.UnitTypeWiki) (.Permission.CanRead $.UnitTypeExternalWiki)}} + + {{.i18n.Tr "repo.wiki"}} + + {{end}} + + {{if and (.Permission.CanReadAny $.UnitTypePullRequests $.UnitTypeIssues $.UnitTypeReleases) (not .IsEmptyRepo)}} + + {{.i18n.Tr "repo.activity"}} + + {{end}} + + {{template "custom/extra_tabs" .}} + + {{if .Permission.IsAdmin}} +
+ + {{.i18n.Tr "repo.settings"}}
{{end}}
-
- -{{end}} - -
-
- {{if .Permission.CanRead $.UnitTypeCode}} - - {{.i18n.Tr "repo.code"}} - - {{end}} - - {{if .Permission.CanRead $.UnitTypeIssues}} - - {{.i18n.Tr "repo.issues"}} {{.Repository.NumOpenIssues}} - - {{end}} - - {{if .Permission.CanRead $.UnitTypeExternalTracker}} - - {{.i18n.Tr "repo.issues"}} - - {{end}} - - {{if and .Repository.CanEnablePulls (.Permission.CanRead $.UnitTypePullRequests)}} - - {{.i18n.Tr "repo.pulls"}} {{.Repository.NumOpenPulls}} - - {{end}} - - {{if and (.Permission.CanRead $.UnitTypeReleases) (not .IsEmptyRepo) }} - - {{.i18n.Tr "repo.releases"}} {{.Repository.NumReleases}} - - {{end}} - - {{if or (.Permission.CanRead $.UnitTypeWiki) (.Permission.CanRead $.UnitTypeExternalWiki)}} - - {{.i18n.Tr "repo.wiki"}} - - {{end}} - - {{if and (.Permission.CanReadAny $.UnitTypePullRequests $.UnitTypeIssues $.UnitTypeReleases) (not .IsEmptyRepo)}} - - {{.i18n.Tr "repo.activity"}} - - {{end}} - - {{template "custom/extra_tabs" .}} - - {{if .Permission.IsAdmin}} -
- - {{.i18n.Tr "repo.settings"}} - -
- {{end}} -
+ {{end}}
- diff --git a/templates/repo/migrating.tmpl b/templates/repo/migrating.tmpl new file mode 100644 index 0000000000..34031d5653 --- /dev/null +++ b/templates/repo/migrating.tmpl @@ -0,0 +1,31 @@ +{{template "base/head" .}} +
+ {{template "repo/header" .}} +
+
+
+ {{template "base/alert" .}} +
+
+
+
+ +
+
+
+
+
+
+

{{.i18n.Tr "repo.migrate.migrating" .CloneAddr | Safe}}

+
+
+

{{.i18n.Tr "repo.migrate.migrating_failed" .CloneAddr | Safe}}

+
+
+
+
+
+
+
+
+{{template "base/footer" .}}