Browse Source

#1984 Better mirror repo management

pull/2125/head
Unknwon 9 years ago
parent
commit
120cd4e471
  1. 2
      .gopmfile
  2. 2
      README.md
  3. 3
      conf/locale/locale_en-US.ini
  4. 2
      gogs.go
  5. 118
      models/repo.go
  6. 16
      modules/auth/repo_form.go
  7. 8
      modules/bindata/bindata.go
  8. 1
      modules/middleware/auth.go
  9. 1
      modules/middleware/repo.go
  10. 2
      routers/api/v1/repo/repo.go
  11. 3
      routers/install.go
  12. 6
      routers/repo/repo.go
  13. 7
      routers/repo/setting.go
  14. 2
      templates/.VERSION
  15. 2
      templates/repo/header.tmpl
  16. 5
      templates/repo/settings/options.tmpl

2
.gopmfile

@ -17,7 +17,7 @@ github.com/go-sql-driver/mysql = commit:d512f20
github.com/go-xorm/core = commit:acb6f00 github.com/go-xorm/core = commit:acb6f00
github.com/go-xorm/xorm = commit:a8fba4d github.com/go-xorm/xorm = commit:a8fba4d
github.com/gogits/chardet = commit:2404f77725 github.com/gogits/chardet = commit:2404f77725
github.com/gogits/git-shell = commit:de77627 github.com/gogits/git-shell =
github.com/gogits/go-gogs-client = commit:4b541fa github.com/gogits/go-gogs-client = commit:4b541fa
github.com/issue9/identicon = commit:f8c0d2c github.com/issue9/identicon = commit:f8c0d2c
github.com/klauspost/compress = commit:bcd0709 github.com/klauspost/compress = commit:bcd0709

2
README.md

@ -5,7 +5,7 @@ Gogs - Go Git Service [![Build Status](https://travis-ci.org/gogits/gogs.svg?bra
![](public/img/gogs-large-resize.png) ![](public/img/gogs-large-resize.png)
##### Current version: 0.7.34 Beta ##### Current version: 0.7.35 Beta
| Web | UI | Preview | | Web | UI | Preview |
|:-------------:|:-------:|:-------:| |:-------------:|:-------:|:-------:|

3
conf/locale/locale_en-US.ini

@ -352,6 +352,8 @@ auto_init = Initialize this repository with selected files and template
create_repo = Create Repository create_repo = Create Repository
default_branch = Default Branch default_branch = Default Branch
mirror_interval = Mirror Interval (hour) mirror_interval = Mirror Interval (hour)
mirror_address = Mirror Address
mirror_address_desc = Please include necessary user credentials in the address.
watchers = Watchers watchers = Watchers
stargazers = Stargazers stargazers = Stargazers
forks = Forks forks = Forks
@ -369,6 +371,7 @@ migrate.permission_denied = You are not allowed to import local repositories.
migrate.invalid_local_path = Invalid local path, it does not exist or not a directory. migrate.invalid_local_path = Invalid local path, it does not exist or not a directory.
migrate.failed = Migration failed: %v migrate.failed = Migration failed: %v
mirror_from = mirror from
forked_from = forked from forked_from = forked from
fork_from_self = You cannot fork a repository you already own! fork_from_self = You cannot fork a repository you already own!
copy_link = Copy copy_link = Copy

2
gogs.go

@ -17,7 +17,7 @@ import (
"github.com/gogits/gogs/modules/setting" "github.com/gogits/gogs/modules/setting"
) )
const APP_VER = "0.7.34.1208 Beta" const APP_VER = "0.7.35.1208 Beta"
func init() { func init() {
runtime.GOMAXPROCS(runtime.NumCPU()) runtime.GOMAXPROCS(runtime.NumCPU())

118
models/repo.go

@ -301,6 +301,10 @@ func (repo *Repository) RepoPath() string {
return repo.repoPath(x) return repo.repoPath(x)
} }
func (repo *Repository) GitConfigPath() string {
return filepath.Join(repo.RepoPath(), "config")
}
func (repo *Repository) RepoLink() string { func (repo *Repository) RepoLink() string {
return setting.AppSubUrl + "/" + repo.MustOwner().Name + "/" + repo.Name return setting.AppSubUrl + "/" + repo.MustOwner().Name + "/" + repo.Name
} }
@ -345,7 +349,7 @@ func (repo *Repository) LocalCopyPath() string {
func updateLocalCopy(repoPath, localPath string) error { func updateLocalCopy(repoPath, localPath string) error {
if !com.IsExist(localPath) { if !com.IsExist(localPath) {
if err := git.Clone(repoPath, localPath); err != nil { if err := git.Clone(repoPath, localPath, git.CloneRepoOptions{}); err != nil {
return fmt.Errorf("Clone: %v", err) return fmt.Errorf("Clone: %v", err)
} }
} else { } else {
@ -484,6 +488,8 @@ type Mirror struct {
Interval int // Hour. Interval int // Hour.
Updated time.Time `xorm:"UPDATED"` Updated time.Time `xorm:"UPDATED"`
NextUpdate time.Time NextUpdate time.Time
address string `xorm:"-"`
} }
func (m *Mirror) AfterSet(colName string, _ xorm.Cell) { func (m *Mirror) AfterSet(colName string, _ xorm.Cell) {
@ -497,6 +503,61 @@ func (m *Mirror) AfterSet(colName string, _ xorm.Cell) {
} }
} }
func (m *Mirror) readAddress() {
if len(m.address) > 0 {
return
}
cfg, err := ini.Load(m.Repo.GitConfigPath())
if err != nil {
log.Error(4, "Load: %v", err)
return
}
m.address = cfg.Section("remote \"origin\"").Key("url").Value()
}
// HandleCloneUserCredentials replaces user credentials from HTTP/HTTPS URL
// with placeholder <credentials>.
// It will fail for any other forms of clone addresses.
func HandleCloneUserCredentials(url string, mosaics bool) string {
i := strings.Index(url, "@")
if i == -1 {
return url
}
start := strings.Index(url, "://")
if start == -1 {
return url
}
if mosaics {
return url[:start+3] + "<credentials>" + url[i:]
}
return url[:start+3] + url[i+1:]
}
// Address returns mirror address from Git repository config without credentials.
func (m *Mirror) Address() string {
m.readAddress()
return HandleCloneUserCredentials(m.address, false)
}
// FullAddress returns mirror address from Git repository config.
func (m *Mirror) FullAddress() string {
m.readAddress()
return m.address
}
// SaveAddress writes new address to Git repository config.
func (m *Mirror) SaveAddress(addr string) error {
configPath := m.Repo.GitConfigPath()
cfg, err := ini.Load(configPath)
if err != nil {
return fmt.Errorf("Load: %v", err)
}
cfg.Section("remote \"origin\"").Key("url").SetValue(addr)
return cfg.SaveToIndent(configPath, "\t")
}
func getMirror(e Engine, repoId int64) (*Mirror, error) { func getMirror(e Engine, repoId int64) (*Mirror, error) {
m := &Mirror{RepoID: repoId} m := &Mirror{RepoID: repoId}
has, err := e.Get(m) has, err := e.Get(m)
@ -527,25 +588,6 @@ func createUpdateHook(repoPath string) error {
fmt.Sprintf(_TPL_UPDATE_HOOK, setting.ScriptType, "\""+setting.AppPath+"\"", setting.CustomConf)) fmt.Sprintf(_TPL_UPDATE_HOOK, setting.ScriptType, "\""+setting.AppPath+"\"", setting.CustomConf))
} }
// MirrorRepository creates a mirror repository from source.
func MirrorRepository(repoId int64, userName, repoName, repoPath, url string) error {
_, stderr, err := process.ExecTimeout(10*time.Minute,
fmt.Sprintf("MirrorRepository: %s/%s", userName, repoName),
"git", "clone", "--mirror", url, repoPath)
if err != nil {
return errors.New("git clone --mirror: " + stderr)
}
if _, err = x.InsertOne(&Mirror{
RepoID: repoId,
Interval: 24,
NextUpdate: time.Now().Add(24 * time.Hour),
}); err != nil {
return err
}
return nil
}
type MigrateRepoOptions struct { type MigrateRepoOptions struct {
Name string Name string
Description string Description string
@ -582,29 +624,35 @@ func MigrateRepository(u *User, opts MigrateRepoOptions) (*Repository, error) {
repo.NumWatches = 1 repo.NumWatches = 1
} }
repo.IsBare = false os.RemoveAll(repoPath)
if err = git.Clone(opts.RemoteAddr, repoPath, git.CloneRepoOptions{
Mirror: true,
Quiet: true,
Timeout: 10 * time.Minute,
}); err != nil {
return repo, fmt.Errorf("Clone: %v", err)
}
if opts.IsMirror { if opts.IsMirror {
if err = MirrorRepository(repo.ID, u.Name, repo.Name, repoPath, opts.RemoteAddr); err != nil { if _, err = x.InsertOne(&Mirror{
return repo, err RepoID: repo.ID,
Interval: 24,
NextUpdate: time.Now().Add(24 * time.Hour),
}); err != nil {
return repo, fmt.Errorf("InsertOne: %v", err)
} }
repo.IsMirror = true repo.IsMirror = true
return repo, UpdateRepository(repo, false) return repo, UpdateRepository(repo, false)
} else {
os.RemoveAll(repoPath)
} }
// FIXME: this command could for both migrate and mirror if err = createUpdateHook(repoPath); err != nil {
_, stderr, err := process.ExecTimeout(10*time.Minute, return repo, fmt.Errorf("createUpdateHook: %v", err)
fmt.Sprintf("MigrateRepository: %s", repoPath),
"git", "clone", "--mirror", "--bare", "--quiet", opts.RemoteAddr, repoPath)
if err != nil {
return repo, fmt.Errorf("git clone --mirror --bare --quiet: %v", stderr)
} else if err = createUpdateHook(repoPath); err != nil {
return repo, fmt.Errorf("create update hook: %v", err)
} }
// Clean up mirror info which prevents "push --all". // Clean up mirror info which prevents "push --all".
configPath := filepath.Join(repoPath, "/config") // This also removes possible user credentials.
configPath := repo.GitConfigPath()
cfg, err := ini.Load(configPath) cfg, err := ini.Load(configPath)
if err != nil { if err != nil {
return repo, fmt.Errorf("open config file: %v", err) return repo, fmt.Errorf("open config file: %v", err)
@ -615,7 +663,7 @@ func MigrateRepository(u *User, opts MigrateRepoOptions) (*Repository, error) {
} }
// Check if repository is empty. // Check if repository is empty.
_, stderr, err = com.ExecCmdDir(repoPath, "git", "log", "-1") _, stderr, err := com.ExecCmdDir(repoPath, "git", "log", "-1")
if err != nil { if err != nil {
if strings.Contains(stderr, "fatal: bad default revision 'HEAD'") { if strings.Contains(stderr, "fatal: bad default revision 'HEAD'") {
repo.IsBare = true repo.IsBare = true

16
modules/auth/repo_form.go

@ -69,6 +69,9 @@ func (f MigrateRepoForm) ParseRemoteAddr(user *models.User) (string, error) {
} }
if len(f.AuthUsername)+len(f.AuthPassword) > 0 { if len(f.AuthUsername)+len(f.AuthPassword) > 0 {
u.User = url.UserPassword(f.AuthUsername, f.AuthPassword) u.User = url.UserPassword(f.AuthUsername, f.AuthPassword)
} else {
// Fake user name and password to prevent prompt and fail quick.
u.User = url.UserPassword("fake_user", "")
} }
remoteAddr = u.String() remoteAddr = u.String()
} else if !user.CanImportLocal() { } else if !user.CanImportLocal() {
@ -81,12 +84,13 @@ func (f MigrateRepoForm) ParseRemoteAddr(user *models.User) (string, error) {
} }
type RepoSettingForm struct { type RepoSettingForm struct {
RepoName string `binding:"Required;AlphaDashDot;MaxSize(100)"` RepoName string `binding:"Required;AlphaDashDot;MaxSize(100)"`
Description string `binding:"MaxSize(255)"` Description string `binding:"MaxSize(255)"`
Website string `binding:"Url;MaxSize(100)"` Website string `binding:"Url;MaxSize(100)"`
Branch string Branch string
Interval int Interval int
Private bool MirrorAddress string
Private bool
// Advanced settings // Advanced settings
EnableWiki bool EnableWiki bool

8
modules/bindata/bindata.go

File diff suppressed because one or more lines are too long

1
modules/middleware/auth.go

@ -116,6 +116,7 @@ func Toggle(options *ToggleOptions) macaron.Handler {
ctx.Handle(500, "AutoSignIn", err) ctx.Handle(500, "AutoSignIn", err)
return return
} else if succeed { } else if succeed {
log.Trace("Auto-login succeed: %s", ctx.Session.Get("uname"))
ctx.Redirect(setting.AppSubUrl + ctx.Req.RequestURI) ctx.Redirect(setting.AppSubUrl + ctx.Req.RequestURI)
return return
} }

1
modules/middleware/repo.go

@ -129,6 +129,7 @@ func RepoAssignment(args ...bool) macaron.Handler {
return return
} }
ctx.Data["MirrorInterval"] = ctx.Repo.Mirror.Interval ctx.Data["MirrorInterval"] = ctx.Repo.Mirror.Interval
ctx.Data["Mirror"] = ctx.Repo.Mirror
} }
ctx.Repo.Repository = repo ctx.Repo.Repository = repo

2
routers/api/v1/repo/repo.go

@ -234,7 +234,7 @@ func Migrate(ctx *middleware.Context, form auth.MigrateRepoForm) {
log.Error(4, "DeleteRepository: %v", errDelete) log.Error(4, "DeleteRepository: %v", errDelete)
} }
} }
ctx.APIError(500, "MigrateRepository", err) ctx.APIError(500, "MigrateRepository", models.HandleCloneUserCredentials(err.Error(), true))
return return
} }

3
routers/install.go

@ -41,7 +41,8 @@ func checkRunMode() {
macaron.Env = macaron.PROD macaron.Env = macaron.PROD
macaron.ColorLog = false macaron.ColorLog = false
setting.ProdMode = true setting.ProdMode = true
git.Debug = false default:
git.Debug = true
} }
log.Info("Run Mode: %s", strings.Title(macaron.Env)) log.Info("Run Mode: %s", strings.Title(macaron.Env))
} }

6
routers/repo/repo.go

@ -192,7 +192,7 @@ func MigratePost(ctx *middleware.Context, form auth.MigrateRepoForm) {
RemoteAddr: remoteAddr, RemoteAddr: remoteAddr,
}) })
if err == nil { if err == nil {
log.Trace("Repository migrated[%d]: %s/%s", repo.ID, ctxUser.Name, form.RepoName) log.Trace("Repository migrated [%d]: %s/%s", repo.ID, ctxUser.Name, form.RepoName)
ctx.Redirect(setting.AppSubUrl + "/" + ctxUser.Name + "/" + form.RepoName) ctx.Redirect(setting.AppSubUrl + "/" + ctxUser.Name + "/" + form.RepoName)
return return
} }
@ -206,11 +206,11 @@ func MigratePost(ctx *middleware.Context, form auth.MigrateRepoForm) {
if strings.Contains(err.Error(), "Authentication failed") || if strings.Contains(err.Error(), "Authentication failed") ||
strings.Contains(err.Error(), "could not read Username") { strings.Contains(err.Error(), "could not read Username") {
ctx.Data["Err_Auth"] = true ctx.Data["Err_Auth"] = true
ctx.RenderWithErr(ctx.Tr("form.auth_failed", strings.Replace(err.Error(), ":"+form.AuthPassword+"@", ":<password>@", 1)), MIGRATE, &form) ctx.RenderWithErr(ctx.Tr("form.auth_failed", models.HandleCloneUserCredentials(err.Error(), true)), MIGRATE, &form)
return return
} else if strings.Contains(err.Error(), "fatal:") { } else if strings.Contains(err.Error(), "fatal:") {
ctx.Data["Err_CloneAddr"] = true ctx.Data["Err_CloneAddr"] = true
ctx.RenderWithErr(ctx.Tr("repo.migrate.failed", strings.Replace(err.Error(), ":"+form.AuthPassword+"@", ":<password>@", 1)), MIGRATE, &form) ctx.RenderWithErr(ctx.Tr("repo.migrate.failed", models.HandleCloneUserCredentials(err.Error(), true)), MIGRATE, &form)
return return
} }

7
routers/repo/setting.go

@ -109,9 +109,14 @@ func SettingsPost(ctx *middleware.Context, form auth.RepoSettingForm) {
ctx.Repo.Mirror.Interval = form.Interval ctx.Repo.Mirror.Interval = form.Interval
ctx.Repo.Mirror.NextUpdate = time.Now().Add(time.Duration(form.Interval) * time.Hour) ctx.Repo.Mirror.NextUpdate = time.Now().Add(time.Duration(form.Interval) * time.Hour)
if err := models.UpdateMirror(ctx.Repo.Mirror); err != nil { if err := models.UpdateMirror(ctx.Repo.Mirror); err != nil {
log.Error(4, "UpdateMirror: %v", err) ctx.Handle(500, "UpdateMirror", err)
return
} }
} }
if err := ctx.Repo.Mirror.SaveAddress(form.MirrorAddress); err != nil {
ctx.Handle(500, "SaveAddress", err)
return
}
} }
ctx.Flash.Success(ctx.Tr("repo.settings.update_settings_success")) ctx.Flash.Success(ctx.Tr("repo.settings.update_settings_success"))

2
templates/.VERSION

@ -1 +1 @@
0.7.34.1208 Beta 0.7.35.1208 Beta

2
templates/repo/header.tmpl

@ -8,7 +8,7 @@
<a href="{{AppSubUrl}}/{{.Owner.Name}}">{{.Owner.Name}}</a> <a href="{{AppSubUrl}}/{{.Owner.Name}}">{{.Owner.Name}}</a>
<div class="divider"> / </div> <div class="divider"> / </div>
<a href="{{$.RepoLink}}">{{.Name}}</a> <a href="{{$.RepoLink}}">{{.Name}}</a>
{{if .IsMirror}}<div class="ui label">{{$.i18n.Tr "mirror"}}</div>{{end}} {{if .IsMirror}}<div class="fork-flag">{{$.i18n.Tr "repo.mirror_from"}} <a target="_blank" href="{{$.MirrorAddress}}">{{$.Mirror.Address}}</a></div>{{end}}
{{if .IsFork}}<div class="fork-flag">{{$.i18n.Tr "repo.forked_from"}} <a href="{{.BaseRepo.RepoLink}}">{{SubStr .BaseRepo.RepoLink 1 -1}}</a></div>{{end}} {{if .IsFork}}<div class="fork-flag">{{$.i18n.Tr "repo.forked_from"}} <a href="{{.BaseRepo.RepoLink}}">{{SubStr .BaseRepo.RepoLink 1 -1}}</a></div>{{end}}
</div> </div>

5
templates/repo/settings/options.tmpl

@ -55,6 +55,11 @@
<label for="interval">{{.i18n.Tr "repo.mirror_interval"}}</label> <label for="interval">{{.i18n.Tr "repo.mirror_interval"}}</label>
<input id="interval" name="interval" type="number" value="{{.MirrorInterval}}"> <input id="interval" name="interval" type="number" value="{{.MirrorInterval}}">
</div> </div>
<div class="field">
<label for="mirror_address">{{.i18n.Tr "repo.mirror_address"}}</label>
<input id="mirror_address" name="mirror_address" value="{{.Mirror.FullAddress}}">
<p class="help">{{.i18n.Tr "repo.mirror_address_desc"}}</p>
</div>
{{end}} {{end}}
<div class="ui divider"></div> <div class="ui divider"></div>

Loading…
Cancel
Save