#1984 Better mirror repo management

.gopmfile

@@ -17,7 +17,7 @@ github.com/go-sql-driver/mysql = commit:d512f20
 github.com/go-xorm/core = commit:acb6f00
 github.com/go-xorm/xorm = commit:a8fba4d
 github.com/gogits/chardet = commit:2404f77725
-github.com/gogits/git-shell = commit:de77627
+github.com/gogits/git-shell = 
 github.com/gogits/go-gogs-client = commit:4b541fa
 github.com/issue9/identicon = commit:f8c0d2c
 github.com/klauspost/compress = commit:bcd0709

README.md

@@ -5,7 +5,7 @@ Gogs - Go Git Service [![Build Status](https://travis-ci.org/gogits/gogs.svg?bra
 
 ![](public/img/gogs-large-resize.png)
 
-##### Current version: 0.7.34 Beta
+##### Current version: 0.7.35 Beta
 
 | Web | UI  | Preview  |
 |:-------------:|:-------:|:-------:|

conf/locale/locale_en-US.ini

@@ -352,6 +352,8 @@ auto_init = Initialize this repository with selected files and template
 create_repo = Create Repository
 default_branch = Default Branch
 mirror_interval = Mirror Interval (hour)
+mirror_address = Mirror Address
+mirror_address_desc = Please include necessary user credentials in the address.
 watchers = Watchers
 stargazers = Stargazers
 forks = Forks
@@ -369,6 +371,7 @@ migrate.permission_denied = You are not allowed to import local repositories.
 migrate.invalid_local_path = Invalid local path, it does not exist or not a directory.
 migrate.failed = Migration failed: %v
 
+mirror_from = mirror from
 forked_from = forked from
 fork_from_self = You cannot fork a repository you already own!
 copy_link = Copy

gogs.go

@@ -17,7 +17,7 @@ import (
 	"github.com/gogits/gogs/modules/setting"
 )
 
-const APP_VER = "0.7.34.1208 Beta"
+const APP_VER = "0.7.35.1208 Beta"
 
 func init() {
 	runtime.GOMAXPROCS(runtime.NumCPU())

models/repo.go

@@ -301,6 +301,10 @@ func (repo *Repository) RepoPath() string {
 	return repo.repoPath(x)
 }
 
+func (repo *Repository) GitConfigPath() string {
+	return filepath.Join(repo.RepoPath(), "config")
+}
+
 func (repo *Repository) RepoLink() string {
 	return setting.AppSubUrl + "/" + repo.MustOwner().Name + "/" + repo.Name
 }
@@ -345,7 +349,7 @@ func (repo *Repository) LocalCopyPath() string {
 
 func updateLocalCopy(repoPath, localPath string) error {
 	if !com.IsExist(localPath) {
-		if err := git.Clone(repoPath, localPath); err != nil {
+		if err := git.Clone(repoPath, localPath, git.CloneRepoOptions{}); err != nil {
 			return fmt.Errorf("Clone: %v", err)
 		}
 	} else {
@@ -484,6 +488,8 @@ type Mirror struct {
 	Interval   int         // Hour.
 	Updated    time.Time   `xorm:"UPDATED"`
 	NextUpdate time.Time
+
+	address string `xorm:"-"`
 }
 
 func (m *Mirror) AfterSet(colName string, _ xorm.Cell) {
@@ -497,6 +503,61 @@ func (m *Mirror) AfterSet(colName string, _ xorm.Cell) {
 	}
 }
 
+func (m *Mirror) readAddress() {
+	if len(m.address) > 0 {
+		return
+	}
+
+	cfg, err := ini.Load(m.Repo.GitConfigPath())
+	if err != nil {
+		log.Error(4, "Load: %v", err)
+		return
+	}
+	m.address = cfg.Section("remote \"origin\"").Key("url").Value()
+}
+
+// HandleCloneUserCredentials replaces user credentials from HTTP/HTTPS URL
+// with placeholder <credentials>.
+// It will fail for any other forms of clone addresses.
+func HandleCloneUserCredentials(url string, mosaics bool) string {
+	i := strings.Index(url, "@")
+	if i == -1 {
+		return url
+	}
+	start := strings.Index(url, "://")
+	if start == -1 {
+		return url
+	}
+	if mosaics {
+		return url[:start+3] + "<credentials>" + url[i:]
+	}
+	return url[:start+3] + url[i+1:]
+}
+
+// Address returns mirror address from Git repository config without credentials.
+func (m *Mirror) Address() string {
+	m.readAddress()
+	return HandleCloneUserCredentials(m.address, false)
+}
+
+// FullAddress returns mirror address from Git repository config.
+func (m *Mirror) FullAddress() string {
+	m.readAddress()
+	return m.address
+}
+
+// SaveAddress writes new address to Git repository config.
+func (m *Mirror) SaveAddress(addr string) error {
+	configPath := m.Repo.GitConfigPath()
+	cfg, err := ini.Load(configPath)
+	if err != nil {
+		return fmt.Errorf("Load: %v", err)
+	}
+
+	cfg.Section("remote \"origin\"").Key("url").SetValue(addr)
+	return cfg.SaveToIndent(configPath, "\t")
+}
+
 func getMirror(e Engine, repoId int64) (*Mirror, error) {
 	m := &Mirror{RepoID: repoId}
 	has, err := e.Get(m)
@@ -527,25 +588,6 @@ func createUpdateHook(repoPath string) error {
 		fmt.Sprintf(_TPL_UPDATE_HOOK, setting.ScriptType, "\""+setting.AppPath+"\"", setting.CustomConf))
 }
 
-// MirrorRepository creates a mirror repository from source.
-func MirrorRepository(repoId int64, userName, repoName, repoPath, url string) error {
-	_, stderr, err := process.ExecTimeout(10*time.Minute,
-		fmt.Sprintf("MirrorRepository: %s/%s", userName, repoName),
-		"git", "clone", "--mirror", url, repoPath)
-	if err != nil {
-		return errors.New("git clone --mirror: " + stderr)
-	}
-
-	if _, err = x.InsertOne(&Mirror{
-		RepoID:     repoId,
-		Interval:   24,
-		NextUpdate: time.Now().Add(24 * time.Hour),
-	}); err != nil {
-		return err
-	}
-	return nil
-}
-
 type MigrateRepoOptions struct {
 	Name        string
 	Description string
@@ -582,29 +624,35 @@ func MigrateRepository(u *User, opts MigrateRepoOptions) (*Repository, error) {
 		repo.NumWatches = 1
 	}
 
-	repo.IsBare = false
+	os.RemoveAll(repoPath)
+	if err = git.Clone(opts.RemoteAddr, repoPath, git.CloneRepoOptions{
+		Mirror:  true,
+		Quiet:   true,
+		Timeout: 10 * time.Minute,
+	}); err != nil {
+		return repo, fmt.Errorf("Clone: %v", err)
+	}
+
 	if opts.IsMirror {
-		if err = MirrorRepository(repo.ID, u.Name, repo.Name, repoPath, opts.RemoteAddr); err != nil {
+		if _, err = x.InsertOne(&Mirror{
-			return repo, err
+			RepoID:     repo.ID,
+			Interval:   24,
+			NextUpdate: time.Now().Add(24 * time.Hour),
+		}); err != nil {
+			return repo, fmt.Errorf("InsertOne: %v", err)
 		}
+
 		repo.IsMirror = true
 		return repo, UpdateRepository(repo, false)
-	} else {
-		os.RemoveAll(repoPath)
 	}
 
-	// FIXME: this command could for both migrate and mirror
+	if err = createUpdateHook(repoPath); err != nil {
-	_, stderr, err := process.ExecTimeout(10*time.Minute,
+		return repo, fmt.Errorf("createUpdateHook: %v", err)
-		fmt.Sprintf("MigrateRepository: %s", repoPath),
-		"git", "clone", "--mirror", "--bare", "--quiet", opts.RemoteAddr, repoPath)
-	if err != nil {
-		return repo, fmt.Errorf("git clone --mirror --bare --quiet: %v", stderr)
-	} else if err = createUpdateHook(repoPath); err != nil {
-		return repo, fmt.Errorf("create update hook: %v", err)
 	}
 
 	// Clean up mirror info which prevents "push --all".
-	configPath := filepath.Join(repoPath, "/config")
+	// This also removes possible user credentials.
+	configPath := repo.GitConfigPath()
 	cfg, err := ini.Load(configPath)
 	if err != nil {
 		return repo, fmt.Errorf("open config file: %v", err)
@@ -615,7 +663,7 @@ func MigrateRepository(u *User, opts MigrateRepoOptions) (*Repository, error) {
 	}
 
 	// Check if repository is empty.
-	_, stderr, err = com.ExecCmdDir(repoPath, "git", "log", "-1")
+	_, stderr, err := com.ExecCmdDir(repoPath, "git", "log", "-1")
 	if err != nil {
 		if strings.Contains(stderr, "fatal: bad default revision 'HEAD'") {
 			repo.IsBare = true

modules/auth/repo_form.go

@@ -69,6 +69,9 @@ func (f MigrateRepoForm) ParseRemoteAddr(user *models.User) (string, error) {
 		}
 		if len(f.AuthUsername)+len(f.AuthPassword) > 0 {
 			u.User = url.UserPassword(f.AuthUsername, f.AuthPassword)
+		} else {
+			// Fake user name and password to prevent prompt and fail quick.
+			u.User = url.UserPassword("fake_user", "")
 		}
 		remoteAddr = u.String()
 	} else if !user.CanImportLocal() {
@@ -81,12 +84,13 @@ func (f MigrateRepoForm) ParseRemoteAddr(user *models.User) (string, error) {
 }
 
 type RepoSettingForm struct {
-	RepoName    string `binding:"Required;AlphaDashDot;MaxSize(100)"`
+	RepoName      string `binding:"Required;AlphaDashDot;MaxSize(100)"`
-	Description string `binding:"MaxSize(255)"`
+	Description   string `binding:"MaxSize(255)"`
-	Website     string `binding:"Url;MaxSize(100)"`
+	Website       string `binding:"Url;MaxSize(100)"`
-	Branch      string
+	Branch        string
-	Interval    int
+	Interval      int
-	Private     bool
+	MirrorAddress string
+	Private       bool
 
 	// Advanced settings
 	EnableWiki            bool

modules/middleware/auth.go

@@ -116,6 +116,7 @@ func Toggle(options *ToggleOptions) macaron.Handler {
 				ctx.Handle(500, "AutoSignIn", err)
 				return
 			} else if succeed {
+				log.Trace("Auto-login succeed: %s", ctx.Session.Get("uname"))
 				ctx.Redirect(setting.AppSubUrl + ctx.Req.RequestURI)
 				return
 			}

modules/middleware/repo.go

@@ -129,6 +129,7 @@ func RepoAssignment(args ...bool) macaron.Handler {
 				return
 			}
 			ctx.Data["MirrorInterval"] = ctx.Repo.Mirror.Interval
+			ctx.Data["Mirror"] = ctx.Repo.Mirror
 		}
 
 		ctx.Repo.Repository = repo

routers/api/v1/repo/repo.go

@@ -234,7 +234,7 @@ func Migrate(ctx *middleware.Context, form auth.MigrateRepoForm) {
 				log.Error(4, "DeleteRepository: %v", errDelete)
 			}
 		}
-		ctx.APIError(500, "MigrateRepository", err)
+		ctx.APIError(500, "MigrateRepository", models.HandleCloneUserCredentials(err.Error(), true))
 		return
 	}
 

routers/install.go

@@ -41,7 +41,8 @@ func checkRunMode() {
 		macaron.Env = macaron.PROD
 		macaron.ColorLog = false
 		setting.ProdMode = true
-		git.Debug = false
+	default:
+		git.Debug = true
 	}
 	log.Info("Run Mode: %s", strings.Title(macaron.Env))
 }

routers/repo/repo.go

@@ -192,7 +192,7 @@ func MigratePost(ctx *middleware.Context, form auth.MigrateRepoForm) {
 		RemoteAddr:  remoteAddr,
 	})
 	if err == nil {
-		log.Trace("Repository migrated[%d]: %s/%s", repo.ID, ctxUser.Name, form.RepoName)
+		log.Trace("Repository migrated [%d]: %s/%s", repo.ID, ctxUser.Name, form.RepoName)
 		ctx.Redirect(setting.AppSubUrl + "/" + ctxUser.Name + "/" + form.RepoName)
 		return
 	}
@@ -206,11 +206,11 @@ func MigratePost(ctx *middleware.Context, form auth.MigrateRepoForm) {
 	if strings.Contains(err.Error(), "Authentication failed") ||
 		strings.Contains(err.Error(), "could not read Username") {
 		ctx.Data["Err_Auth"] = true
-		ctx.RenderWithErr(ctx.Tr("form.auth_failed", strings.Replace(err.Error(), ":"+form.AuthPassword+"@", ":<password>@", 1)), MIGRATE, &form)
+		ctx.RenderWithErr(ctx.Tr("form.auth_failed", models.HandleCloneUserCredentials(err.Error(), true)), MIGRATE, &form)
 		return
 	} else if strings.Contains(err.Error(), "fatal:") {
 		ctx.Data["Err_CloneAddr"] = true
-		ctx.RenderWithErr(ctx.Tr("repo.migrate.failed", strings.Replace(err.Error(), ":"+form.AuthPassword+"@", ":<password>@", 1)), MIGRATE, &form)
+		ctx.RenderWithErr(ctx.Tr("repo.migrate.failed", models.HandleCloneUserCredentials(err.Error(), true)), MIGRATE, &form)
 		return
 	}
 

routers/repo/setting.go

@@ -109,9 +109,14 @@ func SettingsPost(ctx *middleware.Context, form auth.RepoSettingForm) {
 				ctx.Repo.Mirror.Interval = form.Interval
 				ctx.Repo.Mirror.NextUpdate = time.Now().Add(time.Duration(form.Interval) * time.Hour)
 				if err := models.UpdateMirror(ctx.Repo.Mirror); err != nil {
-					log.Error(4, "UpdateMirror: %v", err)
+					ctx.Handle(500, "UpdateMirror", err)
+					return
 				}
 			}
+			if err := ctx.Repo.Mirror.SaveAddress(form.MirrorAddress); err != nil {
+				ctx.Handle(500, "SaveAddress", err)
+				return
+			}
 		}
 
 		ctx.Flash.Success(ctx.Tr("repo.settings.update_settings_success"))

templates/.VERSION

@@ -1 +1 @@
-0.7.34.1208 Beta
+0.7.35.1208 Beta

templates/repo/header.tmpl

@@ -8,7 +8,7 @@
 						<a href="{{AppSubUrl}}/{{.Owner.Name}}">{{.Owner.Name}}</a>
 						<div class="divider"> / </div>
 						<a href="{{$.RepoLink}}">{{.Name}}</a>
-						{{if .IsMirror}}<div class="ui label">{{$.i18n.Tr "mirror"}}</div>{{end}}
+						{{if .IsMirror}}<div class="fork-flag">{{$.i18n.Tr "repo.mirror_from"}} <a target="_blank" href="{{$.MirrorAddress}}">{{$.Mirror.Address}}</a></div>{{end}}
 						{{if .IsFork}}<div class="fork-flag">{{$.i18n.Tr "repo.forked_from"}} <a href="{{.BaseRepo.RepoLink}}">{{SubStr .BaseRepo.RepoLink 1 -1}}</a></div>{{end}}
 					</div>
 

templates/repo/settings/options.tmpl

@@ -55,6 +55,11 @@
 								<label for="interval">{{.i18n.Tr "repo.mirror_interval"}}</label>
 								<input id="interval" name="interval" type="number" value="{{.MirrorInterval}}">
 							</div>
+							<div class="field">
+								<label for="mirror_address">{{.i18n.Tr "repo.mirror_address"}}</label>
+								<input id="mirror_address" name="mirror_address" value="{{.Mirror.FullAddress}}">
+								<p class="help">{{.i18n.Tr "repo.mirror_address_desc"}}</p>
+							</div>
 						{{end}}
 
 						<div class="ui divider"></div>