WIP import gitlab command

cmd/import.go

@@ -0,0 +1,65 @@
+// Copyright 2015 The Gogs Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package cmd
+
+import (
+	"log"
+	"net/url"
+
+	"github.com/codegangsta/cli"
+
+	"github.com/gogits/gogs/models"
+	"github.com/gogits/gogs/modules/importer"
+	"github.com/gogits/gogs/modules/setting"
+)
+
+var CmdImport = cli.Command{
+	Name:  "import",
+	Usage: "Import foreign project tracking service",
+	Description: `Import connects to foreign project tracking service API and imports all common entities such as users, projects, issues into Gogs.
+This can be used to migrate from other service to Gogs.`,
+	Subcommands: []cli.Command{
+		{
+			Name:   "gitlab",
+			Usage:  "import from GitLab",
+			Action: importGitLab,
+			Flags: []cli.Flag{
+				stringFlag("url", "", "GitLab service base URL"),
+				stringFlag("token", "", "private authentication token"),
+			},
+		},
+	},
+	Flags: []cli.Flag{
+		stringFlag("config, c", "custom/conf/app.ini", "Custom configuration file path"),
+		boolFlag("verbose, v", "show process details"),
+	},
+}
+
+func importGitLab(ctx *cli.Context) {
+	baseUrl, err := url.Parse(ctx.String("url"))
+	if err != nil {
+		log.Fatal("Required --url parameter is missing or not valid URL")
+	}
+	if len(baseUrl.Path) > 0 {
+		log.Fatal("Provided --url parameter must not contain path")
+	}
+	token := ctx.String("token")
+	if len(token) == 0 {
+		log.Fatal("Missing required --token parameter")
+	}
+
+	if ctx.IsSet("config") {
+		setting.CustomConf = ctx.String("config")
+	}
+	setting.NewContext()
+	models.LoadConfigs()
+	models.SetEngine()
+
+	if importer.ImportGitLab(baseUrl, token) == nil {
+		log.Println("Finished importing!")
+	} else {
+		log.Println("Import failed!")
+	}
+}

conf/app.ini

@@ -117,6 +117,8 @@ ENABLE_REVERSE_PROXY_AUTO_REGISTRATION = false
 DISABLE_MINIMUM_KEY_SIZE_CHECK = false
 ; Enable captcha validation for registration
 ENABLE_CAPTCHA = true
+; Enable bcrypt password encryption fallback
+BCRYPT_AUTH_FALLBACK = false
 
 ; used to filter keys which are too short
 [service.minimum_key_sizes]

gogs.go

@@ -35,6 +35,7 @@ func main() {
 		cmd.CmdUpdate,
 		cmd.CmdDump,
 		cmd.CmdCert,
+		cmd.CmdImport,
 	}
 	app.Flags = append(app.Flags, []cli.Flag{}...)
 	app.Run(os.Args)

models/org.go

@@ -9,6 +9,7 @@ import (
 	"fmt"
 	"os"
 	"strings"
+	"time"
 
 	"github.com/go-xorm/xorm"
 )
@@ -113,6 +114,9 @@ func CreateOrganization(org, owner *User) (err error) {
 	org.NumMembers = 1
 
 	sess := x.NewSession()
+	if (org.Created != time.Time{}) {
+		sess.NoAutoTime()
+	}
 	defer sessionRelease(sess)
 	if err = sess.Begin(); err != nil {
 		return err

models/publickey.go

@@ -303,7 +303,7 @@ func addKey(e Engine, key *PublicKey) (err error) {
 }
 
 // AddPublicKey adds new public key to database and authorized_keys file.
-func AddPublicKey(ownerID int64, name, content string) (err error) {
+func AddPublicKeyCreated(ownerID int64, name, content string, created time.Time) (err error) {
 	if err = checkKeyContent(content); err != nil {
 		return err
 	}
@@ -318,6 +318,9 @@ func AddPublicKey(ownerID int64, name, content string) (err error) {
 
 	sess := x.NewSession()
 	defer sessionRelease(sess)
+	if (created != time.Time{}) {
+		sess.NoAutoTime()
+	}
 	if err = sess.Begin(); err != nil {
 		return err
 	}
@@ -328,6 +331,7 @@ func AddPublicKey(ownerID int64, name, content string) (err error) {
 		Content: content,
 		Mode:    ACCESS_MODE_WRITE,
 		Type:    KEY_TYPE_USER,
+		Created: created,
 	}
 	if err = addKey(sess, key); err != nil {
 		return fmt.Errorf("addKey: %v", err)
@@ -336,6 +340,11 @@ func AddPublicKey(ownerID int64, name, content string) (err error) {
 	return sess.Commit()
 }
 
+// AddPublicKey adds new public key to database and authorized_keys file.
+func AddPublicKey(ownerID int64, name, content string) (err error) {
+	return AddPublicKeyCreated(ownerID, name, content, time.Time{})
+}
+
 // GetPublicKeyByID returns public key by given ID.
 func GetPublicKeyByID(keyID int64) (*PublicKey, error) {
 	key := new(PublicKey)

models/repo.go

@@ -612,6 +612,7 @@ type CreateRepoOptions struct {
 	IsPrivate   bool
 	IsMirror    bool
 	AutoInit    bool
+	Created     time.Time
 }
 
 func getRepoInitFile(tp, name string) ([]byte, error) {
@@ -756,9 +757,14 @@ func createRepository(e *xorm.Session, u *User, repo *Repository) (err error) {
 		return ErrRepoAlreadyExist{u.Name, repo.Name}
 	}
 
+	prevUseAutoTime := e.Statement.UseAutoTime
+	if (repo.Created != time.Time{}) {
+		e.NoAutoTime()
+	}
 	if _, err = e.Insert(repo); err != nil {
 		return err
 	}
+	e.Statement.UseAutoTime = prevUseAutoTime
 
 	u.NumRepos++
 	// Remember visibility preference.
@@ -800,6 +806,7 @@ func CreateRepository(u *User, opts CreateRepoOptions) (_ *Repository, err error
 		LowerName:   strings.ToLower(opts.Name),
 		Description: opts.Description,
 		IsPrivate:   opts.IsPrivate,
+		Created:     opts.Created,
 	}
 
 	sess := x.NewSession()

models/user.go

@@ -454,6 +454,9 @@ func CreateUser(u *User) (err error) {
 	u.EncodePasswd()
 
 	sess := x.NewSession()
+	if (u.Created != time.Time{}) {
+		sess.NoAutoTime()
+	}
 	defer sess.Close()
 	if err = sess.Begin(); err != nil {
 		return err

modules/importer/gitlab.go

@@ -0,0 +1,303 @@
+// Copyright 2015 The Gogs Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package importer
+
+import (
+	"errors"
+	"fmt"
+	"log"
+	"net/url"
+	"sort"
+	"time"
+
+	"github.com/gogits/gogs/models"
+)
+
+func ImportGitLab(baseUrl *url.URL, token string) error {
+	if err := importGitLabUsers(baseUrl, token); err != nil {
+		return err
+	}
+	if err := importGitLabOrgs(baseUrl, token); err != nil {
+		return err
+	}
+	if err := importGitLabRepos(baseUrl, token); err != nil {
+		return err
+	}
+	return nil
+}
+
+// https://gitlab.com/gitlab-org/gitlab-ce/tree/master/doc/api
+// https://gitlab.com/gitlab-org/gitlab-ce/blob/master/doc/api/users.md
+
+type GitLabUser struct {
+	Id       int64     `json:"id"`
+	Name     string    `json:"username"`
+	FullName string    `json:"name"`
+	Email    string    `json:"email"`
+	Passwd   string    `json:"password_hash"` // this requires API patch (returns bcrypt hash)
+	State    string    `json:"state"`         // "active", "inactive"
+	IsAdmin  bool      `json:"is_admin"`
+	Website  string    `json:"website_url"`
+	Created  time.Time `json:"created_at"`
+}
+type GitLabUsers []GitLabUser
+
+func (slice GitLabUsers) Len() int           { return len(slice) }
+func (slice GitLabUsers) Less(i, j int) bool { return slice[i].Id < slice[j].Id }
+func (slice GitLabUsers) Swap(i, j int)      { slice[i], slice[j] = slice[j], slice[i] }
+
+type GitLabKey struct {
+	Id      int64     `json:"id"`
+	Name    string    `json:"title"`
+	Key     string    `json:"key"`
+	Created time.Time `json:"created_at"`
+}
+type GitLabKeys []GitLabKey
+
+func (slice GitLabKeys) Len() int           { return len(slice) }
+func (slice GitLabKeys) Less(i, j int) bool { return slice[i].Id < slice[j].Id }
+func (slice GitLabKeys) Swap(i, j int)      { slice[i], slice[j] = slice[j], slice[i] }
+
+func importGitLabUsers(baseUrl *url.URL, token string) error {
+	var remoteUsers GitLabUsers
+	if err := fetchObjects("/api/v3/users", baseUrl, token, &remoteUsers); err != nil {
+		return err
+	}
+	sort.Sort(remoteUsers)
+
+	for _, remoteUser := range remoteUsers {
+		if user, _ := models.GetUserByName(remoteUser.Name); user == nil {
+			log.Printf("User: %s (%s)", remoteUser.Name, remoteUser.FullName)
+			user := &models.User{
+				Name:      remoteUser.Name,
+				FullName:  remoteUser.FullName,
+				Email:     remoteUser.Email,
+				IsActive:  remoteUser.State == "active",
+				IsAdmin:   remoteUser.IsAdmin,
+				Website:   remoteUser.Website,
+				Created:   remoteUser.Created,
+				LoginType: models.PLAIN,
+			}
+			err := models.CreateUser(user)
+			if err != nil {
+				log.Fatalf("Cannot create user: %s", err)
+				return err
+			}
+			// extra step needed to put bcrypt hash directly
+			user.Passwd = remoteUser.Passwd
+			models.UpdateUser(user)
+			var remoteKeys GitLabKeys
+			if err := fetchObjects(fmt.Sprintf("/api/v3/users/%d/keys", remoteUser.Id), baseUrl, token, &remoteKeys); err != nil {
+				return err
+			} else {
+				sort.Sort(remoteKeys)
+				for _, remoteKey := range remoteKeys {
+					if err := models.AddPublicKeyCreated(user.Id, remoteKey.Name, remoteKey.Key, remoteKey.Created); err != nil {
+						if models.IsErrKeyNameAlreadyUsed(err) {
+							log.Printf("Duplicate key: %s", remoteKey.Name, remoteUser.Name, err)
+						} else {
+							log.Fatalf("Cannot add public key %s to user %s: %s", remoteKey.Name, remoteUser.Name, err)
+							return err
+						}
+					} else {
+						log.Printf("  Keys <- %s", remoteKey.Name)
+					}
+				}
+			}
+		} else {
+			log.Printf("User %s already exists!", remoteUser.Name)
+		}
+	}
+	return nil
+}
+
+// https://gitlab.com/gitlab-org/gitlab-ce/blob/master/doc/api/groups.md
+
+type GitLabOrg struct {
+	Id          int64  `json:"id"`
+	Name        string `json:"path"`
+	FullName    string `json:"name"`
+	Email       string `json:"email"`
+	Description string `json:"description"`
+}
+type GitLabOrgs []GitLabOrg
+
+func (slice GitLabOrgs) Len() int           { return len(slice) }
+func (slice GitLabOrgs) Less(i, j int) bool { return slice[i].Id < slice[j].Id }
+func (slice GitLabOrgs) Swap(i, j int)      { slice[i], slice[j] = slice[j], slice[i] }
+
+type GitLabAccessLevel int
+
+const (
+	GitLabNoAccess  GitLabAccessLevel = iota
+	GitLabGuest                       = 10
+	GitLabReporter                    = 20
+	GitLabDeveloper                   = 30
+	GitLabMaster                      = 40
+	GitLabOwner                       = 50
+)
+
+type GitLabMember struct {
+	Id          int64             `json:"id"`
+	Name        string            `json:"username"`
+	FullName    string            `json:"name"`
+	Email       string            `json:"email"`
+	AccessLevel GitLabAccessLevel `json:"access_level"`
+}
+type GitLabMembers []GitLabMember
+
+func (slice GitLabMembers) Len() int           { return len(slice) }
+func (slice GitLabMembers) Less(i, j int) bool { return slice[i].AccessLevel > slice[j].AccessLevel }
+func (slice GitLabMembers) Swap(i, j int)      { slice[i], slice[j] = slice[j], slice[i] }
+
+func importGitLabOrgs(baseUrl *url.URL, token string) error {
+	var remoteOrgs GitLabOrgs
+	if err := fetchObjects("/api/v3/groups", baseUrl, token, &remoteOrgs); err != nil {
+		return err
+	}
+	sort.Sort(remoteOrgs)
+
+	for _, remoteOrg := range remoteOrgs {
+		if org, _ := models.GetOrgByName(remoteOrg.Name); org == nil {
+			var remoteMembers GitLabMembers
+			if err := fetchObjects(fmt.Sprintf("/api/v3/groups/%d/members", remoteOrg.Id), baseUrl, token, &remoteMembers); err != nil {
+				return err
+			}
+			if len(remoteMembers) == 0 {
+				log.Fatalf("No members in org: %s! Skipping", remoteOrg.Name)
+				continue
+			}
+			if owner, _ := models.GetUserByName(remoteMembers[0].Name); owner == nil {
+				log.Fatalf("Inexistent owner %s (%s) for org: %s!", remoteMembers[0].FullName, remoteMembers[0].Name, remoteOrg.Name)
+			} else {
+				log.Printf("Org: %s (%s)", remoteOrg.Name, remoteOrg.FullName)
+				log.Printf("  %s (%s) -> Owners", owner.FullName, owner.Name)
+				org := &models.User{
+					Name:     remoteOrg.Name,
+					FullName: remoteOrg.FullName,
+					Created:  owner.Created, // GitLab does not expose creation time, use owner's
+					IsActive: true,
+					Type:     models.ORGANIZATION,
+				}
+				err := models.CreateOrganization(org, owner)
+				if err != nil {
+					log.Fatalf("Cannot create org: %s", err)
+					return err
+				}
+				ownersTeam, err := org.GetOwnerTeam()
+				if err != nil {
+					log.Fatalf("Cannot get org owners team: %s", err)
+					return err
+				}
+				var adminsTeam, writersTeam, readersTeam *models.Team
+				for index, remoteMember := range remoteMembers {
+					if index == 0 {
+						continue
+					}
+					if member, _ := models.GetUserByName(remoteMember.Name); owner == nil {
+						log.Fatalf("Inexistent member %s (%s) for org: %s!", remoteMember.FullName, remoteMember.Name, remoteOrg.Name)
+					} else {
+						var team *models.Team
+						switch {
+						case remoteMember.AccessLevel >= GitLabOwner:
+							team = ownersTeam
+							err = nil
+						case remoteMember.AccessLevel >= GitLabMaster:
+							team, err = getOrgTeam(org, &adminsTeam, "Admins", models.ACCESS_MODE_ADMIN)
+						case remoteMember.AccessLevel >= GitLabDeveloper:
+							team, err = getOrgTeam(org, &writersTeam, "Writers", models.ACCESS_MODE_WRITE)
+						case remoteMember.AccessLevel >= GitLabGuest:
+							team, err = getOrgTeam(org, &readersTeam, "Readers", models.ACCESS_MODE_READ)
+						}
+						if err != nil {
+							return err
+						}
+						if err := team.AddMember(member.Id); err != nil {
+							log.Fatalf("Cannot add member %s (%s) to org: %s, %s", member.FullName, member.Name, remoteOrg.Name, err)
+						}
+						log.Printf("  %s (%s) -> %s", member.FullName, member.Name, team.Name)
+					}
+				}
+			}
+		} else {
+			log.Printf("Org %s already exists!", remoteOrg.Name)
+		}
+	}
+	return nil
+}
+
+// https://gitlab.com/gitlab-org/gitlab-ce/blob/master/doc/api/projects.md
+
+type GitLabNamespace struct {
+	Id          int64  `json:"id"`
+	Name        string `json:"path"`
+	FullName    string `json:"name"`
+	Description string `json:"description"`
+}
+type GitLabRepo struct {
+	Id              int64           `json:"id"`
+	Name            string          `json:"path"`
+	Description     string          `json:"description"`
+	Created         time.Time       `json:"created_at"`
+	IsPublic        bool            `json:"public"`
+	VisibilityLevel int             `json:"visibility_level"` // (0) private, (10) internal, (20) public
+	Owner           GitLabUser      `json:"owner"`
+	Namespace       GitLabNamespace `json:"namespace"`
+	DefaultBranch   string
+}
+type GitLabRepos []GitLabRepo
+
+func importGitLabRepos(baseUrl *url.URL, token string) error {
+	var remoteRepos GitLabRepos
+	if err := fetchObjects("/api/v3/projects/all?order_by=id&sort=asc", baseUrl, token, &remoteRepos); err != nil {
+		return err
+	}
+
+	for _, remoteRepo := range remoteRepos {
+		if owner, _ := models.GetUserByName(remoteRepo.Namespace.Name); owner == nil {
+			log.Fatalf("Cannot find owner: %s for repo: %s", remoteRepo.Namespace.Name, remoteRepo.Name)
+			return errors.New("Cannot find repo owner")
+		} else {
+			if repo, _ := models.GetRepositoryByName(owner.Id, remoteRepo.Name); repo == nil {
+				log.Printf("Repo: %s (%s)", remoteRepo.Name, remoteRepo.Namespace.Name)
+				repo, err := models.CreateRepository(owner, models.CreateRepoOptions{
+					Name:        remoteRepo.Name,
+					Description: remoteRepo.Description,
+					IsPrivate:   remoteRepo.VisibilityLevel == 0,
+					Created:     remoteRepo.Created,
+					IsMirror:    false,
+					AutoInit:    true,
+					Readme:      "Default",
+				})
+				if err != nil {
+					log.Fatalf("Cannot create repo: %s", err)
+					return err
+				}
+				// NOTE: Gogs only supports plain list of collaborators in project, while GitLab has fine grained access control
+				var remoteMembers GitLabMembers
+				if err := fetchObjects(fmt.Sprintf("/api/v3/projects/%d/members", remoteRepo.Id), baseUrl, token, &remoteMembers); err != nil {
+					return err
+				}
+				for _, remoteMember := range remoteMembers {
+					if member, _ := models.GetUserByName(remoteMember.Name); owner == nil {
+						log.Fatalf("Inexistent member %s (%s) for repo: %s!", remoteMember.FullName, remoteMember.Name, remoteRepo.Name)
+					} else {
+						if member.Id == owner.Id {
+							continue // skip if we got owner here
+						}
+						if err := repo.AddCollaborator(member); err != nil {
+							log.Fatalf("Cannot add collaborator %s (%s) to repo: %s, %s", member.FullName, member.Name, remoteRepo.Name, err)
+							return err
+						}
+						log.Printf("  %s (%s) -> Collaborators", member.FullName, member.Name)
+					}
+				}
+			} else {
+				log.Printf("Repo %s already exists!", remoteRepo.Name)
+			}
+		}
+	}
+	return nil
+}

modules/importer/utils.go

@@ -0,0 +1,110 @@
+// Copyright 2015 The Gogs Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package importer
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"net/url"
+	"reflect"
+	"strings"
+
+	"github.com/gogits/gogs/models"
+	"github.com/gogits/gogs/modules/httplib"
+)
+
+func fetchURL(api string, baseUrl *url.URL, token string) ([]byte, error) {
+	baseUrl.Path = api
+	log.Printf("Fetching: %s", baseUrl)
+	req := httplib.Get(baseUrl.String()).Header("PRIVATE-TOKEN", token)
+	resp, err := req.Response()
+	if err != nil {
+		log.Fatalf("Cannot connect: %s", err)
+		return nil, err
+	}
+
+	if resp.StatusCode != 200 {
+		log.Fatalf("Unexpected server reponse: %s", resp.Status)
+		return nil, errors.New(resp.Status)
+	}
+
+	defer resp.Body.Close()
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		log.Fatalf("Cannot read: %s", err)
+		return nil, err
+	}
+
+	return body, nil
+}
+
+func fetchObject(api string, baseUrl *url.URL, token string, out interface{}) error {
+	body, err := fetchURL(api, baseUrl, token)
+	if err != nil {
+		return err
+	}
+	err = json.Unmarshal(body, out)
+	if err != nil {
+		log.Fatalf("Cannot decode: %s", err)
+		return err
+	}
+	return nil
+}
+
+func fetchObjects(api string, baseUrl *url.URL, token string, out interface{}) error {
+	outSliceValue := reflect.ValueOf(out).Elem()
+	slicePtrValue := reflect.New(outSliceValue.Type())
+
+	if strings.Contains(api, "?") {
+		api += "&"
+	} else {
+		api += "?"
+	}
+
+	for page := 1; ; page++ {
+		body, err := fetchURL(fmt.Sprintf("%sper_page=100&page=%d", api, page), baseUrl, token)
+		if err != nil {
+			return err
+		}
+		err = json.Unmarshal(body, slicePtrValue.Interface())
+		if err != nil {
+			log.Fatalf("Cannot decode: %s", err)
+			return err
+		}
+		outSliceValue.Set(reflect.AppendSlice(outSliceValue, slicePtrValue.Elem()))
+		if slicePtrValue.Elem().Len() < 100 {
+			break
+		}
+	}
+
+	return nil
+}
+
+func getOrgTeam(org *models.User, cached **models.Team, name string, access models.AccessMode) (*models.Team, error) {
+	if *cached != nil {
+		return *cached, nil
+	}
+	if team, err := org.GetTeam(name); err != nil && err != models.ErrTeamNotExist {
+		log.Fatalf("Cannot get %s team for org: %s, %s", name, org.Name, err)
+		return nil, err
+	} else if team != nil {
+		*cached = team
+	} else {
+		team := &models.Team{
+			OrgID:     org.Id,
+			Name:      name,
+			Authorize: access,
+		}
+		if err := models.NewTeam(team); err != nil {
+			log.Fatalf("Cannot create %s team for org: %s, %s", name, org.Name, err)
+			return nil, err
+		}
+		*cached = team
+	}
+	return *cached, nil
+}