From f55b3392b721f21d18713f16858b93d27e0fb261 Mon Sep 17 00:00:00 2001 From: Adam Strzelecki Date: Fri, 27 Nov 2015 17:32:56 +0100 Subject: [PATCH] WIP import gitlab command --- cmd/import.go | 65 ++++++++ conf/app.ini | 2 + gogs.go | 1 + models/org.go | 4 + models/publickey.go | 11 +- models/repo.go | 7 + models/user.go | 3 + modules/importer/gitlab.go | 303 +++++++++++++++++++++++++++++++++++++ modules/importer/utils.go | 110 ++++++++++++++ 9 files changed, 505 insertions(+), 1 deletion(-) create mode 100644 cmd/import.go create mode 100644 modules/importer/gitlab.go create mode 100644 modules/importer/utils.go diff --git a/cmd/import.go b/cmd/import.go new file mode 100644 index 000000000..31506a6ef --- /dev/null +++ b/cmd/import.go @@ -0,0 +1,65 @@ +// Copyright 2015 The Gogs Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package cmd + +import ( + "log" + "net/url" + + "github.com/codegangsta/cli" + + "github.com/gogits/gogs/models" + "github.com/gogits/gogs/modules/importer" + "github.com/gogits/gogs/modules/setting" +) + +var CmdImport = cli.Command{ + Name: "import", + Usage: "Import foreign project tracking service", + Description: `Import connects to foreign project tracking service API and imports all common entities such as users, projects, issues into Gogs. +This can be used to migrate from other service to Gogs.`, + Subcommands: []cli.Command{ + { + Name: "gitlab", + Usage: "import from GitLab", + Action: importGitLab, + Flags: []cli.Flag{ + stringFlag("url", "", "GitLab service base URL"), + stringFlag("token", "", "private authentication token"), + }, + }, + }, + Flags: []cli.Flag{ + stringFlag("config, c", "custom/conf/app.ini", "Custom configuration file path"), + boolFlag("verbose, v", "show process details"), + }, +} + +func importGitLab(ctx *cli.Context) { + baseUrl, err := url.Parse(ctx.String("url")) + if err != nil { + log.Fatal("Required --url parameter is missing or not valid URL") + } + if len(baseUrl.Path) > 0 { + log.Fatal("Provided --url parameter must not contain path") + } + token := ctx.String("token") + if len(token) == 0 { + log.Fatal("Missing required --token parameter") + } + + if ctx.IsSet("config") { + setting.CustomConf = ctx.String("config") + } + setting.NewContext() + models.LoadConfigs() + models.SetEngine() + + if importer.ImportGitLab(baseUrl, token) == nil { + log.Println("Finished importing!") + } else { + log.Println("Import failed!") + } +} diff --git a/conf/app.ini b/conf/app.ini index d9d29c9ef..c30949e40 100644 --- a/conf/app.ini +++ b/conf/app.ini @@ -117,6 +117,8 @@ ENABLE_REVERSE_PROXY_AUTO_REGISTRATION = false DISABLE_MINIMUM_KEY_SIZE_CHECK = false ; Enable captcha validation for registration ENABLE_CAPTCHA = true +; Enable bcrypt password encryption fallback +BCRYPT_AUTH_FALLBACK = false ; used to filter keys which are too short [service.minimum_key_sizes] diff --git a/gogs.go b/gogs.go index 253700b80..6f94142fa 100644 --- a/gogs.go +++ b/gogs.go @@ -35,6 +35,7 @@ func main() { cmd.CmdUpdate, cmd.CmdDump, cmd.CmdCert, + cmd.CmdImport, } app.Flags = append(app.Flags, []cli.Flag{}...) app.Run(os.Args) diff --git a/models/org.go b/models/org.go index 5c3b0d12a..6033f2d6f 100644 --- a/models/org.go +++ b/models/org.go @@ -9,6 +9,7 @@ import ( "fmt" "os" "strings" + "time" "github.com/go-xorm/xorm" ) @@ -113,6 +114,9 @@ func CreateOrganization(org, owner *User) (err error) { org.NumMembers = 1 sess := x.NewSession() + if (org.Created != time.Time{}) { + sess.NoAutoTime() + } defer sessionRelease(sess) if err = sess.Begin(); err != nil { return err diff --git a/models/publickey.go b/models/publickey.go index b5646a55b..75b09fac0 100644 --- a/models/publickey.go +++ b/models/publickey.go @@ -303,7 +303,7 @@ func addKey(e Engine, key *PublicKey) (err error) { } // AddPublicKey adds new public key to database and authorized_keys file. -func AddPublicKey(ownerID int64, name, content string) (err error) { +func AddPublicKeyCreated(ownerID int64, name, content string, created time.Time) (err error) { if err = checkKeyContent(content); err != nil { return err } @@ -318,6 +318,9 @@ func AddPublicKey(ownerID int64, name, content string) (err error) { sess := x.NewSession() defer sessionRelease(sess) + if (created != time.Time{}) { + sess.NoAutoTime() + } if err = sess.Begin(); err != nil { return err } @@ -328,6 +331,7 @@ func AddPublicKey(ownerID int64, name, content string) (err error) { Content: content, Mode: ACCESS_MODE_WRITE, Type: KEY_TYPE_USER, + Created: created, } if err = addKey(sess, key); err != nil { return fmt.Errorf("addKey: %v", err) @@ -336,6 +340,11 @@ func AddPublicKey(ownerID int64, name, content string) (err error) { return sess.Commit() } +// AddPublicKey adds new public key to database and authorized_keys file. +func AddPublicKey(ownerID int64, name, content string) (err error) { + return AddPublicKeyCreated(ownerID, name, content, time.Time{}) +} + // GetPublicKeyByID returns public key by given ID. func GetPublicKeyByID(keyID int64) (*PublicKey, error) { key := new(PublicKey) diff --git a/models/repo.go b/models/repo.go index b628e7521..dff1e1ead 100644 --- a/models/repo.go +++ b/models/repo.go @@ -612,6 +612,7 @@ type CreateRepoOptions struct { IsPrivate bool IsMirror bool AutoInit bool + Created time.Time } func getRepoInitFile(tp, name string) ([]byte, error) { @@ -756,9 +757,14 @@ func createRepository(e *xorm.Session, u *User, repo *Repository) (err error) { return ErrRepoAlreadyExist{u.Name, repo.Name} } + prevUseAutoTime := e.Statement.UseAutoTime + if (repo.Created != time.Time{}) { + e.NoAutoTime() + } if _, err = e.Insert(repo); err != nil { return err } + e.Statement.UseAutoTime = prevUseAutoTime u.NumRepos++ // Remember visibility preference. @@ -800,6 +806,7 @@ func CreateRepository(u *User, opts CreateRepoOptions) (_ *Repository, err error LowerName: strings.ToLower(opts.Name), Description: opts.Description, IsPrivate: opts.IsPrivate, + Created: opts.Created, } sess := x.NewSession() diff --git a/models/user.go b/models/user.go index 3181116f6..6ecf63c6d 100644 --- a/models/user.go +++ b/models/user.go @@ -454,6 +454,9 @@ func CreateUser(u *User) (err error) { u.EncodePasswd() sess := x.NewSession() + if (u.Created != time.Time{}) { + sess.NoAutoTime() + } defer sess.Close() if err = sess.Begin(); err != nil { return err diff --git a/modules/importer/gitlab.go b/modules/importer/gitlab.go new file mode 100644 index 000000000..64d29a437 --- /dev/null +++ b/modules/importer/gitlab.go @@ -0,0 +1,303 @@ +// Copyright 2015 The Gogs Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package importer + +import ( + "errors" + "fmt" + "log" + "net/url" + "sort" + "time" + + "github.com/gogits/gogs/models" +) + +func ImportGitLab(baseUrl *url.URL, token string) error { + if err := importGitLabUsers(baseUrl, token); err != nil { + return err + } + if err := importGitLabOrgs(baseUrl, token); err != nil { + return err + } + if err := importGitLabRepos(baseUrl, token); err != nil { + return err + } + return nil +} + +// https://gitlab.com/gitlab-org/gitlab-ce/tree/master/doc/api +// https://gitlab.com/gitlab-org/gitlab-ce/blob/master/doc/api/users.md + +type GitLabUser struct { + Id int64 `json:"id"` + Name string `json:"username"` + FullName string `json:"name"` + Email string `json:"email"` + Passwd string `json:"password_hash"` // this requires API patch (returns bcrypt hash) + State string `json:"state"` // "active", "inactive" + IsAdmin bool `json:"is_admin"` + Website string `json:"website_url"` + Created time.Time `json:"created_at"` +} +type GitLabUsers []GitLabUser + +func (slice GitLabUsers) Len() int { return len(slice) } +func (slice GitLabUsers) Less(i, j int) bool { return slice[i].Id < slice[j].Id } +func (slice GitLabUsers) Swap(i, j int) { slice[i], slice[j] = slice[j], slice[i] } + +type GitLabKey struct { + Id int64 `json:"id"` + Name string `json:"title"` + Key string `json:"key"` + Created time.Time `json:"created_at"` +} +type GitLabKeys []GitLabKey + +func (slice GitLabKeys) Len() int { return len(slice) } +func (slice GitLabKeys) Less(i, j int) bool { return slice[i].Id < slice[j].Id } +func (slice GitLabKeys) Swap(i, j int) { slice[i], slice[j] = slice[j], slice[i] } + +func importGitLabUsers(baseUrl *url.URL, token string) error { + var remoteUsers GitLabUsers + if err := fetchObjects("/api/v3/users", baseUrl, token, &remoteUsers); err != nil { + return err + } + sort.Sort(remoteUsers) + + for _, remoteUser := range remoteUsers { + if user, _ := models.GetUserByName(remoteUser.Name); user == nil { + log.Printf("User: %s (%s)", remoteUser.Name, remoteUser.FullName) + user := &models.User{ + Name: remoteUser.Name, + FullName: remoteUser.FullName, + Email: remoteUser.Email, + IsActive: remoteUser.State == "active", + IsAdmin: remoteUser.IsAdmin, + Website: remoteUser.Website, + Created: remoteUser.Created, + LoginType: models.PLAIN, + } + err := models.CreateUser(user) + if err != nil { + log.Fatalf("Cannot create user: %s", err) + return err + } + // extra step needed to put bcrypt hash directly + user.Passwd = remoteUser.Passwd + models.UpdateUser(user) + var remoteKeys GitLabKeys + if err := fetchObjects(fmt.Sprintf("/api/v3/users/%d/keys", remoteUser.Id), baseUrl, token, &remoteKeys); err != nil { + return err + } else { + sort.Sort(remoteKeys) + for _, remoteKey := range remoteKeys { + if err := models.AddPublicKeyCreated(user.Id, remoteKey.Name, remoteKey.Key, remoteKey.Created); err != nil { + if models.IsErrKeyNameAlreadyUsed(err) { + log.Printf("Duplicate key: %s", remoteKey.Name, remoteUser.Name, err) + } else { + log.Fatalf("Cannot add public key %s to user %s: %s", remoteKey.Name, remoteUser.Name, err) + return err + } + } else { + log.Printf(" Keys <- %s", remoteKey.Name) + } + } + } + } else { + log.Printf("User %s already exists!", remoteUser.Name) + } + } + return nil +} + +// https://gitlab.com/gitlab-org/gitlab-ce/blob/master/doc/api/groups.md + +type GitLabOrg struct { + Id int64 `json:"id"` + Name string `json:"path"` + FullName string `json:"name"` + Email string `json:"email"` + Description string `json:"description"` +} +type GitLabOrgs []GitLabOrg + +func (slice GitLabOrgs) Len() int { return len(slice) } +func (slice GitLabOrgs) Less(i, j int) bool { return slice[i].Id < slice[j].Id } +func (slice GitLabOrgs) Swap(i, j int) { slice[i], slice[j] = slice[j], slice[i] } + +type GitLabAccessLevel int + +const ( + GitLabNoAccess GitLabAccessLevel = iota + GitLabGuest = 10 + GitLabReporter = 20 + GitLabDeveloper = 30 + GitLabMaster = 40 + GitLabOwner = 50 +) + +type GitLabMember struct { + Id int64 `json:"id"` + Name string `json:"username"` + FullName string `json:"name"` + Email string `json:"email"` + AccessLevel GitLabAccessLevel `json:"access_level"` +} +type GitLabMembers []GitLabMember + +func (slice GitLabMembers) Len() int { return len(slice) } +func (slice GitLabMembers) Less(i, j int) bool { return slice[i].AccessLevel > slice[j].AccessLevel } +func (slice GitLabMembers) Swap(i, j int) { slice[i], slice[j] = slice[j], slice[i] } + +func importGitLabOrgs(baseUrl *url.URL, token string) error { + var remoteOrgs GitLabOrgs + if err := fetchObjects("/api/v3/groups", baseUrl, token, &remoteOrgs); err != nil { + return err + } + sort.Sort(remoteOrgs) + + for _, remoteOrg := range remoteOrgs { + if org, _ := models.GetOrgByName(remoteOrg.Name); org == nil { + var remoteMembers GitLabMembers + if err := fetchObjects(fmt.Sprintf("/api/v3/groups/%d/members", remoteOrg.Id), baseUrl, token, &remoteMembers); err != nil { + return err + } + if len(remoteMembers) == 0 { + log.Fatalf("No members in org: %s! Skipping", remoteOrg.Name) + continue + } + if owner, _ := models.GetUserByName(remoteMembers[0].Name); owner == nil { + log.Fatalf("Inexistent owner %s (%s) for org: %s!", remoteMembers[0].FullName, remoteMembers[0].Name, remoteOrg.Name) + } else { + log.Printf("Org: %s (%s)", remoteOrg.Name, remoteOrg.FullName) + log.Printf(" %s (%s) -> Owners", owner.FullName, owner.Name) + org := &models.User{ + Name: remoteOrg.Name, + FullName: remoteOrg.FullName, + Created: owner.Created, // GitLab does not expose creation time, use owner's + IsActive: true, + Type: models.ORGANIZATION, + } + err := models.CreateOrganization(org, owner) + if err != nil { + log.Fatalf("Cannot create org: %s", err) + return err + } + ownersTeam, err := org.GetOwnerTeam() + if err != nil { + log.Fatalf("Cannot get org owners team: %s", err) + return err + } + var adminsTeam, writersTeam, readersTeam *models.Team + for index, remoteMember := range remoteMembers { + if index == 0 { + continue + } + if member, _ := models.GetUserByName(remoteMember.Name); owner == nil { + log.Fatalf("Inexistent member %s (%s) for org: %s!", remoteMember.FullName, remoteMember.Name, remoteOrg.Name) + } else { + var team *models.Team + switch { + case remoteMember.AccessLevel >= GitLabOwner: + team = ownersTeam + err = nil + case remoteMember.AccessLevel >= GitLabMaster: + team, err = getOrgTeam(org, &adminsTeam, "Admins", models.ACCESS_MODE_ADMIN) + case remoteMember.AccessLevel >= GitLabDeveloper: + team, err = getOrgTeam(org, &writersTeam, "Writers", models.ACCESS_MODE_WRITE) + case remoteMember.AccessLevel >= GitLabGuest: + team, err = getOrgTeam(org, &readersTeam, "Readers", models.ACCESS_MODE_READ) + } + if err != nil { + return err + } + if err := team.AddMember(member.Id); err != nil { + log.Fatalf("Cannot add member %s (%s) to org: %s, %s", member.FullName, member.Name, remoteOrg.Name, err) + } + log.Printf(" %s (%s) -> %s", member.FullName, member.Name, team.Name) + } + } + } + } else { + log.Printf("Org %s already exists!", remoteOrg.Name) + } + } + return nil +} + +// https://gitlab.com/gitlab-org/gitlab-ce/blob/master/doc/api/projects.md + +type GitLabNamespace struct { + Id int64 `json:"id"` + Name string `json:"path"` + FullName string `json:"name"` + Description string `json:"description"` +} +type GitLabRepo struct { + Id int64 `json:"id"` + Name string `json:"path"` + Description string `json:"description"` + Created time.Time `json:"created_at"` + IsPublic bool `json:"public"` + VisibilityLevel int `json:"visibility_level"` // (0) private, (10) internal, (20) public + Owner GitLabUser `json:"owner"` + Namespace GitLabNamespace `json:"namespace"` + DefaultBranch string +} +type GitLabRepos []GitLabRepo + +func importGitLabRepos(baseUrl *url.URL, token string) error { + var remoteRepos GitLabRepos + if err := fetchObjects("/api/v3/projects/all?order_by=id&sort=asc", baseUrl, token, &remoteRepos); err != nil { + return err + } + + for _, remoteRepo := range remoteRepos { + if owner, _ := models.GetUserByName(remoteRepo.Namespace.Name); owner == nil { + log.Fatalf("Cannot find owner: %s for repo: %s", remoteRepo.Namespace.Name, remoteRepo.Name) + return errors.New("Cannot find repo owner") + } else { + if repo, _ := models.GetRepositoryByName(owner.Id, remoteRepo.Name); repo == nil { + log.Printf("Repo: %s (%s)", remoteRepo.Name, remoteRepo.Namespace.Name) + repo, err := models.CreateRepository(owner, models.CreateRepoOptions{ + Name: remoteRepo.Name, + Description: remoteRepo.Description, + IsPrivate: remoteRepo.VisibilityLevel == 0, + Created: remoteRepo.Created, + IsMirror: false, + AutoInit: true, + Readme: "Default", + }) + if err != nil { + log.Fatalf("Cannot create repo: %s", err) + return err + } + // NOTE: Gogs only supports plain list of collaborators in project, while GitLab has fine grained access control + var remoteMembers GitLabMembers + if err := fetchObjects(fmt.Sprintf("/api/v3/projects/%d/members", remoteRepo.Id), baseUrl, token, &remoteMembers); err != nil { + return err + } + for _, remoteMember := range remoteMembers { + if member, _ := models.GetUserByName(remoteMember.Name); owner == nil { + log.Fatalf("Inexistent member %s (%s) for repo: %s!", remoteMember.FullName, remoteMember.Name, remoteRepo.Name) + } else { + if member.Id == owner.Id { + continue // skip if we got owner here + } + if err := repo.AddCollaborator(member); err != nil { + log.Fatalf("Cannot add collaborator %s (%s) to repo: %s, %s", member.FullName, member.Name, remoteRepo.Name, err) + return err + } + log.Printf(" %s (%s) -> Collaborators", member.FullName, member.Name) + } + } + } else { + log.Printf("Repo %s already exists!", remoteRepo.Name) + } + } + } + return nil +} diff --git a/modules/importer/utils.go b/modules/importer/utils.go new file mode 100644 index 000000000..61d9ec149 --- /dev/null +++ b/modules/importer/utils.go @@ -0,0 +1,110 @@ +// Copyright 2015 The Gogs Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package importer + +import ( + "encoding/json" + "errors" + "fmt" + "io/ioutil" + "log" + "net/url" + "reflect" + "strings" + + "github.com/gogits/gogs/models" + "github.com/gogits/gogs/modules/httplib" +) + +func fetchURL(api string, baseUrl *url.URL, token string) ([]byte, error) { + baseUrl.Path = api + log.Printf("Fetching: %s", baseUrl) + req := httplib.Get(baseUrl.String()).Header("PRIVATE-TOKEN", token) + resp, err := req.Response() + if err != nil { + log.Fatalf("Cannot connect: %s", err) + return nil, err + } + + if resp.StatusCode != 200 { + log.Fatalf("Unexpected server reponse: %s", resp.Status) + return nil, errors.New(resp.Status) + } + + defer resp.Body.Close() + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + log.Fatalf("Cannot read: %s", err) + return nil, err + } + + return body, nil +} + +func fetchObject(api string, baseUrl *url.URL, token string, out interface{}) error { + body, err := fetchURL(api, baseUrl, token) + if err != nil { + return err + } + err = json.Unmarshal(body, out) + if err != nil { + log.Fatalf("Cannot decode: %s", err) + return err + } + return nil +} + +func fetchObjects(api string, baseUrl *url.URL, token string, out interface{}) error { + outSliceValue := reflect.ValueOf(out).Elem() + slicePtrValue := reflect.New(outSliceValue.Type()) + + if strings.Contains(api, "?") { + api += "&" + } else { + api += "?" + } + + for page := 1; ; page++ { + body, err := fetchURL(fmt.Sprintf("%sper_page=100&page=%d", api, page), baseUrl, token) + if err != nil { + return err + } + err = json.Unmarshal(body, slicePtrValue.Interface()) + if err != nil { + log.Fatalf("Cannot decode: %s", err) + return err + } + outSliceValue.Set(reflect.AppendSlice(outSliceValue, slicePtrValue.Elem())) + if slicePtrValue.Elem().Len() < 100 { + break + } + } + + return nil +} + +func getOrgTeam(org *models.User, cached **models.Team, name string, access models.AccessMode) (*models.Team, error) { + if *cached != nil { + return *cached, nil + } + if team, err := org.GetTeam(name); err != nil && err != models.ErrTeamNotExist { + log.Fatalf("Cannot get %s team for org: %s, %s", name, org.Name, err) + return nil, err + } else if team != nil { + *cached = team + } else { + team := &models.Team{ + OrgID: org.Id, + Name: name, + Authorize: access, + } + if err := models.NewTeam(team); err != nil { + log.Fatalf("Cannot create %s team for org: %s, %s", name, org.Name, err) + return nil, err + } + *cached = team + } + return *cached, nil +}