playground
/
gogs
mirror of https://github.com/gogits/gogs.git
1
0
Fork 0
Code Issues Releases Activity
Browse Source

WIP import gitlab command

pull/2067/head
Adam Strzelecki 9 years ago
parent
927a0c1406
commit
f55b3392b7
9 changed files with 505 additions and 1 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 65
      cmd/import.go
  2. 2
      conf/app.ini
  3. 1
      gogs.go
  4. 4
      models/org.go
  5. 11
      models/publickey.go
  6. 7
      models/repo.go
  7. 3
      models/user.go
  8. 303
      modules/importer/gitlab.go
  9. 110
      modules/importer/utils.go

65
cmd/import.go
Unescape View File

@ -0,0 +1,65 @@
// Copyright 2015 The Gogs Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package cmd
import (
"log"
"net/url"
"github.com/codegangsta/cli"
"github.com/gogits/gogs/models"
"github.com/gogits/gogs/modules/importer"
"github.com/gogits/gogs/modules/setting"
)
var CmdImport = cli.Command{
Name: "import",
Usage: "Import foreign project tracking service",
Description: `Import connects to foreign project tracking service API and imports all common entities such as users, projects, issues into Gogs.
This can be used to migrate from other service to Gogs.`,
Subcommands: []cli.Command{
{
Name: "gitlab",
Usage: "import from GitLab",
Action: importGitLab,
Flags: []cli.Flag{
stringFlag("url", "", "GitLab service base URL"),
stringFlag("token", "", "private authentication token"),
},
},
},
Flags: []cli.Flag{
stringFlag("config, c", "custom/conf/app.ini", "Custom configuration file path"),
boolFlag("verbose, v", "show process details"),
},
}
func importGitLab(ctx *cli.Context) {
baseUrl, err := url.Parse(ctx.String("url"))
if err != nil {
log.Fatal("Required --url parameter is missing or not valid URL")
}
if len(baseUrl.Path) > 0 {
log.Fatal("Provided --url parameter must not contain path")
}
token := ctx.String("token")
if len(token) == 0 {
log.Fatal("Missing required --token parameter")
}
if ctx.IsSet("config") {
setting.CustomConf = ctx.String("config")
}
setting.NewContext()
models.LoadConfigs()
models.SetEngine()
if importer.ImportGitLab(baseUrl, token) == nil {
log.Println("Finished importing!")
} else {
log.Println("Import failed!")
}
}

2
conf/app.ini
Unescape View File

@ -117,6 +117,8 @@ ENABLE_REVERSE_PROXY_AUTO_REGISTRATION = false
DISABLE_MINIMUM_KEY_SIZE_CHECK = false
; Enable captcha validation for registration
ENABLE_CAPTCHA = true
; Enable bcrypt password encryption fallback
BCRYPT_AUTH_FALLBACK = false
; used to filter keys which are too short
[service.minimum_key_sizes]

1
gogs.go
Unescape View File

@ -35,6 +35,7 @@ func main() {
cmd.CmdUpdate,
cmd.CmdDump,
cmd.CmdCert,
cmd.CmdImport,
}
app.Flags = append(app.Flags, []cli.Flag{}...)
app.Run(os.Args)

4
models/org.go
Unescape View File

@ -9,6 +9,7 @@ import (
"fmt"
"os"
"strings"
"time"
"github.com/go-xorm/xorm"
)
@ -113,6 +114,9 @@ func CreateOrganization(org, owner *User) (err error) {
org.NumMembers = 1
sess := x.NewSession()
if (org.Created != time.Time{}) {
sess.NoAutoTime()
}
defer sessionRelease(sess)
if err = sess.Begin(); err != nil {
return err

11
models/publickey.go
Unescape View File

@ -303,7 +303,7 @@ func addKey(e Engine, key *PublicKey) (err error) {
}
// AddPublicKey adds new public key to database and authorized_keys file.
func AddPublicKey(ownerID int64, name, content string) (err error) {
func AddPublicKeyCreated(ownerID int64, name, content string, created time.Time) (err error) {
if err = checkKeyContent(content); err != nil {
return err
}
@ -318,6 +318,9 @@ func AddPublicKey(ownerID int64, name, content string) (err error) {
sess := x.NewSession()
defer sessionRelease(sess)
if (created != time.Time{}) {
sess.NoAutoTime()
}
if err = sess.Begin(); err != nil {
return err
}
@ -328,6 +331,7 @@ func AddPublicKey(ownerID int64, name, content string) (err error) {
Content: content,
Mode: ACCESS_MODE_WRITE,
Type: KEY_TYPE_USER,
Created: created,
}
if err = addKey(sess, key); err != nil {
return fmt.Errorf("addKey: %v", err)
@ -336,6 +340,11 @@ func AddPublicKey(ownerID int64, name, content string) (err error) {
return sess.Commit()
}
// AddPublicKey adds new public key to database and authorized_keys file.
func AddPublicKey(ownerID int64, name, content string) (err error) {
return AddPublicKeyCreated(ownerID, name, content, time.Time{})
}
// GetPublicKeyByID returns public key by given ID.
func GetPublicKeyByID(keyID int64) (*PublicKey, error) {
key := new(PublicKey)

7
models/repo.go
Unescape View File

@ -612,6 +612,7 @@ type CreateRepoOptions struct {
IsPrivate bool
IsMirror bool
AutoInit bool
Created time.Time
}
func getRepoInitFile(tp, name string) ([]byte, error) {
@ -756,9 +757,14 @@ func createRepository(e *xorm.Session, u *User, repo *Repository) (err error) {
return ErrRepoAlreadyExist{u.Name, repo.Name}
}
prevUseAutoTime := e.Statement.UseAutoTime
if (repo.Created != time.Time{}) {
e.NoAutoTime()
}
if _, err = e.Insert(repo); err != nil {
return err
}
e.Statement.UseAutoTime = prevUseAutoTime
u.NumRepos++
// Remember visibility preference.
@ -800,6 +806,7 @@ func CreateRepository(u *User, opts CreateRepoOptions) (_ *Repository, err error
LowerName: strings.ToLower(opts.Name),
Description: opts.Description,
IsPrivate: opts.IsPrivate,
Created: opts.Created,
}
sess := x.NewSession()

3
models/user.go
Unescape View File

@ -454,6 +454,9 @@ func CreateUser(u *User) (err error) {
u.EncodePasswd()
sess := x.NewSession()
if (u.Created != time.Time{}) {
sess.NoAutoTime()
}
defer sess.Close()
if err = sess.Begin(); err != nil {
return err

303
modules/importer/gitlab.go
Unescape View File

@ -0,0 +1,303 @@
// Copyright 2015 The Gogs Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package importer
import (
"errors"
"fmt"
"log"
"net/url"
"sort"
"time"
"github.com/gogits/gogs/models"
)
func ImportGitLab(baseUrl *url.URL, token string) error {
if err := importGitLabUsers(baseUrl, token); err != nil {
return err
}
if err := importGitLabOrgs(baseUrl, token); err != nil {
return err
}
if err := importGitLabRepos(baseUrl, token); err != nil {
return err
}
return nil
}
// https://gitlab.com/gitlab-org/gitlab-ce/tree/master/doc/api
// https://gitlab.com/gitlab-org/gitlab-ce/blob/master/doc/api/users.md
type GitLabUser struct {
Id int64 `json:"id"`
Name string `json:"username"`
FullName string `json:"name"`
Email string `json:"email"`
Passwd string `json:"password_hash"` // this requires API patch (returns bcrypt hash)
State string `json:"state"` // "active", "inactive"
IsAdmin bool `json:"is_admin"`
Website string `json:"website_url"`
Created time.Time `json:"created_at"`
}
type GitLabUsers []GitLabUser
func (slice GitLabUsers) Len() int { return len(slice) }
func (slice GitLabUsers) Less(i, j int) bool { return slice[i].Id < slice[j].Id }
func (slice GitLabUsers) Swap(i, j int) { slice[i], slice[j] = slice[j], slice[i] }
type GitLabKey struct {
Id int64 `json:"id"`
Name string `json:"title"`
Key string `json:"key"`
Created time.Time `json:"created_at"`
}
type GitLabKeys []GitLabKey
func (slice GitLabKeys) Len() int { return len(slice) }
func (slice GitLabKeys) Less(i, j int) bool { return slice[i].Id < slice[j].Id }
func (slice GitLabKeys) Swap(i, j int) { slice[i], slice[j] = slice[j], slice[i] }
func importGitLabUsers(baseUrl *url.URL, token string) error {
var remoteUsers GitLabUsers
if err := fetchObjects("/api/v3/users", baseUrl, token, &remoteUsers); err != nil {
return err
}
sort.Sort(remoteUsers)
for _, remoteUser := range remoteUsers {
if user, _ := models.GetUserByName(remoteUser.Name); user == nil {
log.Printf("User: %s (%s)", remoteUser.Name, remoteUser.FullName)
user := &models.User{
Name: remoteUser.Name,
FullName: remoteUser.FullName,
Email: remoteUser.Email,
IsActive: remoteUser.State == "active",
IsAdmin: remoteUser.IsAdmin,
Website: remoteUser.Website,
Created: remoteUser.Created,
LoginType: models.PLAIN,
}
err := models.CreateUser(user)
if err != nil {
log.Fatalf("Cannot create user: %s", err)
return err
}
// extra step needed to put bcrypt hash directly
user.Passwd = remoteUser.Passwd
models.UpdateUser(user)
var remoteKeys GitLabKeys
if err := fetchObjects(fmt.Sprintf("/api/v3/users/%d/keys", remoteUser.Id), baseUrl, token, &remoteKeys); err != nil {
return err
} else {
sort.Sort(remoteKeys)
for _, remoteKey := range remoteKeys {
if err := models.AddPublicKeyCreated(user.Id, remoteKey.Name, remoteKey.Key, remoteKey.Created); err != nil {
if models.IsErrKeyNameAlreadyUsed(err) {
log.Printf("Duplicate key: %s", remoteKey.Name, remoteUser.Name, err)
} else {
log.Fatalf("Cannot add public key %s to user %s: %s", remoteKey.Name, remoteUser.Name, err)
return err
}
} else {
log.Printf(" Keys <- %s", remoteKey.Name)
}
}
}
} else {
log.Printf("User %s already exists!", remoteUser.Name)
}
}
return nil
}
// https://gitlab.com/gitlab-org/gitlab-ce/blob/master/doc/api/groups.md
type GitLabOrg struct {
Id int64 `json:"id"`
Name string `json:"path"`
FullName string `json:"name"`
Email string `json:"email"`
Description string `json:"description"`
}
type GitLabOrgs []GitLabOrg
func (slice GitLabOrgs) Len() int { return len(slice) }
func (slice GitLabOrgs) Less(i, j int) bool { return slice[i].Id < slice[j].Id }
func (slice GitLabOrgs) Swap(i, j int) { slice[i], slice[j] = slice[j], slice[i] }
type GitLabAccessLevel int
const (
GitLabNoAccess GitLabAccessLevel = iota
GitLabGuest = 10
GitLabReporter = 20
GitLabDeveloper = 30
GitLabMaster = 40
GitLabOwner = 50
)
type GitLabMember struct {
Id int64 `json:"id"`
Name string `json:"username"`
FullName string `json:"name"`
Email string `json:"email"`
AccessLevel GitLabAccessLevel `json:"access_level"`
}
type GitLabMembers []GitLabMember
func (slice GitLabMembers) Len() int { return len(slice) }
func (slice GitLabMembers) Less(i, j int) bool { return slice[i].AccessLevel > slice[j].AccessLevel }
func (slice GitLabMembers) Swap(i, j int) { slice[i], slice[j] = slice[j], slice[i] }
func importGitLabOrgs(baseUrl *url.URL, token string) error {
var remoteOrgs GitLabOrgs
if err := fetchObjects("/api/v3/groups", baseUrl, token, &remoteOrgs); err != nil {
return err
}
sort.Sort(remoteOrgs)
for _, remoteOrg := range remoteOrgs {
if org, _ := models.GetOrgByName(remoteOrg.Name); org == nil {
var remoteMembers GitLabMembers
if err := fetchObjects(fmt.Sprintf("/api/v3/groups/%d/members", remoteOrg.Id), baseUrl, token, &remoteMembers); err != nil {
return err
}
if len(remoteMembers) == 0 {
log.Fatalf("No members in org: %s! Skipping", remoteOrg.Name)
continue
}
if owner, _ := models.GetUserByName(remoteMembers[0].Name); owner == nil {
log.Fatalf("Inexistent owner %s (%s) for org: %s!", remoteMembers[0].FullName, remoteMembers[0].Name, remoteOrg.Name)
} else {
log.Printf("Org: %s (%s)", remoteOrg.Name, remoteOrg.FullName)
log.Printf(" %s (%s) -> Owners", owner.FullName, owner.Name)
org := &models.User{
Name: remoteOrg.Name,
FullName: remoteOrg.FullName,
Created: owner.Created, // GitLab does not expose creation time, use owner's
IsActive: true,
Type: models.ORGANIZATION,
}
err := models.CreateOrganization(org, owner)
if err != nil {
log.Fatalf("Cannot create org: %s", err)
return err
}
ownersTeam, err := org.GetOwnerTeam()
if err != nil {
log.Fatalf("Cannot get org owners team: %s", err)
return err
}
var adminsTeam, writersTeam, readersTeam *models.Team
for index, remoteMember := range remoteMembers {
if index == 0 {
continue
}
if member, _ := models.GetUserByName(remoteMember.Name); owner == nil {
log.Fatalf("Inexistent member %s (%s) for org: %s!", remoteMember.FullName, remoteMember.Name, remoteOrg.Name)
} else {
var team *models.Team
switch {
case remoteMember.AccessLevel >= GitLabOwner:
team = ownersTeam
err = nil
case remoteMember.AccessLevel >= GitLabMaster:
team, err = getOrgTeam(org, &adminsTeam, "Admins", models.ACCESS_MODE_ADMIN)
case remoteMember.AccessLevel >= GitLabDeveloper:
team, err = getOrgTeam(org, &writersTeam, "Writers", models.ACCESS_MODE_WRITE)
case remoteMember.AccessLevel >= GitLabGuest:
team, err = getOrgTeam(org, &readersTeam, "Readers", models.ACCESS_MODE_READ)
}
if err != nil {
return err
}
if err := team.AddMember(member.Id); err != nil {
log.Fatalf("Cannot add member %s (%s) to org: %s, %s", member.FullName, member.Name, remoteOrg.Name, err)
}
log.Printf(" %s (%s) -> %s", member.FullName, member.Name, team.Name)
}
}
}
} else {
log.Printf("Org %s already exists!", remoteOrg.Name)
}
}
return nil
}
// https://gitlab.com/gitlab-org/gitlab-ce/blob/master/doc/api/projects.md
type GitLabNamespace struct {
Id int64 `json:"id"`
Name string `json:"path"`
FullName string `json:"name"`
Description string `json:"description"`
}
type GitLabRepo struct {
Id int64 `json:"id"`
Name string `json:"path"`
Description string `json:"description"`
Created time.Time `json:"created_at"`
IsPublic bool `json:"public"`
VisibilityLevel int `json:"visibility_level"` // (0) private, (10) internal, (20) public
Owner GitLabUser `json:"owner"`
Namespace GitLabNamespace `json:"namespace"`
DefaultBranch string
}
type GitLabRepos []GitLabRepo
func importGitLabRepos(baseUrl *url.URL, token string) error {
var remoteRepos GitLabRepos
if err := fetchObjects("/api/v3/projects/all?order_by=id&sort=asc", baseUrl, token, &remoteRepos); err != nil {
return err
}
for _, remoteRepo := range remoteRepos {
if owner, _ := models.GetUserByName(remoteRepo.Namespace.Name); owner == nil {
log.Fatalf("Cannot find owner: %s for repo: %s", remoteRepo.Namespace.Name, remoteRepo.Name)
return errors.New("Cannot find repo owner")
} else {
if repo, _ := models.GetRepositoryByName(owner.Id, remoteRepo.Name); repo == nil {
log.Printf("Repo: %s (%s)", remoteRepo.Name, remoteRepo.Namespace.Name)
repo, err := models.CreateRepository(owner, models.CreateRepoOptions{
Name: remoteRepo.Name,
Description: remoteRepo.Description,
IsPrivate: remoteRepo.VisibilityLevel == 0,
Created: remoteRepo.Created,
IsMirror: false,
AutoInit: true,
Readme: "Default",
})
if err != nil {
log.Fatalf("Cannot create repo: %s", err)
return err
}
// NOTE: Gogs only supports plain list of collaborators in project, while GitLab has fine grained access control
var remoteMembers GitLabMembers
if err := fetchObjects(fmt.Sprintf("/api/v3/projects/%d/members", remoteRepo.Id), baseUrl, token, &remoteMembers); err != nil {
return err
}
for _, remoteMember := range remoteMembers {
if member, _ := models.GetUserByName(remoteMember.Name); owner == nil {
log.Fatalf("Inexistent member %s (%s) for repo: %s!", remoteMember.FullName, remoteMember.Name, remoteRepo.Name)
} else {
if member.Id == owner.Id {
continue // skip if we got owner here
}
if err := repo.AddCollaborator(member); err != nil {
log.Fatalf("Cannot add collaborator %s (%s) to repo: %s, %s", member.FullName, member.Name, remoteRepo.Name, err)
return err
}
log.Printf(" %s (%s) -> Collaborators", member.FullName, member.Name)
}
}
} else {
log.Printf("Repo %s already exists!", remoteRepo.Name)
}
}
}
return nil
}

110
modules/importer/utils.go
Unescape View File

@ -0,0 +1,110 @@
// Copyright 2015 The Gogs Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package importer
import (
"encoding/json"
"errors"
"fmt"
"io/ioutil"
"log"
"net/url"
"reflect"
"strings"
"github.com/gogits/gogs/models"
"github.com/gogits/gogs/modules/httplib"
)
func fetchURL(api string, baseUrl *url.URL, token string) ([]byte, error) {
baseUrl.Path = api
log.Printf("Fetching: %s", baseUrl)
req := httplib.Get(baseUrl.String()).Header("PRIVATE-TOKEN", token)
resp, err := req.Response()
if err != nil {
log.Fatalf("Cannot connect: %s", err)
return nil, err
}
if resp.StatusCode != 200 {
log.Fatalf("Unexpected server reponse: %s", resp.Status)
return nil, errors.New(resp.Status)
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
log.Fatalf("Cannot read: %s", err)
return nil, err
}
return body, nil
}
func fetchObject(api string, baseUrl *url.URL, token string, out interface{}) error {
body, err := fetchURL(api, baseUrl, token)
if err != nil {
return err
}
err = json.Unmarshal(body, out)
if err != nil {
log.Fatalf("Cannot decode: %s", err)
return err
}
return nil
}
func fetchObjects(api string, baseUrl *url.URL, token string, out interface{}) error {
outSliceValue := reflect.ValueOf(out).Elem()
slicePtrValue := reflect.New(outSliceValue.Type())
if strings.Contains(api, "?") {
api += "&"
} else {
api += "?"
}
for page := 1; ; page++ {
body, err := fetchURL(fmt.Sprintf("%sper_page=100&page=%d", api, page), baseUrl, token)
if err != nil {
return err
}
err = json.Unmarshal(body, slicePtrValue.Interface())
if err != nil {
log.Fatalf("Cannot decode: %s", err)
return err
}
outSliceValue.Set(reflect.AppendSlice(outSliceValue, slicePtrValue.Elem()))
if slicePtrValue.Elem().Len() < 100 {
break
}
}
return nil
}
func getOrgTeam(org *models.User, cached **models.Team, name string, access models.AccessMode) (*models.Team, error) {
if *cached != nil {
return *cached, nil
}
if team, err := org.GetTeam(name); err != nil && err != models.ErrTeamNotExist {
log.Fatalf("Cannot get %s team for org: %s, %s", name, org.Name, err)
return nil, err
} else if team != nil {
*cached = team
} else {
team := &models.Team{
OrgID: org.Id,
Name: name,
Authorize: access,
}
if err := models.NewTeam(team); err != nil {
log.Fatalf("Cannot create %s team for org: %s, %s", name, org.Name, err)
return nil, err
}
*cached = team
}
return *cached, nil
}
Write Preview
Loading…
Cancel
Save