From ffa58da48c9dc521dd92e169ac7ad890543ce909 Mon Sep 17 00:00:00 2001 From: Unknown Date: Tue, 21 May 2013 14:29:58 -0400 Subject: [PATCH] command install add support for bitbucket.org --- README.md | 3 +- doc/bitbucket.go | 194 +++++++++++++++++++++++++++++++++-------------- doc/github.go | 52 +------------ doc/google.go | 51 +------------ doc/vcs.go | 51 +++++++++++++ install.go | 12 +-- utils/utils.go | 13 +++- 7 files changed, 217 insertions(+), 159 deletions(-) diff --git a/README.md b/README.md index 27b2dcdfe..14de26766 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,6 @@ gpm(Go Package Manager) is a Go package manage tool for search, install, update - All errors should have specific title for exactly where were created. - Add i18n support for all strings. - Add feature for downloading through version control tools, and use `checkout` to switch to specific revision; this feature only be enabled when users use bundle or snapshot id. -- When choose which `GOPATH` to install, match with current path first, if it doesn't match any `GOPATH`, then install to the first path in the `GOPATH` variable. -- Add support for downloading by tag for packages in github.com. +- Add support for downloading by tag for packages in github.com, bitbucket.org. - Get author commit time and save in node. - Save node information after downloaded, and check for next time, reduce download times. \ No newline at end of file diff --git a/doc/bitbucket.go b/doc/bitbucket.go index b91fc718f..d74664660 100644 --- a/doc/bitbucket.go +++ b/doc/bitbucket.go @@ -16,99 +16,181 @@ package doc import ( + "archive/tar" + "bytes" + "compress/gzip" + "io" "net/http" + "os" "path" "regexp" - - "github.com/Unknwon/gowalker/utils" + "strings" ) var ( - bitbucketPattern = regexp.MustCompile(`^bitbucket\.org/(?P[a-z0-9A-Z_.\-]+)/(?P[a-z0-9A-Z_.\-]+)(?P/[a-z0-9A-Z_.\-/]*)?$`) + BitbucketPattern = regexp.MustCompile(`^bitbucket\.org/(?P[a-z0-9A-Z_.\-]+)/(?P[a-z0-9A-Z_.\-]+)(?P/[a-z0-9A-Z_.\-/]*)?$`) bitbucketEtagRe = regexp.MustCompile(`^(hg|git)-`) ) -func getBitbucketDoc(client *http.Client, match map[string]string, savedEtag string) (*Package, error) { - - if m := bitbucketEtagRe.FindStringSubmatch(savedEtag); m != nil { +// GetBitbucketDoc downloads tarball from bitbucket.org. +func GetBitbucketDoc(client *http.Client, match map[string]string, installGOPATH, commit string, cmdFlags map[string]bool) (*Package, []string, error) { + // Check version control. + if m := bitbucketEtagRe.FindStringSubmatch(commit); m != nil { match["vcs"] = m[1] } else { var repo struct { Scm string } if err := httpGetJSON(client, expand("https://api.bitbucket.org/1.0/repositories/{owner}/{repo}", match), &repo); err != nil { - return nil, err + return nil, nil, err } match["vcs"] = repo.Scm } - tags := make(map[string]string) - for _, nodeType := range []string{"branches", "tags"} { - var nodes map[string]struct { - Node string - } - if err := httpGetJSON(client, expand("https://api.bitbucket.org/1.0/repositories/{owner}/{repo}/{0}", match, nodeType), &nodes); err != nil { - return nil, err + // bundle and snapshot will have commit 'B' and 'S', + // but does not need to download dependencies. + isCheckImport := len(commit) == 0 + + // Check if download with specific revision. + if isCheckImport || len(commit) == 1 { + tags := make(map[string]string) + for _, nodeType := range []string{"branches", "tags"} { + var nodes map[string]struct { + Node string + } + if err := httpGetJSON(client, expand("https://api.bitbucket.org/1.0/repositories/{owner}/{repo}/{0}", match, nodeType), &nodes); err != nil { + return nil, nil, err + } + for t, n := range nodes { + tags[t] = n.Node + } } - for t, n := range nodes { - tags[t] = n.Node + + // Check revision tag. + var err error + match["tag"], match["commit"], err = bestTag(tags, defaultTags[match["vcs"]]) + if err != nil { + return nil, nil, err } + } else { + match["commit"] = commit } - var err error - match["tag"], match["commit"], err = bestTag(tags, defaultTags[match["vcs"]]) + // We use .tar.gz here. + // zip : https://bitbucket.org/{owner}/{repo}/get/{commit}.zip + // tarball : https://bitbucket.org/{owner}/{repo}/get/{commit}.tar.gz + + // Downlaod archive. + p, err := httpGetBytes(client, expand("https://bitbucket.org/{owner}/{repo}/get/{commit}.tar.gz", match), nil) if err != nil { - return nil, err + return nil, nil, err } - // Check revision tag. - etag := expand("{vcs}-{commit}", match) - if etag == savedEtag { - return nil, errNotModified - } + importPath := "bitbucket.org/" + expand("{owner}/{repo}", match) + installPath := installGOPATH + "/src/" + importPath - var node struct { - Files []struct { - Path string - } - Directories []string - } + // Remove old files. + os.RemoveAll(installPath + "/") + // Create destination directory. + os.MkdirAll(installPath+"/", os.ModePerm) - if err := httpGetJSON(client, expand("https://api.bitbucket.org/1.0/repositories/{owner}/{repo}/src/{tag}{dir}/", match), &node); err != nil { - return nil, err + gzr, err := gzip.NewReader(bytes.NewReader(p)) + if err != nil { + return nil, nil, err } + defer gzr.Close() + tr := tar.NewReader(gzr) + + var autoPath string // Auto path is the root path that generated by bitbucket.org. // Get source file data. - files := make([]*source, 0, 5) - for _, f := range node.Files { - _, name := path.Split(f.Path) - if utils.IsDocFile(name) { - files = append(files, &source{ - name: name, - browseURL: expand("https://bitbucket.org/{owner}/{repo}/src/{tag}/{0}", match, f.Path), - rawURL: expand("https://api.bitbucket.org/1.0/repositories/{owner}/{repo}/raw/{tag}/{0}", match, f.Path), - }) + dirs := make([]string, 0, 5) + for { + h, err := tr.Next() + if err == io.EOF { + break + } else if err != nil { + return nil, nil, err + } + + fn := h.FileInfo().Name() + + // In case that we find directory, usually we should not. + if !strings.HasSuffix(fn, "/") { + // Check root path. + if len(autoPath) == 0 { + autoPath = fn[:strings.Index(fn, "/")] + } + absPath := strings.Replace(fn, autoPath, installPath, 1) + + // Create diretory before create file. + dir := path.Dir(absPath) + if !checkDir(dir, dirs) { + dirs = append(dirs, dir) + os.MkdirAll(dir+"/", os.ModePerm) + } + + // Get data from archive. + fbytes := make([]byte, h.Size) + if _, err := io.ReadFull(tr, fbytes); err != nil { + return nil, nil, err + } + + // Write data to file + fw, err := os.Create(absPath) + if err != nil { + return nil, nil, err + } + + _, err = fw.Write(fbytes) + fw.Close() + if err != nil { + return nil, nil, err + } } } - if len(files) == 0 && len(node.Directories) == 0 { - return nil, NotFoundError{"Directory tree does not contain Go files and subdirs."} + pkg := &Package{ + ImportPath: importPath, + AbsPath: installPath, + Commit: commit, } - // Fetch file from VCS. - if err := fetchFiles(client, files, nil); err != nil { - return nil, err + var imports []string + + // Check if need to check imports. + if isCheckImport { + rootdir, err := os.Open(installPath + "/") + if err != nil { + return nil, nil, err + } + defer rootdir.Close() + + dirs, err := rootdir.Readdir(0) + if err != nil { + return nil, nil, err + } + + for _, d := range dirs { + if d.IsDir() { + absPath := installPath + "/" + d.Name() + "/" + imports, err = checkImports(absPath, importPath) + if err != nil { + return nil, nil, err + } + } + } } - // Start generating data. - w := &walker{ - lineFmt: "#cl-%d", - pdoc: &Package{ - ImportPath: match["importPath"], - ProjectName: match["repo"], - Etag: etag, - Dirs: node.Directories, - }, + return pkg, imports, err +} + +// checkDir checks if current directory has been saved. +func checkDir(dir string, dirs []string) bool { + for _, d := range dirs { + if dir == d { + return true + } } - return w.build(files) + return false } diff --git a/doc/github.go b/doc/github.go index 514baa6e7..66da664a3 100644 --- a/doc/github.go +++ b/doc/github.go @@ -13,8 +13,6 @@ import ( "path" "regexp" "strings" - - "github.com/GPMGo/gpm/utils" ) var ( @@ -28,7 +26,7 @@ func SetGithubCredentials(id, secret string) { } // GetGithubDoc downloads tarball from github.com. -func GetGithubDoc(client *http.Client, match map[string]string, commit string, cmdFlags map[string]bool) (*Package, []string, error) { +func GetGithubDoc(client *http.Client, match map[string]string, installGOPATH, commit string, cmdFlags map[string]bool) (*Package, []string, error) { SetGithubCredentials("1862bcb265171f37f36c", "308d71ab53ccd858416cfceaed52d5d5b7d53c5f") match["cred"] = githubCred @@ -89,9 +87,8 @@ func GetGithubDoc(client *http.Client, match map[string]string, commit string, c } shaName := expand("{repo}-{sha}", match) - paths := utils.GetGOPATH() importPath := "github.com/" + expand("{owner}/{repo}", match) - installPath := paths[0] + "/src/" + importPath + installPath := installGOPATH + "/src/" + importPath // Remove old files. os.RemoveAll(installPath + "/") @@ -112,7 +109,7 @@ func GetGithubDoc(client *http.Client, match map[string]string, commit string, c continue } - // Get files from archive. + // Get file from archive. rc, err := f.Open() if err != nil { return nil, nil, err @@ -146,51 +143,10 @@ func GetGithubDoc(client *http.Client, match map[string]string, commit string, c // Check if need to check imports. if isCheckImport { for _, d := range dirs { - dir, err := os.Open(d) - if err != nil { - return nil, nil, err - } - defer dir.Close() - - // Get file info slice. - fis, err := dir.Readdir(0) + imports, err = checkImports(d, importPath) if err != nil { return nil, nil, err } - - files := make([]*source, 0, 10) - for _, fi := range fis { - // Only handle files. - if strings.HasSuffix(fi.Name(), ".go") { - f, err := os.Open(d + fi.Name()) - if err != nil { - return nil, nil, err - } - - fbytes := make([]byte, fi.Size()) - _, err = f.Read(fbytes) - f.Close() - //fmt.Println(d+fi.Name(), fi.Size(), n) - if err != nil { - return nil, nil, err - } - - files = append(files, &source{ - name: fi.Name(), - data: fbytes, - }) - } - } - - // Check if has Go source files. - if len(files) > 0 { - w := &walker{ImportPath: importPath} - importPkgs, err := w.build(files) - if err != nil { - return nil, nil, err - } - imports = append(imports, importPkgs...) - } } } diff --git a/doc/google.go b/doc/google.go index b98bba873..825124953 100644 --- a/doc/google.go +++ b/doc/google.go @@ -11,8 +11,6 @@ import ( "path" "regexp" "strings" - - "github.com/GPMGo/gpm/utils" ) var ( @@ -50,8 +48,9 @@ func getGoogleVCS(client *http.Client, match map[string]string) error { } // GetGoogleDoc downloads raw files from code.google.com. -func GetGoogleDoc(client *http.Client, match map[string]string, commit string, cmdFlags map[string]bool) (*Package, []string, error) { +func GetGoogleDoc(client *http.Client, match map[string]string, installGOPATH, commit string, cmdFlags map[string]bool) (*Package, []string, error) { setupGoogleMatch(match) + // Check version control. if m := googleEtagRe.FindStringSubmatch(commit); m != nil { match["vcs"] = m[1] } else if err := getGoogleVCS(client, match); err != nil { @@ -76,9 +75,8 @@ func GetGoogleDoc(client *http.Client, match map[string]string, commit string, c errors.New("doc.GetGoogleDoc(): Could not find revision for " + match["importPath"]) } - paths := utils.GetGOPATH() importPath := "code.google.com/p/" + expand("{repo}{dir}", match) - installPath := paths[0] + "/src/" + importPath + installPath := installGOPATH + "/src/" + importPath // Remove old files. os.RemoveAll(installPath + "/") @@ -157,51 +155,10 @@ func GetGoogleDoc(client *http.Client, match map[string]string, commit string, c for _, d := range dirs { if d.IsDir() { absPath := installPath + "/" + d.Name() + "/" - dir, err := os.Open(absPath) - if err != nil { - return nil, nil, err - } - defer dir.Close() - - // Get file info slice. - fis, err := dir.Readdir(0) + imports, err = checkImports(absPath, importPath) if err != nil { return nil, nil, err } - - files := make([]*source, 0, 10) - for _, fi := range fis { - // Only handle files. - if strings.HasSuffix(fi.Name(), ".go") { - f, err := os.Open(absPath + fi.Name()) - if err != nil { - return nil, nil, err - } - - fbytes := make([]byte, fi.Size()) - _, err = f.Read(fbytes) - f.Close() - //fmt.Println(d+fi.Name(), fi.Size(), n) - if err != nil { - return nil, nil, err - } - - files = append(files, &source{ - name: fi.Name(), - data: fbytes, - }) - } - } - - // Check if has Go source files. - if len(files) > 0 { - w := &walker{ImportPath: importPath} - importPkgs, err := w.build(files) - if err != nil { - return nil, nil, err - } - imports = append(imports, importPkgs...) - } } } } diff --git a/doc/vcs.go b/doc/vcs.go index 3d6541f40..bee39e9ce 100644 --- a/doc/vcs.go +++ b/doc/vcs.go @@ -176,3 +176,54 @@ func expand(template string, match map[string]string, subs ...string) string { p = append(p, template...) return string(p) } + +// checkImports checks package denpendencies. +func checkImports(absPath, importPath string) (imports []string, err error) { + dir, err := os.Open(absPath) + if err != nil { + return nil, err + } + defer dir.Close() + + // Get file info slice. + fis, err := dir.Readdir(0) + if err != nil { + return nil, err + } + + files := make([]*source, 0, 10) + for _, fi := range fis { + // Only handle files. + if strings.HasSuffix(fi.Name(), ".go") { + f, err := os.Open(absPath + fi.Name()) + if err != nil { + return nil, err + } + + fbytes := make([]byte, fi.Size()) + _, err = f.Read(fbytes) + f.Close() + //fmt.Println(d+fi.Name(), fi.Size(), n) + if err != nil { + return nil, err + } + + files = append(files, &source{ + name: fi.Name(), + data: fbytes, + }) + } + } + + // Check if has Go source files. + if len(files) > 0 { + w := &walker{ImportPath: importPath} + importPkgs, err := w.build(files) + if err != nil { + return nil, err + } + imports = append(imports, importPkgs...) + } + + return imports, nil +} diff --git a/install.go b/install.go index fe15078c7..c623455b7 100644 --- a/install.go +++ b/install.go @@ -18,6 +18,7 @@ import ( var ( isHasGit, isHasHg bool downloadCache map[string]bool // Saves packages that have downloaded. + installGOPATH string // The GOPATH that packages are downloaded to. ) var cmdInstall = &Command{ @@ -46,8 +47,6 @@ func printPrompt(flag string) { fmt.Printf("You enabled download without installing.\n") case "-e": fmt.Printf("You enabled download dependencies in example.\n") - case "-e": - fmt.Printf("You enabled download dependencies in example.\n") case "-s": fmt.Printf("You enabled download from sources.\n") } @@ -109,6 +108,9 @@ func runInstall(cmd *Command, args []string) { // Check version control tools. checkVCSTool() + installGOPATH = utils.GetBestMatchGOPATH(appPath) + fmt.Printf("Packages will be downloaded to GOPATH(%s).\n", installGOPATH) + // Download packages. commits := make([]string, len(args)) downloadPackages(args, commits) @@ -209,14 +211,14 @@ func checkGoGetFlags() (args []string) { type service struct { pattern *regexp.Regexp prefix string - get func(*http.Client, map[string]string, string, map[string]bool) (*doc.Package, []string, error) + get func(*http.Client, map[string]string, string, string, map[string]bool) (*doc.Package, []string, error) } // services is the list of source code control services handled by gopkgdoc. var services = []*service{ {doc.GithubPattern, "github.com/", doc.GetGithubDoc}, {doc.GooglePattern, "code.google.com/", doc.GetGoogleDoc}, - //{bitbucketPattern, "bitbucket.org/", getBitbucketDoc}, + {doc.BitbucketPattern, "bitbucket.org/", doc.GetBitbucketDoc}, //{launchpadPattern, "launchpad.net/", getLaunchpadDoc}, } @@ -240,7 +242,7 @@ func pureDownload(path, commit string) (pinfo *doc.Package, imports []string, er match[n] = m[i] } } - return s.get(doc.HttpClient, match, commit, cmdInstall.Flags) + return s.get(doc.HttpClient, match, installGOPATH, commit, cmdInstall.Flags) } return nil, nil, doc.ErrNoMatch } diff --git a/utils/utils.go b/utils/utils.go index a51c4e612..34f24dcc2 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -369,7 +369,7 @@ func IsValidRemotePath(importPath string) bool { return true } -// GetGOPATH return all GOPATH in system. +// GetGOPATH returns all paths in GOPATH variable. func GetGOPATH() []string { gopath := os.Getenv("GOPATH") var paths []string @@ -382,6 +382,17 @@ func GetGOPATH() []string { return paths } +// GetGOPATH returns best matched GOPATH. +func GetBestMatchGOPATH(appPath string) string { + paths := GetGOPATH() + for _, p := range paths { + if strings.HasPrefix(p, appPath) { + return p + } + } + return paths[0] +} + var standardPath = map[string]bool{ "builtin": true,