diff --git a/README.md b/README.md
index c4068b4b5..85763bab8 100644
--- a/README.md
+++ b/README.md
@@ -7,11 +7,13 @@ gpm(Go Package Manager) is a Go package manage tool for search, install, update
## Todo
-- Command `install` add support for downloading code from code.google.com, launchpad.net, bitbucket.org; hopefully, support user sources for downloading tarballs.
+- Command `install` add support for downloading code from launchpad.net, bitbucket.org; hopefully, support user sources for downloading tarballs.
- Command `install` installs all packages after downloaded.
- After downloaded all packages in bundles or snapshots, need to check if all dependencies have been downloaded as well.
- Develop user source API server template application to support user sources in bundles.
- Add bundle and snapshot parser code for downloading by bundle or snapshot id.
- Add user system to create, edit, upload, and download bundles or snapshots through gpm client program.
- Add option for whether download dependencies packages in example code or not.
-- Add gpm working principle design.
\ No newline at end of file
+- Add gpm working principle design.
+- Download package from code.google.com only support hg as version control system, probably support git and svn.
+- All errors should have specific title for exactly where were created.
\ No newline at end of file
diff --git a/doc/github.go b/doc/github.go
index f4da49095..514baa6e7 100644
--- a/doc/github.go
+++ b/doc/github.go
@@ -28,7 +28,7 @@ func SetGithubCredentials(id, secret string) {
}
// GetGithubDoc downloads tarball from github.com.
-func GetGithubDoc(client *http.Client, match map[string]string, commit string, isDownloadEx bool) (*Package, []string, error) {
+func GetGithubDoc(client *http.Client, match map[string]string, commit string, cmdFlags map[string]bool) (*Package, []string, error) {
SetGithubCredentials("1862bcb265171f37f36c", "308d71ab53ccd858416cfceaed52d5d5b7d53c5f")
match["cred"] = githubCred
@@ -94,11 +94,10 @@ func GetGithubDoc(client *http.Client, match map[string]string, commit string, i
installPath := paths[0] + "/src/" + importPath
// Remove old files.
- os.RemoveAll(installPath)
+ os.RemoveAll(installPath + "/")
// Create destination directory.
- os.Mkdir(installPath, os.ModePerm)
+ os.MkdirAll(installPath+"/", os.ModePerm)
- //dirMap := make(map[string][]*source)
dirs := make([]string, 0, 5)
for _, f := range r.File {
absPath := strings.Replace(f.FileInfo().Name(), shaName, installPath, 1)
@@ -107,7 +106,7 @@ func GetGithubDoc(client *http.Client, match map[string]string, commit string, i
if strings.HasSuffix(absPath, "/") {
// Directory.
// Check if current directory is example.
- if !(!isDownloadEx && strings.Contains(absPath, "example")) {
+ if !(!cmdFlags["-e"] && strings.Contains(absPath, "example")) {
dirs = append(dirs, absPath)
}
continue
@@ -119,8 +118,9 @@ func GetGithubDoc(client *http.Client, match map[string]string, commit string, i
return nil, nil, err
}
- // Create diretory before create file
+ // Create diretory before create file.
os.MkdirAll(path.Dir(absPath), os.ModePerm)
+
// Write data to file
fw, _ := os.Create(absPath)
if err != nil {
diff --git a/doc/google.go b/doc/google.go
new file mode 100644
index 000000000..b98bba873
--- /dev/null
+++ b/doc/google.go
@@ -0,0 +1,272 @@
+// Copyright (c) 2013 GPMGo Members. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package doc
+
+import (
+ "errors"
+ "net/http"
+ "os"
+ "path"
+ "regexp"
+ "strings"
+
+ "github.com/GPMGo/gpm/utils"
+)
+
+var (
+ googleRepoRe = regexp.MustCompile(`id="checkoutcmd">(hg|git|svn)`)
+ googleRevisionRe = regexp.MustCompile(`(?:[^ ]+ - )?Revision *([^:]+):`)
+ googleEtagRe = regexp.MustCompile(`^(hg|git|svn)-`)
+ googleFileRe = regexp.MustCompile(`
[a-z0-9\-]+)(:?\.(?P[a-z0-9\-]+))?(?P/[a-z0-9A-Z_.\-/]+)?$`)
+)
+
+func setupGoogleMatch(match map[string]string) {
+ if s := match["subrepo"]; s != "" {
+ match["dot"] = "."
+ match["query"] = "?repo=" + s
+ } else {
+ match["dot"] = ""
+ match["query"] = ""
+ }
+}
+
+func getGoogleVCS(client *http.Client, match map[string]string) error {
+ // Scrape the HTML project page to find the VCS.
+ p, err := httpGetBytes(client, expand("http://code.google.com/p/{repo}/source/checkout", match), nil)
+ if err != nil {
+ return err
+ }
+ m := googleRepoRe.FindSubmatch(p)
+ if m == nil {
+ return NotFoundError{"Could not VCS on Google Code project page."}
+ }
+
+ match["vcs"] = string(m[1])
+ return nil
+}
+
+// GetGoogleDoc downloads raw files from code.google.com.
+func GetGoogleDoc(client *http.Client, match map[string]string, commit string, cmdFlags map[string]bool) (*Package, []string, error) {
+ setupGoogleMatch(match)
+ if m := googleEtagRe.FindStringSubmatch(commit); m != nil {
+ match["vcs"] = m[1]
+ } else if err := getGoogleVCS(client, match); err != nil {
+ return nil, nil, err
+ }
+
+ // bundle and snapshot will have commit 'B' and 'S',
+ // but does not need to download dependencies.
+ isCheckImport := len(commit) == 0
+
+ rootPath := expand("http://{subrepo}{dot}{repo}.googlecode.com/{vcs}{dir}/", match)
+
+ // Scrape the repo browser to find the project revision and individual Go files.
+ p, err := httpGetBytes(client, rootPath+"?r="+commit, nil)
+ if err != nil {
+ return nil, nil, err
+ }
+
+ // Check revision tag.
+ if m := googleRevisionRe.FindSubmatch(p); m == nil {
+ return nil, nil,
+ errors.New("doc.GetGoogleDoc(): Could not find revision for " + match["importPath"])
+ }
+
+ paths := utils.GetGOPATH()
+ importPath := "code.google.com/p/" + expand("{repo}{dir}", match)
+ installPath := paths[0] + "/src/" + importPath
+
+ // Remove old files.
+ os.RemoveAll(installPath + "/")
+ // Create destination directory.
+ os.MkdirAll(installPath+"/", os.ModePerm)
+
+ // Get source files in root path.
+ files := make([]*source, 0, 5)
+ for _, m := range googleFileRe.FindAllSubmatch(p, -1) {
+ fname := string(m[1])
+ files = append(files, &source{
+ name: fname,
+ rawURL: expand("http://{subrepo}{dot}{repo}.googlecode.com/{vcs}{dir}/{0}", match, fname) + "?r=" + commit,
+ })
+ }
+
+ // Fetch files from VCS.
+ if err := fetchFiles(client, files, nil); err != nil {
+ return nil, nil, err
+ }
+
+ // Save files.
+ for _, f := range files {
+ absPath := installPath + "/"
+
+ // Create diretory before create file.
+ os.MkdirAll(path.Dir(absPath), os.ModePerm)
+
+ // Write data to file
+ fw, err := os.Create(absPath + f.name)
+ if err != nil {
+ return nil, nil, err
+ }
+
+ _, err = fw.Write(f.data)
+ fw.Close()
+ if err != nil {
+ return nil, nil, err
+ }
+ }
+
+ dirs := make([]string, 0, 3)
+ // Get subdirectories.
+ for _, m := range googleDirRe.FindAllSubmatch(p, -1) {
+ dirName := strings.Split(string(m[1]), "?")[0]
+ if strings.HasSuffix(dirName, "/") {
+ dirs = append(dirs, dirName)
+ }
+ }
+
+ err = downloadFiles(client, match, rootPath, installPath+"/", commit, dirs)
+ if err != nil {
+ return nil, nil, err
+ }
+
+ pkg := &Package{
+ ImportPath: importPath,
+ AbsPath: installPath,
+ Commit: commit,
+ }
+ var imports []string
+
+ // Check if need to check imports.
+ if isCheckImport {
+ rootdir, err := os.Open(installPath + "/")
+ if err != nil {
+ return nil, nil, err
+ }
+ defer rootdir.Close()
+
+ dirs, err := rootdir.Readdir(0)
+ if err != nil {
+ return nil, nil, err
+ }
+
+ for _, d := range dirs {
+ if d.IsDir() {
+ absPath := installPath + "/" + d.Name() + "/"
+ dir, err := os.Open(absPath)
+ if err != nil {
+ return nil, nil, err
+ }
+ defer dir.Close()
+
+ // Get file info slice.
+ fis, err := dir.Readdir(0)
+ if err != nil {
+ return nil, nil, err
+ }
+
+ files := make([]*source, 0, 10)
+ for _, fi := range fis {
+ // Only handle files.
+ if strings.HasSuffix(fi.Name(), ".go") {
+ f, err := os.Open(absPath + fi.Name())
+ if err != nil {
+ return nil, nil, err
+ }
+
+ fbytes := make([]byte, fi.Size())
+ _, err = f.Read(fbytes)
+ f.Close()
+ //fmt.Println(d+fi.Name(), fi.Size(), n)
+ if err != nil {
+ return nil, nil, err
+ }
+
+ files = append(files, &source{
+ name: fi.Name(),
+ data: fbytes,
+ })
+ }
+ }
+
+ // Check if has Go source files.
+ if len(files) > 0 {
+ w := &walker{ImportPath: importPath}
+ importPkgs, err := w.build(files)
+ if err != nil {
+ return nil, nil, err
+ }
+ imports = append(imports, importPkgs...)
+ }
+ }
+ }
+ }
+
+ return pkg, imports, err
+}
+
+func downloadFiles(client *http.Client, match map[string]string, rootPath, installPath, commit string, dirs []string) error {
+ for _, d := range dirs {
+ p, err := httpGetBytes(client, rootPath+d+"?r="+commit, nil)
+ if err != nil {
+ return err
+ }
+
+ // Create destination directory.
+ os.MkdirAll(installPath+d, os.ModePerm)
+
+ // Get source files in current path.
+ files := make([]*source, 0, 5)
+ for _, m := range googleFileRe.FindAllSubmatch(p, -1) {
+ fname := string(m[1])
+ files = append(files, &source{
+ name: fname,
+ rawURL: expand("http://{subrepo}{dot}{repo}.googlecode.com/{vcs}{dir}/", match) + d + fname + "?r=" + commit,
+ })
+ }
+
+ // Fetch files from VCS.
+ if err := fetchFiles(client, files, nil); err != nil {
+ return err
+ }
+
+ // Save files.
+ for _, f := range files {
+ absPath := installPath + d
+
+ // Create diretory before create file.
+ os.MkdirAll(path.Dir(absPath), os.ModePerm)
+
+ // Write data to file
+ fw, err := os.Create(absPath + f.name)
+ if err != nil {
+ return err
+ }
+
+ _, err = fw.Write(f.data)
+ fw.Close()
+ if err != nil {
+ return err
+ }
+ }
+
+ subdirs := make([]string, 0, 3)
+ // Get subdirectories.
+ for _, m := range googleDirRe.FindAllSubmatch(p, -1) {
+ dirName := strings.Split(string(m[1]), "?")[0]
+ if strings.HasSuffix(dirName, "/") {
+ subdirs = append(subdirs, d+dirName)
+ }
+ }
+
+ err = downloadFiles(client, match, rootPath, installPath, commit, subdirs)
+ if err != nil {
+ return err
+ }
+ }
+ return nil
+}
diff --git a/doc/walker.go b/doc/walker.go
index b9f3a247a..b56152e7f 100644
--- a/doc/walker.go
+++ b/doc/walker.go
@@ -136,7 +136,7 @@ func (w *walker) build(srcs []*source) ([]string, error) {
var imports []string
for _, v := range bpkg.Imports {
// Skip strandard library.
- if !utils.IsGoRepoPath(v) && v != w.ImportPath {
+ if !utils.IsGoRepoPath(v) && !strings.HasPrefix(v, w.ImportPath) {
imports = append(imports, v)
}
}
diff --git a/install.go b/install.go
index afff74734..110d252a1 100644
--- a/install.go
+++ b/install.go
@@ -106,7 +106,7 @@ func runInstall(cmd *Command, args []string) {
// Download packages.
commits := make([]string, len(args))
- downloadPackages(args, commits, cmdInstall.Flags["-e"])
+ downloadPackages(args, commits)
if !cmdInstall.Flags["d"] && cmdInstall.Flags["-p"] {
// Install packages all together.
@@ -119,7 +119,7 @@ func runInstall(cmd *Command, args []string) {
// downloadPackages downloads packages with certain commit,
// if the commit is empty string, then it downloads all dependencies,
// otherwise, it only downloada package with specific commit only.
-func downloadPackages(pkgs, commits []string, isDownloadEx bool) {
+func downloadPackages(pkgs, commits []string) {
// Check all packages, they may be bundles, snapshots or raw packages path.
for i, p := range pkgs {
// Check if it is a bundle or snapshot.
@@ -131,11 +131,11 @@ func downloadPackages(pkgs, commits []string, isDownloadEx bool) {
case utils.IsValidRemotePath(p):
if !downloadCache[p] {
// Download package.
- pkg, imports := downloadPackage(p, commits[i], isDownloadEx)
+ pkg, imports := downloadPackage(p, commits[i])
if len(imports) > 0 {
// Need to download dependencies.
tags := make([]string, len(imports))
- downloadPackages(imports, tags, isDownloadEx)
+ downloadPackages(imports, tags)
continue
}
@@ -155,7 +155,7 @@ func downloadPackages(pkgs, commits []string, isDownloadEx bool) {
}
// downloadPackage download package either use version control tools or not.
-func downloadPackage(path, commit string, isDownloadEx bool) (pkg *doc.Package, imports []string) {
+func downloadPackage(path, commit string) (pkg *doc.Package, imports []string) {
// Check if use version control tools.
switch {
case !cmdInstall.Flags["-p"] &&
@@ -176,7 +176,7 @@ func downloadPackage(path, commit string, isDownloadEx bool) (pkg *doc.Package,
downloadCache[path] = true
var err error
- pkg, imports, err = pureDownload(path, commit, isDownloadEx)
+ pkg, imports, err = pureDownload(path, commit)
if err != nil {
fmt.Printf("Fail to download package(%s) with error: %s.\n", path, err)
return nil, nil
@@ -204,19 +204,19 @@ func checkGoGetFlags() (args []string) {
type service struct {
pattern *regexp.Regexp
prefix string
- get func(*http.Client, map[string]string, string, bool) (*doc.Package, []string, error)
+ get func(*http.Client, map[string]string, string, map[string]bool) (*doc.Package, []string, error)
}
// services is the list of source code control services handled by gopkgdoc.
var services = []*service{
{doc.GithubPattern, "github.com/", doc.GetGithubDoc},
- //{googlePattern, "code.google.com/", getGoogleDoc},
+ {doc.GooglePattern, "code.google.com/", doc.GetGoogleDoc},
//{bitbucketPattern, "bitbucket.org/", getBitbucketDoc},
//{launchpadPattern, "launchpad.net/", getLaunchpadDoc},
}
// pureDownload downloads package without version control.
-func pureDownload(path, commit string, isDownloadEx bool) (pinfo *doc.Package, imports []string, err error) {
+func pureDownload(path, commit string) (pinfo *doc.Package, imports []string, err error) {
for _, s := range services {
if s.get == nil || !strings.HasPrefix(path, s.prefix) {
continue
@@ -235,7 +235,7 @@ func pureDownload(path, commit string, isDownloadEx bool) (pinfo *doc.Package, i
match[n] = m[i]
}
}
- return s.get(doc.HttpClient, match, commit, isDownloadEx)
+ return s.get(doc.HttpClient, match, commit, cmdInstall.Flags)
}
return nil, nil, doc.ErrNoMatch
}