command install add support for code.google.com

README.md

@@ -7,11 +7,13 @@ gpm(Go Package Manager) is a Go package manage tool for search, install, update
 
 ## Todo
 
-- Command `install` add support for downloading code from code.google.com, launchpad.net, bitbucket.org; hopefully, support user sources for downloading tarballs.
+- Command `install` add support for downloading code from launchpad.net, bitbucket.org; hopefully, support user sources for downloading tarballs.
 - Command `install` installs all packages after downloaded.
 - After downloaded all packages in bundles or snapshots, need to check if all dependencies have been downloaded as well.
 - Develop user source API server template application to support user sources in bundles.
 - Add bundle and snapshot parser code for downloading by bundle or snapshot id.
 - Add user system to create, edit, upload, and download bundles or snapshots through gpm client program.
 - Add option for whether download dependencies packages in example code or not.
-- Add gpm working principle design.
+- Add gpm working principle design.
+- Download package from code.google.com only support hg as version control system, probably support git and svn.
+- All errors should have specific title for exactly where were created.

doc/github.go

@@ -28,7 +28,7 @@ func SetGithubCredentials(id, secret string) {
 }
 
 // GetGithubDoc downloads tarball from github.com.
-func GetGithubDoc(client *http.Client, match map[string]string, commit string, isDownloadEx bool) (*Package, []string, error) {
+func GetGithubDoc(client *http.Client, match map[string]string, commit string, cmdFlags map[string]bool) (*Package, []string, error) {
 	SetGithubCredentials("1862bcb265171f37f36c", "308d71ab53ccd858416cfceaed52d5d5b7d53c5f")
 	match["cred"] = githubCred
 
@@ -94,11 +94,10 @@ func GetGithubDoc(client *http.Client, match map[string]string, commit string, i
 	installPath := paths[0] + "/src/" + importPath
 
 	// Remove old files.
-	os.RemoveAll(installPath)
+	os.RemoveAll(installPath + "/")
 	// Create destination directory.
-	os.Mkdir(installPath, os.ModePerm)
+	os.MkdirAll(installPath+"/", os.ModePerm)
 
-	//dirMap := make(map[string][]*source)
 	dirs := make([]string, 0, 5)
 	for _, f := range r.File {
 		absPath := strings.Replace(f.FileInfo().Name(), shaName, installPath, 1)
@@ -107,7 +106,7 @@ func GetGithubDoc(client *http.Client, match map[string]string, commit string, i
 		if strings.HasSuffix(absPath, "/") {
 			// Directory.
 			// Check if current directory is example.
-			if !(!isDownloadEx && strings.Contains(absPath, "example")) {
+			if !(!cmdFlags["-e"] && strings.Contains(absPath, "example")) {
 				dirs = append(dirs, absPath)
 			}
 			continue
@@ -119,8 +118,9 @@ func GetGithubDoc(client *http.Client, match map[string]string, commit string, i
 			return nil, nil, err
 		}
 
-		// Create diretory before create file
+		// Create diretory before create file.
 		os.MkdirAll(path.Dir(absPath), os.ModePerm)
+
 		// Write data to file
 		fw, _ := os.Create(absPath)
 		if err != nil {

doc/google.go

@@ -0,0 +1,272 @@
+// Copyright (c) 2013 GPMGo Members. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package doc
+
+import (
+	"errors"
+	"net/http"
+	"os"
+	"path"
+	"regexp"
+	"strings"
+
+	"github.com/GPMGo/gpm/utils"
+)
+
+var (
+	googleRepoRe     = regexp.MustCompile(`id="checkoutcmd">(hg|git|svn)`)
+	googleRevisionRe = regexp.MustCompile(`<h2>(?:[^ ]+ - )?Revision *([^:]+):`)
+	googleEtagRe     = regexp.MustCompile(`^(hg|git|svn)-`)
+	googleFileRe     = regexp.MustCompile(`<li><a href="([^"/]+)"`)
+	googleDirRe      = regexp.MustCompile(`<li><a href="([^".]+)"`)
+	GooglePattern    = regexp.MustCompile(`^code\.google\.com/p/(?P<repo>[a-z0-9\-]+)(:?\.(?P<subrepo>[a-z0-9\-]+))?(?P<dir>/[a-z0-9A-Z_.\-/]+)?$`)
+)
+
+func setupGoogleMatch(match map[string]string) {
+	if s := match["subrepo"]; s != "" {
+		match["dot"] = "."
+		match["query"] = "?repo=" + s
+	} else {
+		match["dot"] = ""
+		match["query"] = ""
+	}
+}
+
+func getGoogleVCS(client *http.Client, match map[string]string) error {
+	// Scrape the HTML project page to find the VCS.
+	p, err := httpGetBytes(client, expand("http://code.google.com/p/{repo}/source/checkout", match), nil)
+	if err != nil {
+		return err
+	}
+	m := googleRepoRe.FindSubmatch(p)
+	if m == nil {
+		return NotFoundError{"Could not VCS on Google Code project page."}
+	}
+
+	match["vcs"] = string(m[1])
+	return nil
+}
+
+// GetGoogleDoc downloads raw files from code.google.com.
+func GetGoogleDoc(client *http.Client, match map[string]string, commit string, cmdFlags map[string]bool) (*Package, []string, error) {
+	setupGoogleMatch(match)
+	if m := googleEtagRe.FindStringSubmatch(commit); m != nil {
+		match["vcs"] = m[1]
+	} else if err := getGoogleVCS(client, match); err != nil {
+		return nil, nil, err
+	}
+
+	// bundle and snapshot will have commit 'B' and 'S',
+	// but does not need to download dependencies.
+	isCheckImport := len(commit) == 0
+
+	rootPath := expand("http://{subrepo}{dot}{repo}.googlecode.com/{vcs}{dir}/", match)
+
+	// Scrape the repo browser to find the project revision and individual Go files.
+	p, err := httpGetBytes(client, rootPath+"?r="+commit, nil)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	// Check revision tag.
+	if m := googleRevisionRe.FindSubmatch(p); m == nil {
+		return nil, nil,
+			errors.New("doc.GetGoogleDoc(): Could not find revision for " + match["importPath"])
+	}
+
+	paths := utils.GetGOPATH()
+	importPath := "code.google.com/p/" + expand("{repo}{dir}", match)
+	installPath := paths[0] + "/src/" + importPath
+
+	// Remove old files.
+	os.RemoveAll(installPath + "/")
+	// Create destination directory.
+	os.MkdirAll(installPath+"/", os.ModePerm)
+
+	// Get source files in root path.
+	files := make([]*source, 0, 5)
+	for _, m := range googleFileRe.FindAllSubmatch(p, -1) {
+		fname := string(m[1])
+		files = append(files, &source{
+			name:   fname,
+			rawURL: expand("http://{subrepo}{dot}{repo}.googlecode.com/{vcs}{dir}/{0}", match, fname) + "?r=" + commit,
+		})
+	}
+
+	// Fetch files from VCS.
+	if err := fetchFiles(client, files, nil); err != nil {
+		return nil, nil, err
+	}
+
+	// Save files.
+	for _, f := range files {
+		absPath := installPath + "/"
+
+		// Create diretory before create file.
+		os.MkdirAll(path.Dir(absPath), os.ModePerm)
+
+		// Write data to file
+		fw, err := os.Create(absPath + f.name)
+		if err != nil {
+			return nil, nil, err
+		}
+
+		_, err = fw.Write(f.data)
+		fw.Close()
+		if err != nil {
+			return nil, nil, err
+		}
+	}
+
+	dirs := make([]string, 0, 3)
+	// Get subdirectories.
+	for _, m := range googleDirRe.FindAllSubmatch(p, -1) {
+		dirName := strings.Split(string(m[1]), "?")[0]
+		if strings.HasSuffix(dirName, "/") {
+			dirs = append(dirs, dirName)
+		}
+	}
+
+	err = downloadFiles(client, match, rootPath, installPath+"/", commit, dirs)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	pkg := &Package{
+		ImportPath: importPath,
+		AbsPath:    installPath,
+		Commit:     commit,
+	}
+	var imports []string
+
+	// Check if need to check imports.
+	if isCheckImport {
+		rootdir, err := os.Open(installPath + "/")
+		if err != nil {
+			return nil, nil, err
+		}
+		defer rootdir.Close()
+
+		dirs, err := rootdir.Readdir(0)
+		if err != nil {
+			return nil, nil, err
+		}
+
+		for _, d := range dirs {
+			if d.IsDir() {
+				absPath := installPath + "/" + d.Name() + "/"
+				dir, err := os.Open(absPath)
+				if err != nil {
+					return nil, nil, err
+				}
+				defer dir.Close()
+
+				// Get file info slice.
+				fis, err := dir.Readdir(0)
+				if err != nil {
+					return nil, nil, err
+				}
+
+				files := make([]*source, 0, 10)
+				for _, fi := range fis {
+					// Only handle files.
+					if strings.HasSuffix(fi.Name(), ".go") {
+						f, err := os.Open(absPath + fi.Name())
+						if err != nil {
+							return nil, nil, err
+						}
+
+						fbytes := make([]byte, fi.Size())
+						_, err = f.Read(fbytes)
+						f.Close()
+						//fmt.Println(d+fi.Name(), fi.Size(), n)
+						if err != nil {
+							return nil, nil, err
+						}
+
+						files = append(files, &source{
+							name: fi.Name(),
+							data: fbytes,
+						})
+					}
+				}
+
+				// Check if has Go source files.
+				if len(files) > 0 {
+					w := &walker{ImportPath: importPath}
+					importPkgs, err := w.build(files)
+					if err != nil {
+						return nil, nil, err
+					}
+					imports = append(imports, importPkgs...)
+				}
+			}
+		}
+	}
+
+	return pkg, imports, err
+}
+
+func downloadFiles(client *http.Client, match map[string]string, rootPath, installPath, commit string, dirs []string) error {
+	for _, d := range dirs {
+		p, err := httpGetBytes(client, rootPath+d+"?r="+commit, nil)
+		if err != nil {
+			return err
+		}
+
+		// Create destination directory.
+		os.MkdirAll(installPath+d, os.ModePerm)
+
+		// Get source files in current path.
+		files := make([]*source, 0, 5)
+		for _, m := range googleFileRe.FindAllSubmatch(p, -1) {
+			fname := string(m[1])
+			files = append(files, &source{
+				name:   fname,
+				rawURL: expand("http://{subrepo}{dot}{repo}.googlecode.com/{vcs}{dir}/", match) + d + fname + "?r=" + commit,
+			})
+		}
+
+		// Fetch files from VCS.
+		if err := fetchFiles(client, files, nil); err != nil {
+			return err
+		}
+
+		// Save files.
+		for _, f := range files {
+			absPath := installPath + d
+
+			// Create diretory before create file.
+			os.MkdirAll(path.Dir(absPath), os.ModePerm)
+
+			// Write data to file
+			fw, err := os.Create(absPath + f.name)
+			if err != nil {
+				return err
+			}
+
+			_, err = fw.Write(f.data)
+			fw.Close()
+			if err != nil {
+				return err
+			}
+		}
+
+		subdirs := make([]string, 0, 3)
+		// Get subdirectories.
+		for _, m := range googleDirRe.FindAllSubmatch(p, -1) {
+			dirName := strings.Split(string(m[1]), "?")[0]
+			if strings.HasSuffix(dirName, "/") {
+				subdirs = append(subdirs, d+dirName)
+			}
+		}
+
+		err = downloadFiles(client, match, rootPath, installPath, commit, subdirs)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}

doc/walker.go

@@ -136,7 +136,7 @@ func (w *walker) build(srcs []*source) ([]string, error) {
 	var imports []string
 	for _, v := range bpkg.Imports {
 		// Skip strandard library.
-		if !utils.IsGoRepoPath(v) && v != w.ImportPath {
+		if !utils.IsGoRepoPath(v) && !strings.HasPrefix(v, w.ImportPath) {
 			imports = append(imports, v)
 		}
 	}

install.go

@@ -106,7 +106,7 @@ func runInstall(cmd *Command, args []string) {
 
 	// Download packages.
 	commits := make([]string, len(args))
-	downloadPackages(args, commits, cmdInstall.Flags["-e"])
+	downloadPackages(args, commits)
 
 	if !cmdInstall.Flags["d"] && cmdInstall.Flags["-p"] {
 		// Install packages all together.
@@ -119,7 +119,7 @@ func runInstall(cmd *Command, args []string) {
 // downloadPackages downloads packages with certain commit,
 // if the commit is empty string, then it downloads all dependencies,
 // otherwise, it only downloada package with specific commit only.
-func downloadPackages(pkgs, commits []string, isDownloadEx bool) {
+func downloadPackages(pkgs, commits []string) {
 	// Check all packages, they may be bundles, snapshots or raw packages path.
 	for i, p := range pkgs {
 		// Check if it is a bundle or snapshot.
@@ -131,11 +131,11 @@ func downloadPackages(pkgs, commits []string, isDownloadEx bool) {
 		case utils.IsValidRemotePath(p):
 			if !downloadCache[p] {
 				// Download package.
-				pkg, imports := downloadPackage(p, commits[i], isDownloadEx)
+				pkg, imports := downloadPackage(p, commits[i])
 				if len(imports) > 0 {
 					// Need to download dependencies.
 					tags := make([]string, len(imports))
-					downloadPackages(imports, tags, isDownloadEx)
+					downloadPackages(imports, tags)
 					continue
 				}
 
@@ -155,7 +155,7 @@ func downloadPackages(pkgs, commits []string, isDownloadEx bool) {
 }
 
 // downloadPackage download package either use version control tools or not.
-func downloadPackage(path, commit string, isDownloadEx bool) (pkg *doc.Package, imports []string) {
+func downloadPackage(path, commit string) (pkg *doc.Package, imports []string) {
 	// Check if use version control tools.
 	switch {
 	case !cmdInstall.Flags["-p"] &&
@@ -176,7 +176,7 @@ func downloadPackage(path, commit string, isDownloadEx bool) (pkg *doc.Package,
 		downloadCache[path] = true
 
 		var err error
-		pkg, imports, err = pureDownload(path, commit, isDownloadEx)
+		pkg, imports, err = pureDownload(path, commit)
 		if err != nil {
 			fmt.Printf("Fail to download package(%s) with error: %s.\n", path, err)
 			return nil, nil
@@ -204,19 +204,19 @@ func checkGoGetFlags() (args []string) {
 type service struct {
 	pattern *regexp.Regexp
 	prefix  string
-	get     func(*http.Client, map[string]string, string, bool) (*doc.Package, []string, error)
+	get     func(*http.Client, map[string]string, string, map[string]bool) (*doc.Package, []string, error)
 }
 
 // services is the list of source code control services handled by gopkgdoc.
 var services = []*service{
 	{doc.GithubPattern, "github.com/", doc.GetGithubDoc},
-	//{googlePattern, "code.google.com/", getGoogleDoc},
+	{doc.GooglePattern, "code.google.com/", doc.GetGoogleDoc},
 	//{bitbucketPattern, "bitbucket.org/", getBitbucketDoc},
 	//{launchpadPattern, "launchpad.net/", getLaunchpadDoc},
 }
 
 // pureDownload downloads package without version control.
-func pureDownload(path, commit string, isDownloadEx bool) (pinfo *doc.Package, imports []string, err error) {
+func pureDownload(path, commit string) (pinfo *doc.Package, imports []string, err error) {
 	for _, s := range services {
 		if s.get == nil || !strings.HasPrefix(path, s.prefix) {
 			continue
@@ -235,7 +235,7 @@ func pureDownload(path, commit string, isDownloadEx bool) (pinfo *doc.Package, i
 				match[n] = m[i]
 			}
 		}
-		return s.get(doc.HttpClient, match, commit, isDownloadEx)
+		return s.get(doc.HttpClient, match, commit, cmdInstall.Flags)
 	}
 	return nil, nil, doc.ErrNoMatch
 }