Added download repos of google code

cmd/get.go

@@ -166,7 +166,7 @@ func downloadPackages(nodes []*doc.Node) {
 		if doc.IsValidRemotePath(n.ImportPath) {
 			if !CmdGet.Flags["-u"] {
 				// Check if package has been downloaded.
-				installPath := installRepoPath + "/" + n.ImportPath
+				installPath := installRepoPath + "/" + doc.GetProjectPath(n.ImportPath)
 				if len(n.Value) > 0 {
 					installPath += "." + n.Value
 				}
@@ -261,7 +261,7 @@ type service struct {
 // services is the list of source code control services handled by gopkgdoc.
 var services = []*service{
 	{doc.GithubPattern, "github.com/", doc.GetGithubDoc},
-	// {doc.GooglePattern, "code.google.com/", doc.GetGoogleDoc},
+	{doc.GooglePattern, "code.google.com/", doc.GetGoogleDoc},
 	// {doc.BitbucketPattern, "bitbucket.org/", doc.GetBitbucketDoc},
 	// {doc.LaunchpadPattern, "launchpad.net/", doc.GetLaunchpadDoc},
 }
@@ -290,46 +290,6 @@ func pureDownload(nod *doc.Node) ([]string, error) {
 	return nil, errors.New("Cannot match any package service by given path")
 }
 
-// func joinPath(paths ...string) string {
-// 	if len(paths) < 1 {
-// 		return ""
-// 	}
-// 	res := ""
-// 	for _, p := range paths {
-// 		res = path.Join(res, p)
-// 	}
-// 	return res
-// }
-
-// func download(url string, localfile string) error {
-// 	fmt.Println("Downloading", url, "...")
-// 	resp, err := http.Get(url)
-// 	if err != nil {
-// 		return err
-// 	}
-// 	defer resp.Body.Close()
-
-// 	localdir := filepath.Dir(localfile)
-// 	if !dirExists(localdir) {
-// 		err = os.MkdirAll(localdir, 0777)
-// 		if err != nil {
-// 			return err
-// 		}
-// 	}
-
-// 	if !fileExists(localfile) {
-// 		f, err := os.Create(localfile)
-// 		if err == nil {
-// 			_, err = io.Copy(f, resp.Body)
-// 		}
-// 		if err != nil {
-// 			return err
-// 		}
-// 	}
-
-// 	return nil
-// }
-
 // func extractPkg(pkg *Pkg, localfile string, update bool) error {
 // 	fmt.Println("Extracting package", pkg.Name, "...")
 
@@ -406,38 +366,3 @@ func pureDownload(nod *doc.Node) ([]string, error) {
 // 	}
 // 	return nil
 // }
-
-// func getPackage(pkg *Pkg, url string) error {
-// 	curUser, err := user.Current()
-// 	if err != nil {
-// 		return err
-// 	}
-
-// 	reposDir = strings.Replace(reposDir, "~", curUser.HomeDir, -1)
-// 	localdir := path.Join(reposDir, pkg.Name)
-// 	localdir, err = filepath.Abs(localdir)
-// 	if err != nil {
-// 		return err
-// 	}
-
-// 	localfile := path.Join(localdir, pkg.FileName())
-
-// 	err = download(url, localfile)
-// 	if err != nil {
-// 		return err
-// 	}
-
-// 	return extractPkg(pkg, localfile, false)
-// }
-
-// func getDirect(pkg *Pkg) error {
-// 	return getPackage(pkg, pkg.Url())
-// }
-
-/*func getFromSource(pkgName string, ver string, source string) error {
-	urlTempl := "https://%v/%v"
-	//urlTempl := "https://%v/archive/master.zip"
-	url := fmt.Sprintf(urlTempl, source, pkgName)
-
-	return getPackage(pkgName, ver, url)
-}*/

doc/github.go

@@ -97,7 +97,7 @@ func GetGithubDoc(client *http.Client, match map[string]string, installRepoPath
 	// tarball : https://github.com/{owner}/{repo}/tarball/{sha}
 
 	// Downlaod archive.
-	p, err := httpGetBytes(client, expand("https://github.com/{owner}/{repo}/archive/{sha}.zip", match), nil)
+	p, err := HttpGetBytes(client, expand("https://github.com/{owner}/{repo}/archive/{sha}.zip", match), nil)
 	if err != nil {
 		return nil, err
 	}

doc/google.go

@@ -0,0 +1,227 @@
+// Copyright 2013 gopm authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"): you may
+// not use this file except in compliance with the License. You may obtain
+// a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations
+// under the License.
+
+package doc
+
+import (
+	"net/http"
+	"os"
+	"path"
+	"regexp"
+	"strings"
+)
+
+var (
+	googleRepoRe     = regexp.MustCompile(`id="checkoutcmd">(hg|git|svn)`)
+	googleRevisionRe = regexp.MustCompile(`<h2>(?:[^ ]+ - )?Revision *([^:]+):`)
+	googleEtagRe     = regexp.MustCompile(`^(hg|git|svn)-`)
+	googleFileRe     = regexp.MustCompile(`<li><a href="([^"/]+)"`)
+	googleDirRe      = regexp.MustCompile(`<li><a href="([^".]+)"`)
+	GooglePattern    = regexp.MustCompile(`^code\.google\.com/p/(?P<repo>[a-z0-9\-]+)(:?\.(?P<subrepo>[a-z0-9\-]+))?(?P<dir>/[a-z0-9A-Z_.\-/]+)?$`)
+)
+
+func setupGoogleMatch(match map[string]string) {
+	if s := match["subrepo"]; s != "" {
+		match["dot"] = "."
+		match["query"] = "?repo=" + s
+	} else {
+		match["dot"] = ""
+		match["query"] = ""
+	}
+}
+
+func getGoogleVCS(client *http.Client, match map[string]string) error {
+	// Scrape the HTML project page to find the VCS.
+	p, err := HttpGetBytes(client, expand("http://code.google.com/p/{repo}/source/checkout", match), nil)
+	if err != nil {
+		return err
+	}
+	m := googleRepoRe.FindSubmatch(p)
+	if m == nil {
+		return NotFoundError{"Could not VCS on Google Code project page."}
+	}
+
+	match["vcs"] = string(m[1])
+	return nil
+}
+
+// GetGoogleDoc downloads raw files from code.google.com.
+func GetGoogleDoc(client *http.Client, match map[string]string, installRepoPath string, nod *Node, cmdFlags map[string]bool) ([]string, error) {
+	setupGoogleMatch(match)
+	// Check version control.
+	if m := googleEtagRe.FindStringSubmatch(nod.Value); m != nil {
+		match["vcs"] = m[1]
+	} else if err := getGoogleVCS(client, match); err != nil {
+		return nil, err
+	}
+
+	rootPath := expand("http://{subrepo}{dot}{repo}.googlecode.com/{vcs}{dir}/", match)
+
+	// Scrape the repo browser to find the project revision and individual Go files.
+	p, err := HttpGetBytes(client, rootPath+"?r="+nod.Value, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	suf := "." + nod.Value
+	if len(suf) == 1 {
+		suf = ""
+	}
+
+	projectPath := expand("code.google.com/p/{repo}{dot}{subrepo}{dir}", match)
+	installPath := installRepoPath + "/" + projectPath + suf
+	nod.ImportPath = projectPath
+
+	// Remove old files.
+	os.RemoveAll(installPath + "/")
+	os.MkdirAll(installPath+"/", os.ModePerm)
+
+	// Get source files in root path.
+	files := make([]*source, 0, 5)
+	for _, m := range googleFileRe.FindAllSubmatch(p, -1) {
+		fname := strings.Split(string(m[1]), "?")[0]
+		if strings.HasPrefix(fname, ".") {
+			continue
+		}
+
+		files = append(files, &source{
+			name:   fname,
+			rawURL: expand("http://{subrepo}{dot}{repo}.googlecode.com/{vcs}{dir}/{0}", match, fname) + "?r=" + nod.Value,
+		})
+	}
+
+	// Fetch files from VCS.
+	if err := fetchFiles(client, files, nil); err != nil {
+		return nil, err
+	}
+
+	// Save files.
+	for _, f := range files {
+		absPath := installPath + "/"
+
+		// Create diretory before create file.
+		os.MkdirAll(path.Dir(absPath), os.ModePerm)
+
+		// Write data to file
+		fw, err := os.Create(absPath + f.name)
+		if err != nil {
+			return nil, err
+		}
+
+		_, err = fw.Write(f.data)
+		fw.Close()
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	dirs := make([]string, 0, 3)
+	// Get subdirectories.
+	for _, m := range googleDirRe.FindAllSubmatch(p, -1) {
+		dirName := strings.Split(string(m[1]), "?")[0]
+		if strings.HasSuffix(dirName, "/") {
+			dirs = append(dirs, dirName)
+		}
+	}
+
+	err = downloadFiles(client, match, rootPath, installPath+"/", nod.Value, dirs)
+	if err != nil {
+		return nil, err
+	}
+
+	var imports []string
+
+	// Check if need to check imports.
+	if nod.IsGetDeps {
+		dirs, err := GetDirsInfo(installPath + "/")
+		if err != nil {
+			return nil, err
+		}
+
+		for _, d := range dirs {
+			if d.IsDir() && !(!cmdFlags["-e"] && strings.Contains(d.Name(), "example")) {
+				absPath := installPath + "/" + d.Name() + "/"
+				importPkgs, err := CheckImports(absPath, match["importPath"])
+				if err != nil {
+					return nil, err
+				}
+				imports = append(imports, importPkgs...)
+			}
+		}
+	}
+
+	return imports, err
+}
+
+func downloadFiles(client *http.Client, match map[string]string, rootPath, installPath, commit string, dirs []string) error {
+	for _, d := range dirs {
+		p, err := HttpGetBytes(client, rootPath+d+"?r="+commit, nil)
+		if err != nil {
+			return err
+		}
+
+		// Create destination directory.
+		os.MkdirAll(installPath+d, os.ModePerm)
+
+		// Get source files in current path.
+		files := make([]*source, 0, 5)
+		for _, m := range googleFileRe.FindAllSubmatch(p, -1) {
+			fname := strings.Split(string(m[1]), "?")[0]
+			files = append(files, &source{
+				name:   fname,
+				rawURL: expand("http://{subrepo}{dot}{repo}.googlecode.com/{vcs}{dir}/", match) + d + fname + "?r=" + commit,
+			})
+		}
+
+		// Fetch files from VCS.
+		if err := fetchFiles(client, files, nil); err != nil {
+			return err
+		}
+
+		// Save files.
+		for _, f := range files {
+			absPath := installPath + d
+
+			// Create diretory before create file.
+			os.MkdirAll(path.Dir(absPath), os.ModePerm)
+
+			// Write data to file
+			fw, err := os.Create(absPath + f.name)
+			if err != nil {
+				return err
+			}
+
+			_, err = fw.Write(f.data)
+			fw.Close()
+			if err != nil {
+				return err
+			}
+		}
+
+		subdirs := make([]string, 0, 3)
+		// Get subdirectories.
+		for _, m := range googleDirRe.FindAllSubmatch(p, -1) {
+			dirName := strings.Split(string(m[1]), "?")[0]
+			if strings.HasSuffix(dirName, "/") {
+				subdirs = append(subdirs, d+dirName)
+			}
+		}
+
+		err = downloadFiles(client, match, rootPath, installPath, commit, subdirs)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}

doc/http.go

@@ -59,7 +59,7 @@ var (
 
 // httpGet gets the specified resource. ErrNotFound is returned if the server
 // responds with status 404.
-func httpGetBytes(client *http.Client, url string, header http.Header) ([]byte, error) {
+func HttpGetBytes(client *http.Client, url string, header http.Header) ([]byte, error) {
 	rc, err := httpGet(client, url, header)
 	if err != nil {
 		return nil, err