diff --git a/cmd/get.go b/cmd/get.go index 0b41c684b..17bb1d86c 100644 --- a/cmd/get.go +++ b/cmd/get.go @@ -166,7 +166,7 @@ func downloadPackages(nodes []*doc.Node) { if doc.IsValidRemotePath(n.ImportPath) { if !CmdGet.Flags["-u"] { // Check if package has been downloaded. - installPath := installRepoPath + "/" + n.ImportPath + installPath := installRepoPath + "/" + doc.GetProjectPath(n.ImportPath) if len(n.Value) > 0 { installPath += "." + n.Value } @@ -261,7 +261,7 @@ type service struct { // services is the list of source code control services handled by gopkgdoc. var services = []*service{ {doc.GithubPattern, "github.com/", doc.GetGithubDoc}, - // {doc.GooglePattern, "code.google.com/", doc.GetGoogleDoc}, + {doc.GooglePattern, "code.google.com/", doc.GetGoogleDoc}, // {doc.BitbucketPattern, "bitbucket.org/", doc.GetBitbucketDoc}, // {doc.LaunchpadPattern, "launchpad.net/", doc.GetLaunchpadDoc}, } @@ -290,46 +290,6 @@ func pureDownload(nod *doc.Node) ([]string, error) { return nil, errors.New("Cannot match any package service by given path") } -// func joinPath(paths ...string) string { -// if len(paths) < 1 { -// return "" -// } -// res := "" -// for _, p := range paths { -// res = path.Join(res, p) -// } -// return res -// } - -// func download(url string, localfile string) error { -// fmt.Println("Downloading", url, "...") -// resp, err := http.Get(url) -// if err != nil { -// return err -// } -// defer resp.Body.Close() - -// localdir := filepath.Dir(localfile) -// if !dirExists(localdir) { -// err = os.MkdirAll(localdir, 0777) -// if err != nil { -// return err -// } -// } - -// if !fileExists(localfile) { -// f, err := os.Create(localfile) -// if err == nil { -// _, err = io.Copy(f, resp.Body) -// } -// if err != nil { -// return err -// } -// } - -// return nil -// } - // func extractPkg(pkg *Pkg, localfile string, update bool) error { // fmt.Println("Extracting package", pkg.Name, "...") @@ -406,38 +366,3 @@ func pureDownload(nod *doc.Node) ([]string, error) { // } // return nil // } - -// func getPackage(pkg *Pkg, url string) error { -// curUser, err := user.Current() -// if err != nil { -// return err -// } - -// reposDir = strings.Replace(reposDir, "~", curUser.HomeDir, -1) -// localdir := path.Join(reposDir, pkg.Name) -// localdir, err = filepath.Abs(localdir) -// if err != nil { -// return err -// } - -// localfile := path.Join(localdir, pkg.FileName()) - -// err = download(url, localfile) -// if err != nil { -// return err -// } - -// return extractPkg(pkg, localfile, false) -// } - -// func getDirect(pkg *Pkg) error { -// return getPackage(pkg, pkg.Url()) -// } - -/*func getFromSource(pkgName string, ver string, source string) error { - urlTempl := "https://%v/%v" - //urlTempl := "https://%v/archive/master.zip" - url := fmt.Sprintf(urlTempl, source, pkgName) - - return getPackage(pkgName, ver, url) -}*/ diff --git a/doc/github.go b/doc/github.go index 1fe87db67..e5ef672f5 100644 --- a/doc/github.go +++ b/doc/github.go @@ -97,7 +97,7 @@ func GetGithubDoc(client *http.Client, match map[string]string, installRepoPath // tarball : https://github.com/{owner}/{repo}/tarball/{sha} // Downlaod archive. - p, err := httpGetBytes(client, expand("https://github.com/{owner}/{repo}/archive/{sha}.zip", match), nil) + p, err := HttpGetBytes(client, expand("https://github.com/{owner}/{repo}/archive/{sha}.zip", match), nil) if err != nil { return nil, err } diff --git a/doc/google.go b/doc/google.go new file mode 100644 index 000000000..7147deb99 --- /dev/null +++ b/doc/google.go @@ -0,0 +1,227 @@ +// Copyright 2013 gopm authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"): you may +// not use this file except in compliance with the License. You may obtain +// a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations +// under the License. + +package doc + +import ( + "net/http" + "os" + "path" + "regexp" + "strings" +) + +var ( + googleRepoRe = regexp.MustCompile(`id="checkoutcmd">(hg|git|svn)`) + googleRevisionRe = regexp.MustCompile(`

(?:[^ ]+ - )?Revision *([^:]+):`) + googleEtagRe = regexp.MustCompile(`^(hg|git|svn)-`) + googleFileRe = regexp.MustCompile(`
  • [a-z0-9\-]+)(:?\.(?P[a-z0-9\-]+))?(?P/[a-z0-9A-Z_.\-/]+)?$`) +) + +func setupGoogleMatch(match map[string]string) { + if s := match["subrepo"]; s != "" { + match["dot"] = "." + match["query"] = "?repo=" + s + } else { + match["dot"] = "" + match["query"] = "" + } +} + +func getGoogleVCS(client *http.Client, match map[string]string) error { + // Scrape the HTML project page to find the VCS. + p, err := HttpGetBytes(client, expand("http://code.google.com/p/{repo}/source/checkout", match), nil) + if err != nil { + return err + } + m := googleRepoRe.FindSubmatch(p) + if m == nil { + return NotFoundError{"Could not VCS on Google Code project page."} + } + + match["vcs"] = string(m[1]) + return nil +} + +// GetGoogleDoc downloads raw files from code.google.com. +func GetGoogleDoc(client *http.Client, match map[string]string, installRepoPath string, nod *Node, cmdFlags map[string]bool) ([]string, error) { + setupGoogleMatch(match) + // Check version control. + if m := googleEtagRe.FindStringSubmatch(nod.Value); m != nil { + match["vcs"] = m[1] + } else if err := getGoogleVCS(client, match); err != nil { + return nil, err + } + + rootPath := expand("http://{subrepo}{dot}{repo}.googlecode.com/{vcs}{dir}/", match) + + // Scrape the repo browser to find the project revision and individual Go files. + p, err := HttpGetBytes(client, rootPath+"?r="+nod.Value, nil) + if err != nil { + return nil, err + } + + suf := "." + nod.Value + if len(suf) == 1 { + suf = "" + } + + projectPath := expand("code.google.com/p/{repo}{dot}{subrepo}{dir}", match) + installPath := installRepoPath + "/" + projectPath + suf + nod.ImportPath = projectPath + + // Remove old files. + os.RemoveAll(installPath + "/") + os.MkdirAll(installPath+"/", os.ModePerm) + + // Get source files in root path. + files := make([]*source, 0, 5) + for _, m := range googleFileRe.FindAllSubmatch(p, -1) { + fname := strings.Split(string(m[1]), "?")[0] + if strings.HasPrefix(fname, ".") { + continue + } + + files = append(files, &source{ + name: fname, + rawURL: expand("http://{subrepo}{dot}{repo}.googlecode.com/{vcs}{dir}/{0}", match, fname) + "?r=" + nod.Value, + }) + } + + // Fetch files from VCS. + if err := fetchFiles(client, files, nil); err != nil { + return nil, err + } + + // Save files. + for _, f := range files { + absPath := installPath + "/" + + // Create diretory before create file. + os.MkdirAll(path.Dir(absPath), os.ModePerm) + + // Write data to file + fw, err := os.Create(absPath + f.name) + if err != nil { + return nil, err + } + + _, err = fw.Write(f.data) + fw.Close() + if err != nil { + return nil, err + } + } + + dirs := make([]string, 0, 3) + // Get subdirectories. + for _, m := range googleDirRe.FindAllSubmatch(p, -1) { + dirName := strings.Split(string(m[1]), "?")[0] + if strings.HasSuffix(dirName, "/") { + dirs = append(dirs, dirName) + } + } + + err = downloadFiles(client, match, rootPath, installPath+"/", nod.Value, dirs) + if err != nil { + return nil, err + } + + var imports []string + + // Check if need to check imports. + if nod.IsGetDeps { + dirs, err := GetDirsInfo(installPath + "/") + if err != nil { + return nil, err + } + + for _, d := range dirs { + if d.IsDir() && !(!cmdFlags["-e"] && strings.Contains(d.Name(), "example")) { + absPath := installPath + "/" + d.Name() + "/" + importPkgs, err := CheckImports(absPath, match["importPath"]) + if err != nil { + return nil, err + } + imports = append(imports, importPkgs...) + } + } + } + + return imports, err +} + +func downloadFiles(client *http.Client, match map[string]string, rootPath, installPath, commit string, dirs []string) error { + for _, d := range dirs { + p, err := HttpGetBytes(client, rootPath+d+"?r="+commit, nil) + if err != nil { + return err + } + + // Create destination directory. + os.MkdirAll(installPath+d, os.ModePerm) + + // Get source files in current path. + files := make([]*source, 0, 5) + for _, m := range googleFileRe.FindAllSubmatch(p, -1) { + fname := strings.Split(string(m[1]), "?")[0] + files = append(files, &source{ + name: fname, + rawURL: expand("http://{subrepo}{dot}{repo}.googlecode.com/{vcs}{dir}/", match) + d + fname + "?r=" + commit, + }) + } + + // Fetch files from VCS. + if err := fetchFiles(client, files, nil); err != nil { + return err + } + + // Save files. + for _, f := range files { + absPath := installPath + d + + // Create diretory before create file. + os.MkdirAll(path.Dir(absPath), os.ModePerm) + + // Write data to file + fw, err := os.Create(absPath + f.name) + if err != nil { + return err + } + + _, err = fw.Write(f.data) + fw.Close() + if err != nil { + return err + } + } + + subdirs := make([]string, 0, 3) + // Get subdirectories. + for _, m := range googleDirRe.FindAllSubmatch(p, -1) { + dirName := strings.Split(string(m[1]), "?")[0] + if strings.HasSuffix(dirName, "/") { + subdirs = append(subdirs, d+dirName) + } + } + + err = downloadFiles(client, match, rootPath, installPath, commit, subdirs) + if err != nil { + return err + } + } + return nil +} diff --git a/doc/http.go b/doc/http.go index 7b307cd45..d63e31da9 100644 --- a/doc/http.go +++ b/doc/http.go @@ -59,7 +59,7 @@ var ( // httpGet gets the specified resource. ErrNotFound is returned if the server // responds with status 404. -func httpGetBytes(client *http.Client, url string, header http.Header) ([]byte, error) { +func HttpGetBytes(client *http.Client, url string, header http.Header) ([]byte, error) { rc, err := httpGet(client, url, header) if err != nil { return nil, err