From 7681d9bf884c65a9f19b00238dc1b46df12521fc Mon Sep 17 00:00:00 2001 From: Unknown Date: Mon, 20 May 2013 20:25:27 -0400 Subject: [PATCH] command install add support for code.google.com --- README.md | 6 +- doc/github.go | 12 +-- doc/google.go | 272 ++++++++++++++++++++++++++++++++++++++++++++++++++ doc/walker.go | 2 +- install.go | 20 ++-- 5 files changed, 293 insertions(+), 19 deletions(-) create mode 100644 doc/google.go diff --git a/README.md b/README.md index c4068b4b5..85763bab8 100644 --- a/README.md +++ b/README.md @@ -7,11 +7,13 @@ gpm(Go Package Manager) is a Go package manage tool for search, install, update ## Todo -- Command `install` add support for downloading code from code.google.com, launchpad.net, bitbucket.org; hopefully, support user sources for downloading tarballs. +- Command `install` add support for downloading code from launchpad.net, bitbucket.org; hopefully, support user sources for downloading tarballs. - Command `install` installs all packages after downloaded. - After downloaded all packages in bundles or snapshots, need to check if all dependencies have been downloaded as well. - Develop user source API server template application to support user sources in bundles. - Add bundle and snapshot parser code for downloading by bundle or snapshot id. - Add user system to create, edit, upload, and download bundles or snapshots through gpm client program. - Add option for whether download dependencies packages in example code or not. -- Add gpm working principle design. \ No newline at end of file +- Add gpm working principle design. +- Download package from code.google.com only support hg as version control system, probably support git and svn. +- All errors should have specific title for exactly where were created. \ No newline at end of file diff --git a/doc/github.go b/doc/github.go index f4da49095..514baa6e7 100644 --- a/doc/github.go +++ b/doc/github.go @@ -28,7 +28,7 @@ func SetGithubCredentials(id, secret string) { } // GetGithubDoc downloads tarball from github.com. -func GetGithubDoc(client *http.Client, match map[string]string, commit string, isDownloadEx bool) (*Package, []string, error) { +func GetGithubDoc(client *http.Client, match map[string]string, commit string, cmdFlags map[string]bool) (*Package, []string, error) { SetGithubCredentials("1862bcb265171f37f36c", "308d71ab53ccd858416cfceaed52d5d5b7d53c5f") match["cred"] = githubCred @@ -94,11 +94,10 @@ func GetGithubDoc(client *http.Client, match map[string]string, commit string, i installPath := paths[0] + "/src/" + importPath // Remove old files. - os.RemoveAll(installPath) + os.RemoveAll(installPath + "/") // Create destination directory. - os.Mkdir(installPath, os.ModePerm) + os.MkdirAll(installPath+"/", os.ModePerm) - //dirMap := make(map[string][]*source) dirs := make([]string, 0, 5) for _, f := range r.File { absPath := strings.Replace(f.FileInfo().Name(), shaName, installPath, 1) @@ -107,7 +106,7 @@ func GetGithubDoc(client *http.Client, match map[string]string, commit string, i if strings.HasSuffix(absPath, "/") { // Directory. // Check if current directory is example. - if !(!isDownloadEx && strings.Contains(absPath, "example")) { + if !(!cmdFlags["-e"] && strings.Contains(absPath, "example")) { dirs = append(dirs, absPath) } continue @@ -119,8 +118,9 @@ func GetGithubDoc(client *http.Client, match map[string]string, commit string, i return nil, nil, err } - // Create diretory before create file + // Create diretory before create file. os.MkdirAll(path.Dir(absPath), os.ModePerm) + // Write data to file fw, _ := os.Create(absPath) if err != nil { diff --git a/doc/google.go b/doc/google.go new file mode 100644 index 000000000..b98bba873 --- /dev/null +++ b/doc/google.go @@ -0,0 +1,272 @@ +// Copyright (c) 2013 GPMGo Members. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package doc + +import ( + "errors" + "net/http" + "os" + "path" + "regexp" + "strings" + + "github.com/GPMGo/gpm/utils" +) + +var ( + googleRepoRe = regexp.MustCompile(`id="checkoutcmd">(hg|git|svn)`) + googleRevisionRe = regexp.MustCompile(`

(?:[^ ]+ - )?Revision *([^:]+):`) + googleEtagRe = regexp.MustCompile(`^(hg|git|svn)-`) + googleFileRe = regexp.MustCompile(`
  • [a-z0-9\-]+)(:?\.(?P[a-z0-9\-]+))?(?P/[a-z0-9A-Z_.\-/]+)?$`) +) + +func setupGoogleMatch(match map[string]string) { + if s := match["subrepo"]; s != "" { + match["dot"] = "." + match["query"] = "?repo=" + s + } else { + match["dot"] = "" + match["query"] = "" + } +} + +func getGoogleVCS(client *http.Client, match map[string]string) error { + // Scrape the HTML project page to find the VCS. + p, err := httpGetBytes(client, expand("http://code.google.com/p/{repo}/source/checkout", match), nil) + if err != nil { + return err + } + m := googleRepoRe.FindSubmatch(p) + if m == nil { + return NotFoundError{"Could not VCS on Google Code project page."} + } + + match["vcs"] = string(m[1]) + return nil +} + +// GetGoogleDoc downloads raw files from code.google.com. +func GetGoogleDoc(client *http.Client, match map[string]string, commit string, cmdFlags map[string]bool) (*Package, []string, error) { + setupGoogleMatch(match) + if m := googleEtagRe.FindStringSubmatch(commit); m != nil { + match["vcs"] = m[1] + } else if err := getGoogleVCS(client, match); err != nil { + return nil, nil, err + } + + // bundle and snapshot will have commit 'B' and 'S', + // but does not need to download dependencies. + isCheckImport := len(commit) == 0 + + rootPath := expand("http://{subrepo}{dot}{repo}.googlecode.com/{vcs}{dir}/", match) + + // Scrape the repo browser to find the project revision and individual Go files. + p, err := httpGetBytes(client, rootPath+"?r="+commit, nil) + if err != nil { + return nil, nil, err + } + + // Check revision tag. + if m := googleRevisionRe.FindSubmatch(p); m == nil { + return nil, nil, + errors.New("doc.GetGoogleDoc(): Could not find revision for " + match["importPath"]) + } + + paths := utils.GetGOPATH() + importPath := "code.google.com/p/" + expand("{repo}{dir}", match) + installPath := paths[0] + "/src/" + importPath + + // Remove old files. + os.RemoveAll(installPath + "/") + // Create destination directory. + os.MkdirAll(installPath+"/", os.ModePerm) + + // Get source files in root path. + files := make([]*source, 0, 5) + for _, m := range googleFileRe.FindAllSubmatch(p, -1) { + fname := string(m[1]) + files = append(files, &source{ + name: fname, + rawURL: expand("http://{subrepo}{dot}{repo}.googlecode.com/{vcs}{dir}/{0}", match, fname) + "?r=" + commit, + }) + } + + // Fetch files from VCS. + if err := fetchFiles(client, files, nil); err != nil { + return nil, nil, err + } + + // Save files. + for _, f := range files { + absPath := installPath + "/" + + // Create diretory before create file. + os.MkdirAll(path.Dir(absPath), os.ModePerm) + + // Write data to file + fw, err := os.Create(absPath + f.name) + if err != nil { + return nil, nil, err + } + + _, err = fw.Write(f.data) + fw.Close() + if err != nil { + return nil, nil, err + } + } + + dirs := make([]string, 0, 3) + // Get subdirectories. + for _, m := range googleDirRe.FindAllSubmatch(p, -1) { + dirName := strings.Split(string(m[1]), "?")[0] + if strings.HasSuffix(dirName, "/") { + dirs = append(dirs, dirName) + } + } + + err = downloadFiles(client, match, rootPath, installPath+"/", commit, dirs) + if err != nil { + return nil, nil, err + } + + pkg := &Package{ + ImportPath: importPath, + AbsPath: installPath, + Commit: commit, + } + var imports []string + + // Check if need to check imports. + if isCheckImport { + rootdir, err := os.Open(installPath + "/") + if err != nil { + return nil, nil, err + } + defer rootdir.Close() + + dirs, err := rootdir.Readdir(0) + if err != nil { + return nil, nil, err + } + + for _, d := range dirs { + if d.IsDir() { + absPath := installPath + "/" + d.Name() + "/" + dir, err := os.Open(absPath) + if err != nil { + return nil, nil, err + } + defer dir.Close() + + // Get file info slice. + fis, err := dir.Readdir(0) + if err != nil { + return nil, nil, err + } + + files := make([]*source, 0, 10) + for _, fi := range fis { + // Only handle files. + if strings.HasSuffix(fi.Name(), ".go") { + f, err := os.Open(absPath + fi.Name()) + if err != nil { + return nil, nil, err + } + + fbytes := make([]byte, fi.Size()) + _, err = f.Read(fbytes) + f.Close() + //fmt.Println(d+fi.Name(), fi.Size(), n) + if err != nil { + return nil, nil, err + } + + files = append(files, &source{ + name: fi.Name(), + data: fbytes, + }) + } + } + + // Check if has Go source files. + if len(files) > 0 { + w := &walker{ImportPath: importPath} + importPkgs, err := w.build(files) + if err != nil { + return nil, nil, err + } + imports = append(imports, importPkgs...) + } + } + } + } + + return pkg, imports, err +} + +func downloadFiles(client *http.Client, match map[string]string, rootPath, installPath, commit string, dirs []string) error { + for _, d := range dirs { + p, err := httpGetBytes(client, rootPath+d+"?r="+commit, nil) + if err != nil { + return err + } + + // Create destination directory. + os.MkdirAll(installPath+d, os.ModePerm) + + // Get source files in current path. + files := make([]*source, 0, 5) + for _, m := range googleFileRe.FindAllSubmatch(p, -1) { + fname := string(m[1]) + files = append(files, &source{ + name: fname, + rawURL: expand("http://{subrepo}{dot}{repo}.googlecode.com/{vcs}{dir}/", match) + d + fname + "?r=" + commit, + }) + } + + // Fetch files from VCS. + if err := fetchFiles(client, files, nil); err != nil { + return err + } + + // Save files. + for _, f := range files { + absPath := installPath + d + + // Create diretory before create file. + os.MkdirAll(path.Dir(absPath), os.ModePerm) + + // Write data to file + fw, err := os.Create(absPath + f.name) + if err != nil { + return err + } + + _, err = fw.Write(f.data) + fw.Close() + if err != nil { + return err + } + } + + subdirs := make([]string, 0, 3) + // Get subdirectories. + for _, m := range googleDirRe.FindAllSubmatch(p, -1) { + dirName := strings.Split(string(m[1]), "?")[0] + if strings.HasSuffix(dirName, "/") { + subdirs = append(subdirs, d+dirName) + } + } + + err = downloadFiles(client, match, rootPath, installPath, commit, subdirs) + if err != nil { + return err + } + } + return nil +} diff --git a/doc/walker.go b/doc/walker.go index b9f3a247a..b56152e7f 100644 --- a/doc/walker.go +++ b/doc/walker.go @@ -136,7 +136,7 @@ func (w *walker) build(srcs []*source) ([]string, error) { var imports []string for _, v := range bpkg.Imports { // Skip strandard library. - if !utils.IsGoRepoPath(v) && v != w.ImportPath { + if !utils.IsGoRepoPath(v) && !strings.HasPrefix(v, w.ImportPath) { imports = append(imports, v) } } diff --git a/install.go b/install.go index afff74734..110d252a1 100644 --- a/install.go +++ b/install.go @@ -106,7 +106,7 @@ func runInstall(cmd *Command, args []string) { // Download packages. commits := make([]string, len(args)) - downloadPackages(args, commits, cmdInstall.Flags["-e"]) + downloadPackages(args, commits) if !cmdInstall.Flags["d"] && cmdInstall.Flags["-p"] { // Install packages all together. @@ -119,7 +119,7 @@ func runInstall(cmd *Command, args []string) { // downloadPackages downloads packages with certain commit, // if the commit is empty string, then it downloads all dependencies, // otherwise, it only downloada package with specific commit only. -func downloadPackages(pkgs, commits []string, isDownloadEx bool) { +func downloadPackages(pkgs, commits []string) { // Check all packages, they may be bundles, snapshots or raw packages path. for i, p := range pkgs { // Check if it is a bundle or snapshot. @@ -131,11 +131,11 @@ func downloadPackages(pkgs, commits []string, isDownloadEx bool) { case utils.IsValidRemotePath(p): if !downloadCache[p] { // Download package. - pkg, imports := downloadPackage(p, commits[i], isDownloadEx) + pkg, imports := downloadPackage(p, commits[i]) if len(imports) > 0 { // Need to download dependencies. tags := make([]string, len(imports)) - downloadPackages(imports, tags, isDownloadEx) + downloadPackages(imports, tags) continue } @@ -155,7 +155,7 @@ func downloadPackages(pkgs, commits []string, isDownloadEx bool) { } // downloadPackage download package either use version control tools or not. -func downloadPackage(path, commit string, isDownloadEx bool) (pkg *doc.Package, imports []string) { +func downloadPackage(path, commit string) (pkg *doc.Package, imports []string) { // Check if use version control tools. switch { case !cmdInstall.Flags["-p"] && @@ -176,7 +176,7 @@ func downloadPackage(path, commit string, isDownloadEx bool) (pkg *doc.Package, downloadCache[path] = true var err error - pkg, imports, err = pureDownload(path, commit, isDownloadEx) + pkg, imports, err = pureDownload(path, commit) if err != nil { fmt.Printf("Fail to download package(%s) with error: %s.\n", path, err) return nil, nil @@ -204,19 +204,19 @@ func checkGoGetFlags() (args []string) { type service struct { pattern *regexp.Regexp prefix string - get func(*http.Client, map[string]string, string, bool) (*doc.Package, []string, error) + get func(*http.Client, map[string]string, string, map[string]bool) (*doc.Package, []string, error) } // services is the list of source code control services handled by gopkgdoc. var services = []*service{ {doc.GithubPattern, "github.com/", doc.GetGithubDoc}, - //{googlePattern, "code.google.com/", getGoogleDoc}, + {doc.GooglePattern, "code.google.com/", doc.GetGoogleDoc}, //{bitbucketPattern, "bitbucket.org/", getBitbucketDoc}, //{launchpadPattern, "launchpad.net/", getLaunchpadDoc}, } // pureDownload downloads package without version control. -func pureDownload(path, commit string, isDownloadEx bool) (pinfo *doc.Package, imports []string, err error) { +func pureDownload(path, commit string) (pinfo *doc.Package, imports []string, err error) { for _, s := range services { if s.get == nil || !strings.HasPrefix(path, s.prefix) { continue @@ -235,7 +235,7 @@ func pureDownload(path, commit string, isDownloadEx bool) (pinfo *doc.Package, i match[n] = m[i] } } - return s.get(doc.HttpClient, match, commit, isDownloadEx) + return s.get(doc.HttpClient, match, commit, cmdInstall.Flags) } return nil, nil, doc.ErrNoMatch }