diff --git a/cmd/get.go b/cmd/get.go index 1c967d133..ef4cd7da0 100644 --- a/cmd/get.go +++ b/cmd/get.go @@ -17,10 +17,8 @@ package cmd import ( "errors" "fmt" - "net/http" "os/user" //"path" - "regexp" "strings" "github.com/gpmgo/gopm/doc" @@ -30,6 +28,7 @@ var ( installRepoPath string downloadCache map[string]bool // Saves packages that have been downloaded. downloadCount int + failConut int ) var CmdGet = &Command{ @@ -144,16 +143,18 @@ func runGet(cmd *Command, args []string) { } nodes = append(nodes, &doc.Node{ - ImportPath: args[0], - Type: t, - Value: ver, - IsGetDeps: true, + ImportPath: args[0], + DownloadURL: args[0], + Type: t, + Value: ver, + IsGetDeps: true, }) // Download package(s). downloadPackages(nodes) - doc.ColorLog("[INFO] %d package(s) downloaded.\n", downloadCount) + doc.ColorLog("[INFO] %d package(s) downloaded, %d failed.\n", + downloadCount, failConut) } // downloadPackages downloads packages with certain commit, @@ -186,9 +187,10 @@ func downloadPackages(nodes []*doc.Node) { nodes := make([]*doc.Node, len(imports)) for i := range nodes { nodes[i] = &doc.Node{ - ImportPath: imports[i], - Type: doc.BRANCH, - IsGetDeps: true, + ImportPath: imports[i], + DownloadURL: imports[i], + Type: doc.BRANCH, + IsGetDeps: true, } } downloadPackages(nodes) @@ -221,10 +223,11 @@ func downloadPackage(nod *doc.Node) (*doc.Node, []string) { // Mark as donwloaded. downloadCache[nod.ImportPath] = true - imports, err := pureDownload(nod) + imports, err := doc.PureDownload(nod, installRepoPath, CmdGet.Flags) if err != nil { doc.ColorLog("[ERRO] Download falied[ %s ]\n", err) + failConut++ return nil, nil } return nod, imports @@ -250,119 +253,3 @@ func validPath(info string) (string, string, error) { return "", "", errors.New("Cannot match any case") } } - -// service represents a source code control service. -type service struct { - pattern *regexp.Regexp - prefix string - get func(*http.Client, map[string]string, string, *doc.Node, map[string]bool) ([]string, error) -} - -// services is the list of source code control services handled by gopkgdoc. -var services = []*service{ - {doc.GithubPattern, "github.com/", doc.GetGithubDoc}, - {doc.GooglePattern, "code.google.com/", doc.GetGoogleDoc}, - {doc.BitbucketPattern, "bitbucket.org/", doc.GetBitbucketDoc}, - // {doc.LaunchpadPattern, "launchpad.net/", doc.GetLaunchpadDoc}, -} - -// pureDownload downloads package without version control. -func pureDownload(nod *doc.Node) ([]string, error) { - for _, s := range services { - if s.get == nil || !strings.HasPrefix(nod.ImportPath, s.prefix) { - continue - } - m := s.pattern.FindStringSubmatch(nod.ImportPath) - if m == nil { - if s.prefix != "" { - return nil, errors.New("Cannot match package service prefix by given path") - } - continue - } - match := map[string]string{"importPath": nod.ImportPath} - for i, n := range s.pattern.SubexpNames() { - if n != "" { - match[n] = m[i] - } - } - return s.get(doc.HttpClient, match, installRepoPath, nod, CmdGet.Flags) - } - return nil, errors.New("Cannot match any package service by given path") -} - -// func extractPkg(pkg *Pkg, localfile string, update bool) error { -// fmt.Println("Extracting package", pkg.Name, "...") - -// gopath := os.Getenv("GOPATH") -// var childDirs []string = strings.Split(pkg.Name, "/") - -// if pkg.Ver != TRUNK { -// childDirs[len(childDirs)-1] = fmt.Sprintf("%v_%v_%v", childDirs[len(childDirs)-1], pkg.Ver, pkg.VerId) -// } -// dstDir := joinPath(gopath, "src", joinPath(childDirs...)) -// //fmt.Println(dstDir) -// var err error -// if !update { -// if dirExists(dstDir) { -// return nil -// } -// err = os.MkdirAll(dstDir, 0777) -// } else { -// if dirExists(dstDir) { -// err = os.Remove(dstDir) -// } else { -// err = os.MkdirAll(dstDir, 0777) -// } -// } - -// if err != nil { -// return err -// } - -// if path.Ext(localfile) != ".zip" { -// return errors.New("Not implemented!") -// } - -// r, err := zip.OpenReader(localfile) -// if err != nil { -// return err -// } -// defer r.Close() - -// for _, f := range r.File { -// fmt.Printf("Contents of %s:\n", f.Name) -// if f.FileInfo().IsDir() { -// continue -// } - -// paths := strings.Split(f.Name, "/")[1:] -// //fmt.Println(paths) -// if len(paths) < 1 { -// continue -// } - -// if len(paths) > 1 { -// childDir := joinPath(dstDir, joinPath(paths[0:len(paths)-1]...)) -// //fmt.Println("creating", childDir) -// err = os.MkdirAll(childDir, 0777) -// if err != nil { -// return err -// } -// } - -// rc, err := f.Open() -// if err != nil { -// return err -// } - -// newF, err := os.Create(path.Join(dstDir, joinPath(paths...))) -// if err == nil { -// _, err = io.Copy(newF, rc) -// } -// if err != nil { -// return err -// } -// rc.Close() -// } -// return nil -// } diff --git a/doc/launchpad.go b/doc/launchpad.go new file mode 100644 index 000000000..4024e2a92 --- /dev/null +++ b/doc/launchpad.go @@ -0,0 +1,140 @@ +// Copyright 2013 gopm authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"): you may +// not use this file except in compliance with the License. You may obtain +// a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations +// under the License. + +package doc + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "io" + "net/http" + "os" + //"path" + "regexp" + "strings" +) + +var LaunchpadPattern = regexp.MustCompile(`^launchpad\.net/(?P(?P[a-z0-9A-Z_.\-]+)(?P/[a-z0-9A-Z_.\-]+)?|~[a-z0-9A-Z_.\-]+/(\+junk|[a-z0-9A-Z_.\-]+)/[a-z0-9A-Z_.\-]+)(?P/[a-z0-9A-Z_.\-/]+)*$`) + +// GetLaunchpadDoc downloads tarball from launchpad.net. +func GetLaunchpadDoc(client *http.Client, match map[string]string, installRepoPath string, nod *Node, cmdFlags map[string]bool) ([]string, error) { + + if match["project"] != "" && match["series"] != "" { + rc, err := httpGet(client, expand("https://code.launchpad.net/{project}{series}/.bzr/branch-format", match), nil) + switch { + case err == nil: + rc.Close() + // The structure of the import path is launchpad.net/{root}/{dir}. + case isNotFound(err): + // The structure of the import path is is launchpad.net/{project}/{dir}. + match["repo"] = match["project"] + match["dir"] = expand("{series}{dir}", match) + default: + return nil, err + } + } + + var downloadPath string + // Check if download with specific revision. + if len(nod.Value) == 0 { + downloadPath = expand("https://bazaar.launchpad.net/+branch/{repo}/tarball", match) + } else { + downloadPath = expand("https://bazaar.launchpad.net/+branch/{repo}/tarball/"+nod.Value, match) + } + + // Scrape the repo browser to find the project revision and individual Go files. + p, err := HttpGetBytes(client, downloadPath, nil) + if err != nil { + return nil, err + } + + installPath := installRepoPath + "/" + nod.ImportPath + + // Remove old files. + os.RemoveAll(installPath + "/") + os.MkdirAll(installPath+"/", os.ModePerm) + + gzr, err := gzip.NewReader(bytes.NewReader(p)) + if err != nil { + return nil, err + } + defer gzr.Close() + + tr := tar.NewReader(gzr) + + var autoPath string // Auto path is the root path that generated by bitbucket.org. + // Get source file data. + dirs := make([]string, 0, 5) + for { + h, err := tr.Next() + if err == io.EOF { + break + } else if err != nil { + return nil, err + } + + fn := h.FileInfo().Name() + // Check root path. + if len(autoPath) == 0 { + autoPath = fn[:strings.Index(fn, match["repo"])+len(match["repo"])] + } + absPath := strings.Replace(fn, autoPath, installPath, 1) + + switch { + case h.FileInfo().IsDir(): // Directory. + // Create diretory before create file. + os.MkdirAll(absPath+"/", os.ModePerm) + // Check if current directory is example. + if !(!cmdFlags["-e"] && strings.Contains(absPath, "example")) { + dirs = append(dirs, absPath) + } + case !strings.HasPrefix(fn, "."): + //os.MkdirAll(path.Dir(absPath)+"/", os.ModePerm) + + // Get data from archive. + fbytes := make([]byte, h.Size) + if _, err := io.ReadFull(tr, fbytes); err != nil { + return nil, err + } + + // Write data to file + fw, err := os.Create(absPath) + if err != nil { + return nil, err + } + + _, err = fw.Write(fbytes) + fw.Close() + if err != nil { + return nil, err + } + } + } + + var imports []string + + // Check if need to check imports. + if nod.IsGetDeps { + for _, d := range dirs { + importPkgs, err := CheckImports(d+"/", match["importPath"]) + if err != nil { + return nil, err + } + imports = append(imports, importPkgs...) + } + } + + return imports, err +} diff --git a/doc/struct.go b/doc/struct.go index 96b769e83..968ddf982 100644 --- a/doc/struct.go +++ b/doc/struct.go @@ -16,7 +16,9 @@ package doc import ( "go/token" + "net/http" "os" + "regexp" "time" ) @@ -30,10 +32,11 @@ const ( ) type Node struct { - ImportPath string - Type string - Value string // Branch, tag or commit. - IsGetDeps bool + ImportPath string + DownloadURL string + Type string + Value string // Branch, tag or commit. + IsGetDeps bool } // source is source code file. @@ -56,3 +59,18 @@ type walker struct { srcs map[string]*source // Source files. fset *token.FileSet } + +// service represents a source code control service. +type service struct { + pattern *regexp.Regexp + prefix string + get func(*http.Client, map[string]string, string, *Node, map[string]bool) ([]string, error) +} + +// services is the list of source code control services handled by gopkgdoc. +var services = []*service{ + {GithubPattern, "github.com/", GetGithubDoc}, + {GooglePattern, "code.google.com/", GetGoogleDoc}, + {BitbucketPattern, "bitbucket.org/", GetBitbucketDoc}, + {LaunchpadPattern, "launchpad.net/", GetLaunchpadDoc}, +} diff --git a/doc/vcs.go b/doc/vcs.go index 1ab3231d2..c54ef4435 100644 --- a/doc/vcs.go +++ b/doc/vcs.go @@ -16,8 +16,12 @@ package doc import ( "bytes" + "encoding/xml" + "errors" + "io" "io/ioutil" "log" + "net/http" "os" "os/exec" "path" @@ -244,3 +248,149 @@ func CheckImports(absPath, importPath string) (importPkgs []string, err error) { return importPkgs, err } + +// PureDownload downloads package without version control. +func PureDownload(nod *Node, installRepoPath string, flags map[string]bool) ([]string, error) { + for _, s := range services { + if s.get == nil || !strings.HasPrefix(nod.DownloadURL, s.prefix) { + continue + } + m := s.pattern.FindStringSubmatch(nod.DownloadURL) + if m == nil { + if s.prefix != "" { + return nil, errors.New("Cannot match package service prefix by given path") + } + continue + } + match := map[string]string{"importPath": nod.DownloadURL} + for i, n := range s.pattern.SubexpNames() { + if n != "" { + match[n] = m[i] + } + } + return s.get(HttpClient, match, installRepoPath, nod, flags) + } + + ColorLog("[TRAC] Cannot match any service, getting dynamic...\n") + return getDynamic(HttpClient, nod, installRepoPath, flags) +} + +func getDynamic(client *http.Client, nod *Node, installRepoPath string, flags map[string]bool) ([]string, error) { + match, err := fetchMeta(client, nod.ImportPath) + if err != nil { + return nil, err + } + + if match["projectRoot"] != nod.ImportPath { + rootMatch, err := fetchMeta(client, match["projectRoot"]) + if err != nil { + return nil, err + } + if rootMatch["projectRoot"] != match["projectRoot"] { + return nil, NotFoundError{"Project root mismatch."} + } + } + + nod.DownloadURL = expand("{repo}{dir}", match) + return PureDownload(nod, installRepoPath, flags) +} + +func fetchMeta(client *http.Client, importPath string) (map[string]string, error) { + uri := importPath + if !strings.Contains(uri, "/") { + // Add slash for root of domain. + uri = uri + "/" + } + uri = uri + "?go-get=1" + + scheme := "https" + resp, err := client.Get(scheme + "://" + uri) + if err != nil || resp.StatusCode != 200 { + if err == nil { + resp.Body.Close() + } + scheme = "http" + resp, err = client.Get(scheme + "://" + uri) + if err != nil { + return nil, &RemoteError{strings.SplitN(importPath, "/", 2)[0], err} + } + } + defer resp.Body.Close() + return parseMeta(scheme, importPath, resp.Body) +} + +func attrValue(attrs []xml.Attr, name string) string { + for _, a := range attrs { + if strings.EqualFold(a.Name.Local, name) { + return a.Value + } + } + return "" +} + +func parseMeta(scheme, importPath string, r io.Reader) (map[string]string, error) { + var match map[string]string + + d := xml.NewDecoder(r) + d.Strict = false +metaScan: + for { + t, tokenErr := d.Token() + if tokenErr != nil { + break metaScan + } + switch t := t.(type) { + case xml.EndElement: + if strings.EqualFold(t.Name.Local, "head") { + break metaScan + } + case xml.StartElement: + if strings.EqualFold(t.Name.Local, "body") { + break metaScan + } + if !strings.EqualFold(t.Name.Local, "meta") || + attrValue(t.Attr, "name") != "go-import" { + continue metaScan + } + f := strings.Fields(attrValue(t.Attr, "content")) + if len(f) != 3 || + !strings.HasPrefix(importPath, f[0]) || + !(len(importPath) == len(f[0]) || importPath[len(f[0])] == '/') { + continue metaScan + } + if match != nil { + return nil, NotFoundError{"More than one found at " + scheme + "://" + importPath} + } + + projectRoot, vcs, repo := f[0], f[1], f[2] + + repo = strings.TrimSuffix(repo, "."+vcs) + i := strings.Index(repo, "://") + if i < 0 { + return nil, NotFoundError{"Bad repo URL in ."} + } + proto := repo[:i] + repo = repo[i+len("://"):] + + match = map[string]string{ + // Used in getVCSDoc, same as vcsPattern matches. + "importPath": importPath, + "repo": repo, + "vcs": vcs, + "dir": importPath[len(projectRoot):], + + // Used in getVCSDoc + "scheme": proto, + + // Used in getDynamic. + "projectRoot": projectRoot, + "projectName": path.Base(projectRoot), + "projectURL": scheme + "://" + projectRoot, + } + } + } + if match == nil { + return nil, NotFoundError{" not found."} + } + return match, nil +} diff --git a/doc/walker.go b/doc/walker.go index 0bc597e24..797c301ae 100644 --- a/doc/walker.go +++ b/doc/walker.go @@ -16,7 +16,6 @@ package doc import ( "bytes" - "errors" "go/ast" "go/build" "go/parser" @@ -114,7 +113,8 @@ func (w *walker) build(srcs []*source) ([]string, error) { if nogo { err = nil } else { - return nil, errors.New("doc.walker.build(): " + err.Error()) + ColorLog("[WARN] Error occurs when check imports[ %s ]\n", err) + return nil, nil } }