Browse Source

Almost done get command

pull/103/head
Unknown 12 years ago
parent
commit
be9d0004c8
  1. 141
      cmd/get.go
  2. 140
      doc/launchpad.go
  3. 26
      doc/struct.go
  4. 150
      doc/vcs.go
  5. 4
      doc/walker.go

141
cmd/get.go

@ -17,10 +17,8 @@ package cmd
import ( import (
"errors" "errors"
"fmt" "fmt"
"net/http"
"os/user" "os/user"
//"path" //"path"
"regexp"
"strings" "strings"
"github.com/gpmgo/gopm/doc" "github.com/gpmgo/gopm/doc"
@ -30,6 +28,7 @@ var (
installRepoPath string installRepoPath string
downloadCache map[string]bool // Saves packages that have been downloaded. downloadCache map[string]bool // Saves packages that have been downloaded.
downloadCount int downloadCount int
failConut int
) )
var CmdGet = &Command{ var CmdGet = &Command{
@ -144,16 +143,18 @@ func runGet(cmd *Command, args []string) {
} }
nodes = append(nodes, &doc.Node{ nodes = append(nodes, &doc.Node{
ImportPath: args[0], ImportPath: args[0],
Type: t, DownloadURL: args[0],
Value: ver, Type: t,
IsGetDeps: true, Value: ver,
IsGetDeps: true,
}) })
// Download package(s). // Download package(s).
downloadPackages(nodes) downloadPackages(nodes)
doc.ColorLog("[INFO] %d package(s) downloaded.\n", downloadCount) doc.ColorLog("[INFO] %d package(s) downloaded, %d failed.\n",
downloadCount, failConut)
} }
// downloadPackages downloads packages with certain commit, // downloadPackages downloads packages with certain commit,
@ -186,9 +187,10 @@ func downloadPackages(nodes []*doc.Node) {
nodes := make([]*doc.Node, len(imports)) nodes := make([]*doc.Node, len(imports))
for i := range nodes { for i := range nodes {
nodes[i] = &doc.Node{ nodes[i] = &doc.Node{
ImportPath: imports[i], ImportPath: imports[i],
Type: doc.BRANCH, DownloadURL: imports[i],
IsGetDeps: true, Type: doc.BRANCH,
IsGetDeps: true,
} }
} }
downloadPackages(nodes) downloadPackages(nodes)
@ -221,10 +223,11 @@ func downloadPackage(nod *doc.Node) (*doc.Node, []string) {
// Mark as donwloaded. // Mark as donwloaded.
downloadCache[nod.ImportPath] = true downloadCache[nod.ImportPath] = true
imports, err := pureDownload(nod) imports, err := doc.PureDownload(nod, installRepoPath, CmdGet.Flags)
if err != nil { if err != nil {
doc.ColorLog("[ERRO] Download falied[ %s ]\n", err) doc.ColorLog("[ERRO] Download falied[ %s ]\n", err)
failConut++
return nil, nil return nil, nil
} }
return nod, imports return nod, imports
@ -250,119 +253,3 @@ func validPath(info string) (string, string, error) {
return "", "", errors.New("Cannot match any case") return "", "", errors.New("Cannot match any case")
} }
} }
// service represents a source code control service.
type service struct {
pattern *regexp.Regexp
prefix string
get func(*http.Client, map[string]string, string, *doc.Node, map[string]bool) ([]string, error)
}
// services is the list of source code control services handled by gopkgdoc.
var services = []*service{
{doc.GithubPattern, "github.com/", doc.GetGithubDoc},
{doc.GooglePattern, "code.google.com/", doc.GetGoogleDoc},
{doc.BitbucketPattern, "bitbucket.org/", doc.GetBitbucketDoc},
// {doc.LaunchpadPattern, "launchpad.net/", doc.GetLaunchpadDoc},
}
// pureDownload downloads package without version control.
func pureDownload(nod *doc.Node) ([]string, error) {
for _, s := range services {
if s.get == nil || !strings.HasPrefix(nod.ImportPath, s.prefix) {
continue
}
m := s.pattern.FindStringSubmatch(nod.ImportPath)
if m == nil {
if s.prefix != "" {
return nil, errors.New("Cannot match package service prefix by given path")
}
continue
}
match := map[string]string{"importPath": nod.ImportPath}
for i, n := range s.pattern.SubexpNames() {
if n != "" {
match[n] = m[i]
}
}
return s.get(doc.HttpClient, match, installRepoPath, nod, CmdGet.Flags)
}
return nil, errors.New("Cannot match any package service by given path")
}
// func extractPkg(pkg *Pkg, localfile string, update bool) error {
// fmt.Println("Extracting package", pkg.Name, "...")
// gopath := os.Getenv("GOPATH")
// var childDirs []string = strings.Split(pkg.Name, "/")
// if pkg.Ver != TRUNK {
// childDirs[len(childDirs)-1] = fmt.Sprintf("%v_%v_%v", childDirs[len(childDirs)-1], pkg.Ver, pkg.VerId)
// }
// dstDir := joinPath(gopath, "src", joinPath(childDirs...))
// //fmt.Println(dstDir)
// var err error
// if !update {
// if dirExists(dstDir) {
// return nil
// }
// err = os.MkdirAll(dstDir, 0777)
// } else {
// if dirExists(dstDir) {
// err = os.Remove(dstDir)
// } else {
// err = os.MkdirAll(dstDir, 0777)
// }
// }
// if err != nil {
// return err
// }
// if path.Ext(localfile) != ".zip" {
// return errors.New("Not implemented!")
// }
// r, err := zip.OpenReader(localfile)
// if err != nil {
// return err
// }
// defer r.Close()
// for _, f := range r.File {
// fmt.Printf("Contents of %s:\n", f.Name)
// if f.FileInfo().IsDir() {
// continue
// }
// paths := strings.Split(f.Name, "/")[1:]
// //fmt.Println(paths)
// if len(paths) < 1 {
// continue
// }
// if len(paths) > 1 {
// childDir := joinPath(dstDir, joinPath(paths[0:len(paths)-1]...))
// //fmt.Println("creating", childDir)
// err = os.MkdirAll(childDir, 0777)
// if err != nil {
// return err
// }
// }
// rc, err := f.Open()
// if err != nil {
// return err
// }
// newF, err := os.Create(path.Join(dstDir, joinPath(paths...)))
// if err == nil {
// _, err = io.Copy(newF, rc)
// }
// if err != nil {
// return err
// }
// rc.Close()
// }
// return nil
// }

140
doc/launchpad.go

@ -0,0 +1,140 @@
// Copyright 2013 gopm authors.
//
// Licensed under the Apache License, Version 2.0 (the "License"): you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.
package doc
import (
"archive/tar"
"bytes"
"compress/gzip"
"io"
"net/http"
"os"
//"path"
"regexp"
"strings"
)
var LaunchpadPattern = regexp.MustCompile(`^launchpad\.net/(?P<repo>(?P<project>[a-z0-9A-Z_.\-]+)(?P<series>/[a-z0-9A-Z_.\-]+)?|~[a-z0-9A-Z_.\-]+/(\+junk|[a-z0-9A-Z_.\-]+)/[a-z0-9A-Z_.\-]+)(?P<dir>/[a-z0-9A-Z_.\-/]+)*$`)
// GetLaunchpadDoc downloads tarball from launchpad.net.
func GetLaunchpadDoc(client *http.Client, match map[string]string, installRepoPath string, nod *Node, cmdFlags map[string]bool) ([]string, error) {
if match["project"] != "" && match["series"] != "" {
rc, err := httpGet(client, expand("https://code.launchpad.net/{project}{series}/.bzr/branch-format", match), nil)
switch {
case err == nil:
rc.Close()
// The structure of the import path is launchpad.net/{root}/{dir}.
case isNotFound(err):
// The structure of the import path is is launchpad.net/{project}/{dir}.
match["repo"] = match["project"]
match["dir"] = expand("{series}{dir}", match)
default:
return nil, err
}
}
var downloadPath string
// Check if download with specific revision.
if len(nod.Value) == 0 {
downloadPath = expand("https://bazaar.launchpad.net/+branch/{repo}/tarball", match)
} else {
downloadPath = expand("https://bazaar.launchpad.net/+branch/{repo}/tarball/"+nod.Value, match)
}
// Scrape the repo browser to find the project revision and individual Go files.
p, err := HttpGetBytes(client, downloadPath, nil)
if err != nil {
return nil, err
}
installPath := installRepoPath + "/" + nod.ImportPath
// Remove old files.
os.RemoveAll(installPath + "/")
os.MkdirAll(installPath+"/", os.ModePerm)
gzr, err := gzip.NewReader(bytes.NewReader(p))
if err != nil {
return nil, err
}
defer gzr.Close()
tr := tar.NewReader(gzr)
var autoPath string // Auto path is the root path that generated by bitbucket.org.
// Get source file data.
dirs := make([]string, 0, 5)
for {
h, err := tr.Next()
if err == io.EOF {
break
} else if err != nil {
return nil, err
}
fn := h.FileInfo().Name()
// Check root path.
if len(autoPath) == 0 {
autoPath = fn[:strings.Index(fn, match["repo"])+len(match["repo"])]
}
absPath := strings.Replace(fn, autoPath, installPath, 1)
switch {
case h.FileInfo().IsDir(): // Directory.
// Create diretory before create file.
os.MkdirAll(absPath+"/", os.ModePerm)
// Check if current directory is example.
if !(!cmdFlags["-e"] && strings.Contains(absPath, "example")) {
dirs = append(dirs, absPath)
}
case !strings.HasPrefix(fn, "."):
//os.MkdirAll(path.Dir(absPath)+"/", os.ModePerm)
// Get data from archive.
fbytes := make([]byte, h.Size)
if _, err := io.ReadFull(tr, fbytes); err != nil {
return nil, err
}
// Write data to file
fw, err := os.Create(absPath)
if err != nil {
return nil, err
}
_, err = fw.Write(fbytes)
fw.Close()
if err != nil {
return nil, err
}
}
}
var imports []string
// Check if need to check imports.
if nod.IsGetDeps {
for _, d := range dirs {
importPkgs, err := CheckImports(d+"/", match["importPath"])
if err != nil {
return nil, err
}
imports = append(imports, importPkgs...)
}
}
return imports, err
}

26
doc/struct.go

@ -16,7 +16,9 @@ package doc
import ( import (
"go/token" "go/token"
"net/http"
"os" "os"
"regexp"
"time" "time"
) )
@ -30,10 +32,11 @@ const (
) )
type Node struct { type Node struct {
ImportPath string ImportPath string
Type string DownloadURL string
Value string // Branch, tag or commit. Type string
IsGetDeps bool Value string // Branch, tag or commit.
IsGetDeps bool
} }
// source is source code file. // source is source code file.
@ -56,3 +59,18 @@ type walker struct {
srcs map[string]*source // Source files. srcs map[string]*source // Source files.
fset *token.FileSet fset *token.FileSet
} }
// service represents a source code control service.
type service struct {
pattern *regexp.Regexp
prefix string
get func(*http.Client, map[string]string, string, *Node, map[string]bool) ([]string, error)
}
// services is the list of source code control services handled by gopkgdoc.
var services = []*service{
{GithubPattern, "github.com/", GetGithubDoc},
{GooglePattern, "code.google.com/", GetGoogleDoc},
{BitbucketPattern, "bitbucket.org/", GetBitbucketDoc},
{LaunchpadPattern, "launchpad.net/", GetLaunchpadDoc},
}

150
doc/vcs.go

@ -16,8 +16,12 @@ package doc
import ( import (
"bytes" "bytes"
"encoding/xml"
"errors"
"io"
"io/ioutil" "io/ioutil"
"log" "log"
"net/http"
"os" "os"
"os/exec" "os/exec"
"path" "path"
@ -244,3 +248,149 @@ func CheckImports(absPath, importPath string) (importPkgs []string, err error) {
return importPkgs, err return importPkgs, err
} }
// PureDownload downloads package without version control.
func PureDownload(nod *Node, installRepoPath string, flags map[string]bool) ([]string, error) {
for _, s := range services {
if s.get == nil || !strings.HasPrefix(nod.DownloadURL, s.prefix) {
continue
}
m := s.pattern.FindStringSubmatch(nod.DownloadURL)
if m == nil {
if s.prefix != "" {
return nil, errors.New("Cannot match package service prefix by given path")
}
continue
}
match := map[string]string{"importPath": nod.DownloadURL}
for i, n := range s.pattern.SubexpNames() {
if n != "" {
match[n] = m[i]
}
}
return s.get(HttpClient, match, installRepoPath, nod, flags)
}
ColorLog("[TRAC] Cannot match any service, getting dynamic...\n")
return getDynamic(HttpClient, nod, installRepoPath, flags)
}
func getDynamic(client *http.Client, nod *Node, installRepoPath string, flags map[string]bool) ([]string, error) {
match, err := fetchMeta(client, nod.ImportPath)
if err != nil {
return nil, err
}
if match["projectRoot"] != nod.ImportPath {
rootMatch, err := fetchMeta(client, match["projectRoot"])
if err != nil {
return nil, err
}
if rootMatch["projectRoot"] != match["projectRoot"] {
return nil, NotFoundError{"Project root mismatch."}
}
}
nod.DownloadURL = expand("{repo}{dir}", match)
return PureDownload(nod, installRepoPath, flags)
}
func fetchMeta(client *http.Client, importPath string) (map[string]string, error) {
uri := importPath
if !strings.Contains(uri, "/") {
// Add slash for root of domain.
uri = uri + "/"
}
uri = uri + "?go-get=1"
scheme := "https"
resp, err := client.Get(scheme + "://" + uri)
if err != nil || resp.StatusCode != 200 {
if err == nil {
resp.Body.Close()
}
scheme = "http"
resp, err = client.Get(scheme + "://" + uri)
if err != nil {
return nil, &RemoteError{strings.SplitN(importPath, "/", 2)[0], err}
}
}
defer resp.Body.Close()
return parseMeta(scheme, importPath, resp.Body)
}
func attrValue(attrs []xml.Attr, name string) string {
for _, a := range attrs {
if strings.EqualFold(a.Name.Local, name) {
return a.Value
}
}
return ""
}
func parseMeta(scheme, importPath string, r io.Reader) (map[string]string, error) {
var match map[string]string
d := xml.NewDecoder(r)
d.Strict = false
metaScan:
for {
t, tokenErr := d.Token()
if tokenErr != nil {
break metaScan
}
switch t := t.(type) {
case xml.EndElement:
if strings.EqualFold(t.Name.Local, "head") {
break metaScan
}
case xml.StartElement:
if strings.EqualFold(t.Name.Local, "body") {
break metaScan
}
if !strings.EqualFold(t.Name.Local, "meta") ||
attrValue(t.Attr, "name") != "go-import" {
continue metaScan
}
f := strings.Fields(attrValue(t.Attr, "content"))
if len(f) != 3 ||
!strings.HasPrefix(importPath, f[0]) ||
!(len(importPath) == len(f[0]) || importPath[len(f[0])] == '/') {
continue metaScan
}
if match != nil {
return nil, NotFoundError{"More than one <meta> found at " + scheme + "://" + importPath}
}
projectRoot, vcs, repo := f[0], f[1], f[2]
repo = strings.TrimSuffix(repo, "."+vcs)
i := strings.Index(repo, "://")
if i < 0 {
return nil, NotFoundError{"Bad repo URL in <meta>."}
}
proto := repo[:i]
repo = repo[i+len("://"):]
match = map[string]string{
// Used in getVCSDoc, same as vcsPattern matches.
"importPath": importPath,
"repo": repo,
"vcs": vcs,
"dir": importPath[len(projectRoot):],
// Used in getVCSDoc
"scheme": proto,
// Used in getDynamic.
"projectRoot": projectRoot,
"projectName": path.Base(projectRoot),
"projectURL": scheme + "://" + projectRoot,
}
}
}
if match == nil {
return nil, NotFoundError{"<meta> not found."}
}
return match, nil
}

4
doc/walker.go

@ -16,7 +16,6 @@ package doc
import ( import (
"bytes" "bytes"
"errors"
"go/ast" "go/ast"
"go/build" "go/build"
"go/parser" "go/parser"
@ -114,7 +113,8 @@ func (w *walker) build(srcs []*source) ([]string, error) {
if nogo { if nogo {
err = nil err = nil
} else { } else {
return nil, errors.New("doc.walker.build(): " + err.Error()) ColorLog("[WARN] Error occurs when check imports[ %s ]\n", err)
return nil, nil
} }
} }

Loading…
Cancel
Save