diff --git a/cmd/cmd.go b/cmd/cmd.go index bf4176e04..19c96bb59 100644 --- a/cmd/cmd.go +++ b/cmd/cmd.go @@ -21,6 +21,7 @@ import ( ) var ( + AppPath string reposDir string = "~/.gopm/repos" ) diff --git a/cmd/get.go b/cmd/get.go index 2c710f7dd..96f5ffb7a 100644 --- a/cmd/get.go +++ b/cmd/get.go @@ -15,22 +15,21 @@ package cmd import ( - "archive/zip" "errors" "fmt" - "io" "net/http" - "os" "os/user" - "path" - "path/filepath" + //"path" + "regexp" "strings" - "../doc" + "github.com/gpmgo/gopm/doc" ) var ( - installGOPATH string // The GOPATH that packages are downloaded to. + installRepoPath string + downloadCache map[string]bool // Saves packages that have been downloaded. + downloadCount int ) var CmdGet = &Command{ @@ -68,10 +67,6 @@ func init() { } } -func isStandalone() bool { - return true -} - // printGetPrompt prints prompt information to users to // let them know what's going on. func printGetPrompt(flag string) { @@ -127,195 +122,304 @@ func runGet(cmd *Command, args []string) { return } - if len(args) > 0 { - var ver string = TRUNK - if len(args) == 2 { - ver = args[1] - } - pkg := NewPkg(args[0], ver) - if pkg == nil { - doc.ColorLog("[ERROR] Unrecognized package %v.\n", args[0]) - return - } - - if isStandalone() { - err := getDirect(pkg) - if err != nil { - doc.ColorLog("[ERROR] %v\n", err) - } else { - fmt.Println("done.") - } - } else { - fmt.Println("Not implemented.") - //getSource(pkgName) - } - } -} - -func dirExists(dir string) bool { - d, e := os.Stat(dir) - switch { - case e != nil: - return false - case !d.IsDir(): - return false - } - - return true -} - -func fileExists(dir string) bool { - info, err := os.Stat(dir) + curUser, err := user.Current() if err != nil { - return false + doc.ColorLog("[ERROR] Fail to get current user[ %s ]\n", err) + return } - return !info.IsDir() -} - -func joinPath(paths ...string) string { - if len(paths) < 1 { - return "" - } - res := "" - for _, p := range paths { - res = path.Join(res, p) - } - return res -} + installRepoPath = strings.Replace(reposDir, "~", curUser.HomeDir, -1) + doc.ColorLog("[INFO] Packages will be installed into( %s )\n", installRepoPath) -func download(url string, localfile string) error { - fmt.Println("Downloading", url, "...") - resp, err := http.Get(url) - if err != nil { - return err - } - defer resp.Body.Close() + nodes := []*doc.Node{} + // ver describles branch, tag or commit. + var t, ver string = doc.BRANCH, doc.TRUNK - localdir := filepath.Dir(localfile) - if !dirExists(localdir) { - err = os.MkdirAll(localdir, 0777) + if len(args) >= 2 { + t, ver, err = validPath(args[1]) if err != nil { - return err + doc.ColorLog("[ERROR] Fail to parse 'args'[ %s ]\n", err) + return } } - if !fileExists(localfile) { - f, err := os.Create(localfile) - if err == nil { - _, err = io.Copy(f, resp.Body) - } - if err != nil { - return err - } - } + nodes = append(nodes, &doc.Node{ + ImportPath: args[0], + Type: t, + Value: ver, + IsGetDeps: true, + }) - return nil -} + // Download package(s). + downloadPackages(nodes) -func extractPkg(pkg *Pkg, localfile string, update bool) error { - fmt.Println("Extracting package", pkg.Name, "...") + doc.ColorLog("[INFO] %d package(s) downloaded.\n", downloadCount) +} - gopath := os.Getenv("GOPATH") - var childDirs []string = strings.Split(pkg.Name, "/") +// downloadPackages downloads packages with certain commit, +// if the commit is empty string, then it downloads all dependencies, +// otherwise, it only downloada package with specific commit only. +func downloadPackages(nodes []*doc.Node) { + // Check all packages, they may be raw packages path. + for _, n := range nodes { + // Check if it is a valid remote path. + if doc.IsValidRemotePath(n.ImportPath) { + if !CmdGet.Flags["-u"] { + // Check if package has been downloaded. + if _, ok := doc.CheckIsExistInGOPATH(n.ImportPath); ok { + doc.ColorLog("[WARN] Skipped installed package( %s => %s:%s )\n", + n.ImportPath, n.Type, n.Value) + continue + } + } - if pkg.Ver != TRUNK { - childDirs[len(childDirs)-1] = fmt.Sprintf("%v_%v_%v", childDirs[len(childDirs)-1], pkg.Ver, pkg.VerId) - } - dstDir := joinPath(gopath, "src", joinPath(childDirs...)) - //fmt.Println(dstDir) - var err error - if !update { - if dirExists(dstDir) { - return nil - } - err = os.MkdirAll(dstDir, 0777) - } else { - if dirExists(dstDir) { - err = os.Remove(dstDir) + if !downloadCache[n.ImportPath] { + // Download package. + nod, imports := downloadPackage(n) + if len(imports) > 0 { + // Need to download dependencies. + // Generate temporary nodes. + nodes := make([]*doc.Node, len(imports)) + for i := range nodes { + nodes[i] = new(doc.Node) + nodes[i].ImportPath = imports[i] + } + downloadPackages(nodes) + } + + // Only save package information with specific commit. + if nod != nil { + // Save record in local nodes. + doc.ColorLog("[SUCC] Downloaded package( %s => %s:%s )\n", + n.ImportPath, n.Type, n.Value) + downloadCount++ + //saveNode(nod) + } + } else { + doc.ColorLog("[WARN] Skipped downloaded package( %s => %s:%s )\n", + n.ImportPath, n.Type, n.Value) + } } else { - err = os.MkdirAll(dstDir, 0777) + // Invalid import path. + doc.ColorLog("[WARN] Skipped invalid package path( %s => %s:%s )\n", + n.ImportPath, n.Type, n.Value) } } +} + +// downloadPackage downloads package either use version control tools or not. +func downloadPackage(nod *doc.Node) (*doc.Node, []string) { + // Mark as donwloaded. + downloadCache[nod.ImportPath] = true + + imports, err := pureDownload(nod) if err != nil { - return err + doc.ColorLog("[ERRO] Download falied[ %s ]\n", err) + return nil, nil } + return nod, imports +} - if path.Ext(localfile) != ".zip" { - return errors.New("Not implemented!") - } +// validPath checks if the information of the package is valid. +func validPath(info string) (string, string, error) { + infos := strings.Split(info, ":") - r, err := zip.OpenReader(localfile) - if err != nil { - return err + l := len(infos) + switch { + case l > 2: + return "", "", errors.New("Invalid information of package") + case l == 1: + return doc.BRANCH, doc.TRUNK, nil + case l == 2: + return infos[0], infos[1], nil + default: + return "", "", errors.New("Cannot match any case") } - defer r.Close() +} - for _, f := range r.File { - fmt.Printf("Contents of %s:\n", f.Name) - if f.FileInfo().IsDir() { - continue - } +// service represents a source code control service. +type service struct { + pattern *regexp.Regexp + prefix string + get func(*http.Client, map[string]string, string, *doc.Node, map[string]bool) ([]string, error) +} - paths := strings.Split(f.Name, "/")[1:] - //fmt.Println(paths) - if len(paths) < 1 { +// services is the list of source code control services handled by gopkgdoc. +var services = []*service{ + {doc.GithubPattern, "github.com/", doc.GetGithubDoc}, + // {doc.GooglePattern, "code.google.com/", doc.GetGoogleDoc}, + // {doc.BitbucketPattern, "bitbucket.org/", doc.GetBitbucketDoc}, + // {doc.LaunchpadPattern, "launchpad.net/", doc.GetLaunchpadDoc}, +} + +// pureDownload downloads package without version control. +func pureDownload(nod *doc.Node) ([]string, error) { + for _, s := range services { + if s.get == nil || !strings.HasPrefix(nod.ImportPath, s.prefix) { continue } - - if len(paths) > 1 { - childDir := joinPath(dstDir, joinPath(paths[0:len(paths)-1]...)) - //fmt.Println("creating", childDir) - err = os.MkdirAll(childDir, 0777) - if err != nil { - return err + m := s.pattern.FindStringSubmatch(nod.ImportPath) + if m == nil { + if s.prefix != "" { + return nil, errors.New("Cannot match package service prefix by given path") } + continue } - - rc, err := f.Open() - if err != nil { - return err - } - - newF, err := os.Create(path.Join(dstDir, joinPath(paths...))) - if err == nil { - _, err = io.Copy(newF, rc) - } - if err != nil { - return err + match := map[string]string{"importPath": nod.ImportPath} + for i, n := range s.pattern.SubexpNames() { + if n != "" { + match[n] = m[i] + } } - rc.Close() - } - return nil -} - -func getPackage(pkg *Pkg, url string) error { - curUser, err := user.Current() - if err != nil { - return err + return s.get(doc.HttpClient, match, installRepoPath, nod, CmdGet.Flags) } - - reposDir = strings.Replace(reposDir, "~", curUser.HomeDir, -1) - localdir := path.Join(reposDir, pkg.Name) - localdir, err = filepath.Abs(localdir) - if err != nil { - return err - } - - localfile := path.Join(localdir, pkg.FileName()) - - err = download(url, localfile) - if err != nil { - return err - } - - return extractPkg(pkg, localfile, false) + return nil, errors.New("Cannot match any package service by given path") } -func getDirect(pkg *Pkg) error { - return getPackage(pkg, pkg.Url()) -} +// func joinPath(paths ...string) string { +// if len(paths) < 1 { +// return "" +// } +// res := "" +// for _, p := range paths { +// res = path.Join(res, p) +// } +// return res +// } + +// func download(url string, localfile string) error { +// fmt.Println("Downloading", url, "...") +// resp, err := http.Get(url) +// if err != nil { +// return err +// } +// defer resp.Body.Close() + +// localdir := filepath.Dir(localfile) +// if !dirExists(localdir) { +// err = os.MkdirAll(localdir, 0777) +// if err != nil { +// return err +// } +// } + +// if !fileExists(localfile) { +// f, err := os.Create(localfile) +// if err == nil { +// _, err = io.Copy(f, resp.Body) +// } +// if err != nil { +// return err +// } +// } + +// return nil +// } + +// func extractPkg(pkg *Pkg, localfile string, update bool) error { +// fmt.Println("Extracting package", pkg.Name, "...") + +// gopath := os.Getenv("GOPATH") +// var childDirs []string = strings.Split(pkg.Name, "/") + +// if pkg.Ver != TRUNK { +// childDirs[len(childDirs)-1] = fmt.Sprintf("%v_%v_%v", childDirs[len(childDirs)-1], pkg.Ver, pkg.VerId) +// } +// dstDir := joinPath(gopath, "src", joinPath(childDirs...)) +// //fmt.Println(dstDir) +// var err error +// if !update { +// if dirExists(dstDir) { +// return nil +// } +// err = os.MkdirAll(dstDir, 0777) +// } else { +// if dirExists(dstDir) { +// err = os.Remove(dstDir) +// } else { +// err = os.MkdirAll(dstDir, 0777) +// } +// } + +// if err != nil { +// return err +// } + +// if path.Ext(localfile) != ".zip" { +// return errors.New("Not implemented!") +// } + +// r, err := zip.OpenReader(localfile) +// if err != nil { +// return err +// } +// defer r.Close() + +// for _, f := range r.File { +// fmt.Printf("Contents of %s:\n", f.Name) +// if f.FileInfo().IsDir() { +// continue +// } + +// paths := strings.Split(f.Name, "/")[1:] +// //fmt.Println(paths) +// if len(paths) < 1 { +// continue +// } + +// if len(paths) > 1 { +// childDir := joinPath(dstDir, joinPath(paths[0:len(paths)-1]...)) +// //fmt.Println("creating", childDir) +// err = os.MkdirAll(childDir, 0777) +// if err != nil { +// return err +// } +// } + +// rc, err := f.Open() +// if err != nil { +// return err +// } + +// newF, err := os.Create(path.Join(dstDir, joinPath(paths...))) +// if err == nil { +// _, err = io.Copy(newF, rc) +// } +// if err != nil { +// return err +// } +// rc.Close() +// } +// return nil +// } + +// func getPackage(pkg *Pkg, url string) error { +// curUser, err := user.Current() +// if err != nil { +// return err +// } + +// reposDir = strings.Replace(reposDir, "~", curUser.HomeDir, -1) +// localdir := path.Join(reposDir, pkg.Name) +// localdir, err = filepath.Abs(localdir) +// if err != nil { +// return err +// } + +// localfile := path.Join(localdir, pkg.FileName()) + +// err = download(url, localfile) +// if err != nil { +// return err +// } + +// return extractPkg(pkg, localfile, false) +// } + +// func getDirect(pkg *Pkg) error { +// return getPackage(pkg, pkg.Url()) +// } /*func getFromSource(pkgName string, ver string, source string) error { urlTempl := "https://%v/%v" diff --git a/cmd/search.go b/cmd/search.go index 58864f449..dd30e8c44 100644 --- a/cmd/search.go +++ b/cmd/search.go @@ -15,11 +15,12 @@ package cmd import ( - "../doc" "encoding/json" "fmt" "io/ioutil" "net/http" + + "github.com/gpmgo/gopm/doc" ) var CmdSearch = &Command{ diff --git a/cmd/serve.go b/cmd/serve.go deleted file mode 100644 index aad54f252..000000000 --- a/cmd/serve.go +++ /dev/null @@ -1,394 +0,0 @@ -package cmd - -import ( - "../doc" - "fmt" - "github.com/syndtr/goleveldb/leveldb" - "github.com/syndtr/goleveldb/leveldb/errors" - "github.com/syndtr/goleveldb/leveldb/opt" - "io/ioutil" - "net/http" - "os" - "os/user" - "strconv" - "strings" -) - -var ( - dbDir = "~/.gopm/db" -) - -const ( - STOP = iota - LOCALRUN - RUNNING -) - -var CmdServe = &Command{ - UsageLine: "serve [:port]", - Short: "serve for package search", - Long: ` - serve provide a web service to search packages, download packages - -The serve flags are: - - -l - only service for localhost ip -`, -} - -func init() { - CmdServe.Run = runServe - CmdServe.Flags = map[string]bool{ - "-l": false, - } -} - -func printServePrompt(flag string) { - switch flag { - case "-l": - doc.ColorLog("[INFO] You enabled start a service only localhost.\n") - } -} - -// Not implemented -func autoPort() string { - return "8991" -} - -// search packages -func runServe(cmd *Command, args []string) { - // Check flags. - num := checkFlags(cmd.Flags, args, printServePrompt) - if num == -1 { - return - } - args = args[num:] - - var listen string - var port string - if cmd.Flags["-l"] { - listen += "127.0.0.1:" - port = autoPort() - } else { - listen += "0.0.0.0:" - port = "8991" - } - - // Check length of arguments. - if len(args) >= 1 { - port = args[0] - } - - startService(listen + port) -} - -func splitWord(word string, res *map[string]bool) { - for i, _ := range word { - for j, _ := range word[i:] { - w := word[i : i+j+1] - (*res)[w] = true - } - } - return -} - -func splitPkgName(pkgName string) (res map[string]bool) { - //var src string - ps := strings.Split(pkgName, "/") - if len(ps) > 1 { - ps = ps[1:] - } - - res = make(map[string]bool, 0) - res[strings.Join(ps, "/")] = true - for _, w := range ps { - splitWord(w, &res) - } - return -} - -var ( - ro *opt.ReadOptions = &opt.ReadOptions{} - wo *opt.WriteOptions = &opt.WriteOptions{} -) - -func dbGet(key string) (string, error) { - v, err := db.Get([]byte(key), ro) - return string(v), err -} - -func dbPut(key string, value string) error { - fmt.Println("put ", key, ": ", value) - return db.Put([]byte(key), []byte(value), wo) -} - -func batchPut(batch *leveldb.Batch, key string, value string) error { - fmt.Println("put ", key, ": ", value) - batch.Put([]byte(key), []byte(value)) - return nil -} - -func addPkg(pkg *Pkg) error { - batch := new(leveldb.Batch) - strLastId, err := dbGet("lastId") - if err != nil { - if err == errors.ErrNotFound { - strLastId = "0" - err = batchPut(batch, "lastId", strLastId) - } else { - return err - } - } - if err != nil { - return err - } - - fmt.Println("last id is ", strLastId) - - lastId, err := strconv.ParseInt(strLastId, 0, 64) - if err != nil { - return err - } - - pkgKey := fmt.Sprintf("index:%v", pkg.Name) - - id, err := dbGet(pkgKey) - if err != nil { - if err == errors.ErrNotFound { - id = fmt.Sprintf("%v", lastId+1) - fmt.Println(id) - err = batchPut(batch, "lastId", id) - if err == nil { - err = batchPut(batch, pkgKey, id) - } - if err == nil { - err = batchPut(batch, "pkg:"+id, pkg.Name) - } - total, err := dbGet("total") - if err != nil { - if err == errors.ErrNotFound { - total = "1" - } else { - return err - } - } else { - totalInt, err := strconv.ParseInt(total, 0, 64) - if err != nil { - return err - } - totalInt = totalInt + 1 - total = fmt.Sprintf("%v", totalInt) - } - - err = batchPut(batch, "total", total) - } else { - return err - } - } - - if err != nil { - return err - } - - vers, err := dbGet("ver:" + id) - needSplit := (err == errors.ErrNotFound) - if err != nil { - if err != errors.ErrNotFound { - return err - } - } else { - return nil - } - - if vers == "" { - fmt.Println(pkg) - vers = pkg.VerString() - } else { - if !strings.Contains(vers, pkg.VerString()) { - vers = vers + "," + pkg.VerString() - } else { - return nil - } - } - - err = batchPut(batch, "ver:"+id, vers) - if err != nil { - return err - } - - if !needSplit { - return nil - } - - keys := splitPkgName(pkg.Name) - - for key, _ := range keys { - err = batchPut(batch, fmt.Sprintf("key:%v:%v", key, id), "") - if err != nil { - return err - } - } - - return db.Write(batch, wo) -} - -func rmPkg(pkg *Pkg) { - -} - -var db *leveldb.DB - -// service should be run -func autoRun() { - s, _, _ := runningStatus() - if s == STOP { - os.StartProcess("gopm", []string{"serve", "-l"}, nil) - } -} - -func runningStatus() (int, int, int) { - contentByte, err := ioutil.ReadFile("~/.gopm/var/pid") - if err != nil { - return STOP, 0, 0 - } - content := string(contentByte) - if len(content) < 0 || !strings.Contains(content, ",") { - return STOP, 0, 0 - } - cs := strings.Split(string(content), ",") - if len(cs) != 3 { - return STOP, 0, 0 - } - status, err := strconv.Atoi(cs[0]) - if err != nil { - return STOP, 0, 0 - } - if status < STOP || status > RUNNING { - return STOP, 0, 0 - } - pid, err := strconv.Atoi(cs[1]) - if err != nil { - return STOP, 0, 0 - } - - _, err = os.FindProcess(pid) - if err != nil { - return STOP, 0, 0 - } - - port, err := strconv.Atoi(cs[2]) - if err != nil { - return STOP, 0, 0 - } - - return status, pid, port -} - -func startService(listen string) { - // check the pre serve's type - curUser, err := user.Current() - if err != nil { - fmt.Println(err) - return - } - - dbDir = strings.Replace(dbDir, "~", curUser.HomeDir, -1) - - db, err = leveldb.OpenFile(dbDir, &opt.Options{Flag: opt.OFCreateIfMissing}) - if err != nil { - fmt.Println(err) - return - } - defer db.Close() - - // these handlers should only access by localhost - http.HandleFunc("/add", addHandler) - http.HandleFunc("/rm", rmHandler) - - // these handlers can be accessed according listen's ip - http.HandleFunc("/search", searchHandler) - http.HandleFunc("/searche", searcheHandler) - http.ListenAndServe(listen, nil) -} - -func searchHandler(w http.ResponseWriter, r *http.Request) { - r.ParseForm() - ids := make(map[string]bool) - for key, _ := range r.Form { - iter := db.NewIterator(ro) - rkey := fmt.Sprintf("key:%v:", key) - if iter.Seek([]byte(rkey)) { - k := iter.Key() - if !strings.HasPrefix(string(k), rkey) { - break - } else { - ids[string(k)] = true - } - } - for iter.Next() { - k := iter.Key() - if !strings.HasPrefix(string(k), rkey) { - break - } - ids[string(k)] = true - } - } - - pkgs := make([]string, 0) - - for id, _ := range ids { - idkeys := strings.SplitN(id, ":", -1) - rId := idkeys[len(idkeys)-1] - fmt.Println(rId) - pkg, err := dbGet(fmt.Sprintf("pkg:%v", rId)) - if err != nil { - doc.ColorLog(err.Error()) - continue - } - pkgs = append(pkgs, pkg) - } - - w.Write([]byte("[\"" + strings.Join(pkgs, "\", \"") + "\"]")) -} - -func searcheHandler(w http.ResponseWriter, r *http.Request) { - //if r.Method == "POST" { - r.ParseForm() - pkgs := make([]string, 0) - for key, _ := range r.Form { - _, err := dbGet("index:" + key) - - if err != nil { - doc.ColorLog(err.Error()) - continue - } - - pkgs = append(pkgs, key) - } - - w.Write([]byte("[\"" + strings.Join(pkgs, "\", \"") + "\"]")) - //} -} - -func addHandler(w http.ResponseWriter, r *http.Request) { - //if r.Method == "POST" { - r.ParseForm() - for key, _ := range r.Form { - fmt.Println(key) - pkg := NewPkg(key, "") - if pkg != nil { - err := addPkg(pkg) - if err != nil { - fmt.Println(err) - } - } else { - fmt.Println(key) - } - } - //} -} - -func rmHandler(w http.ResponseWriter, r *http.Request) { - -} diff --git a/cmd/service.go b/cmd/service.go deleted file mode 100644 index f50630cc0..000000000 --- a/cmd/service.go +++ /dev/null @@ -1,245 +0,0 @@ -// Copyright 2013 gopm authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"): you may -// not use this file except in compliance with the License. You may obtain -// a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -// License for the specific language governing permissions and limitations -// under the License. - -package cmd - -import ( - //"errors" - "fmt" - "strings" -) - -const ( - TRUNK = "trunk" - TAG = "tag" - BRANCH = "branch" - COMMIT = "commit" -) - -var ( - downloadCache map[string]bool // Saves packages that have been downloaded. - services []Service = []Service{ - &GithubService{}, - &GitOscService{}, - &BitBucketService{}, - //&GitCafeService{}, - //&CodeCSDNSource{}, - } -) - -func getService(pkgName string) Service { - for _, service := range services { - if service.HasPkg(pkgName) { - return service - } - } - return nil -} - -type Service interface { - PkgUrl(pkg *Pkg) string - HasPkg(pkgName string) bool - PkgExt() string -} - -type Pkg struct { - Service Service - Name string - Ver string - VerId string -} - -func (p *Pkg) VerSimpleString() string { - if p.VerId != "" { - return p.VerId - } - return p.Ver -} - -func (p *Pkg) VerString() string { - if p.VerId == "" { - return p.Ver - } - return fmt.Sprintf("%v:%v", p.Ver, p.VerId) -} - -func (p *Pkg) Url() string { - return p.Service.PkgUrl(p) -} - -func (p *Pkg) FileName() string { - return fmt.Sprintf("%v.%v", p.VerSimpleString(), p.Service.PkgExt()) -} - -func NewPkg(pkgName string, ver string) *Pkg { - vers := strings.Split(ver, ":") - if len(vers) > 2 { - return nil - } - - var verId string - if len(vers) == 2 { - verId = vers[1] - } - - if len(vers) == 1 { - vers[0] = TRUNK - } - - service := getService(pkgName) - if service == nil { - return nil - } - - return &Pkg{service, pkgName, vers[0], verId} -} - -// github repository -type GithubService struct { -} - -func (s *GithubService) PkgUrl(pkg *Pkg) string { - var verPath string - if pkg.Ver == TRUNK { - verPath = "master" - } else { - verPath = pkg.VerId - } - return fmt.Sprintf("https://%v/archive/%v.zip", pkg.Name, verPath) -} - -func (s *GithubService) HasPkg(pkgName string) bool { - return strings.HasPrefix(pkgName, "github.com") -} - -func (s *GithubService) PkgExt() string { - return "zip" -} - -// git osc repos -type GitOscService struct { -} - -func (s *GitOscService) PkgUrl(pkg *Pkg) string { - var verPath string - if pkg.Ver == TRUNK { - verPath = "master" - } else { - verPath = pkg.VerId - } - return fmt.Sprintf("https://%v/repository/archive?ref=%v", pkg.Name, verPath) -} - -func (s *GitOscService) HasPkg(pkgName string) bool { - return strings.HasPrefix(pkgName, "git.oschina.net") -} - -func (s *GitOscService) PkgExt() string { - return "zip" -} - -// bitbucket.org -type BitBucketService struct { -} - -func (s *BitBucketService) PkgUrl(pkg *Pkg) string { - var verPath string - if pkg.Ver == TRUNK { - verPath = "default" - } else { - verPath = pkg.VerId - } - - return fmt.Sprintf("https://%v/get/%v.zip", pkg.Name, verPath) -} - -func (s *BitBucketService) HasPkg(pkgName string) bool { - return strings.HasPrefix(pkgName, "bitbucket.org") -} - -func (s *BitBucketService) PkgExt() string { - return "zip" -} - -type GitCafeService struct { -} - -func (s *GitCafeService) PkgUrl(pkg *Pkg) string { - var verPath string - if pkg.Ver == TRUNK { - verPath = "master" - } else { - verPath = pkg.VerId - } - - return fmt.Sprintf("https://%v/tarball/%v", pkg.Name, verPath) -} - -func (s *GitCafeService) HasPkg(pkgName string) bool { - return strings.HasPrefix(pkgName, "gitcafe.com") -} - -func (s *GitCafeService) PkgExt() string { - return "tar.gz" -} - -// git lab repos, not completed -type GitLabService struct { - DomainOrIp string - Username string - Passwd string - PrivateKey string -} - -func (s *GitLabService) PkgUrl(pkg *Pkg) string { - var verPath string - if pkg.Ver == TRUNK { - verPath = "master" - } else { - verPath = pkg.VerId - } - - return fmt.Sprintf("https://%v/repository/archive/%v", pkg.Name, verPath) -} - -func (s *GitLabService) HasPkg(pkgName string) bool { - return strings.HasPrefix(pkgName, s.DomainOrIp) -} - -func (s *GitLabService) PkgExt() string { - return "tar.gz" -} - -// code.csdn.net -type CodeCSDNService struct { -} - -func (s *CodeCSDNService) PkgUrl(pkg *Pkg) string { - var verPath string - if pkg.Ver == TRUNK { - verPath = "master" - } else { - verPath = pkg.VerId - } - - return fmt.Sprintf("https://%v/repository/archive?ref=%v", pkg.Name, verPath) -} - -func (s *CodeCSDNService) HasPkg(pkgName string) bool { - return strings.HasPrefix(pkgName, "code.csdn.net") -} - -func (s *CodeCSDNService) PkgExt() string { - return "zip" -} diff --git a/doc/error.go b/doc/error.go new file mode 100644 index 000000000..3c6ea8406 --- /dev/null +++ b/doc/error.go @@ -0,0 +1,47 @@ +// Copyright 2013 gopm authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"): you may +// not use this file except in compliance with the License. You may obtain +// a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations +// under the License. + +package doc + +import ( + "errors" +) + +var ( + errNotModified = errors.New("Package not modified") + errNoMatch = errors.New("no match") + errUpdateTimeout = errors.New("update timeout") +) + +type NotFoundError struct { + Message string +} + +func (e NotFoundError) Error() string { + return e.Message +} + +func isNotFound(err error) bool { + _, ok := err.(NotFoundError) + return ok +} + +type RemoteError struct { + Host string + err error +} + +func (e *RemoteError) Error() string { + return e.err.Error() +} diff --git a/doc/github.go b/doc/github.go new file mode 100644 index 000000000..fd025a489 --- /dev/null +++ b/doc/github.go @@ -0,0 +1,196 @@ +// Copyright 2013 gopm authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"): you may +// not use this file except in compliance with the License. You may obtain +// a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations +// under the License. + +package doc + +import ( + "archive/zip" + "bytes" + "errors" + "io" + "net/http" + "os" + "path" + "regexp" + "strings" +) + +var ( + githubRawHeader = http.Header{"Accept": {"application/vnd.github-blob.raw"}} + GithubPattern = regexp.MustCompile(`^github\.com/(?P[a-z0-9A-Z_.\-]+)/(?P[a-z0-9A-Z_.\-]+)(?P/[a-z0-9A-Z_.\-/]*)?$`) + githubCred string +) + +/*func SetGithubCredentials(id, secret string) { + //githubCred = "client_id=" + id + "&client_secret=" + secret +}*/ + +func SetGithubCredentials(token string) { + if len(token) > 0 { + githubCred = "access_token=" + token + } +} + +// GetGithubDoc downloads tarball from github.com. +func GetGithubDoc(client *http.Client, match map[string]string, installRepoPath string, nod *Node, cmdFlags map[string]bool) ([]string, error) { + match["cred"] = githubCred + + if nod.Type == BRANCH { + nod.Value = MASTER + match["sha"] = nod.Value + } + + ColorLog("[TRAC] Downloading package( %s => %s:%s )\n", + nod.ImportPath, nod.Type, nod.Value) + + // JSON struct for github.com. + var refs []*struct { + Ref string + Url string + Object struct { + Sha string + Type string + Url string + } + } + + if nod.IsGetDeps { + if nod.Type == COMMIT { + // Get up-to-date version. + err := httpGetJSON(client, expand("https://api.github.com/repos/{owner}/{repo}/git/refs?{cred}", match), &refs) + if err != nil { + return nil, err + } + + for _, ref := range refs { + if strings.HasPrefix(ref.Ref, "refs/heads/master") { + match["sha"] = ref.Object.Sha + break + } + } + + nod.Value = match["sha"] + } + } else { + // Check downlaod type. + switch nod.Type { + case TAG, COMMIT, BRANCH: + match["sha"] = nod.Value + default: + return nil, errors.New("Unknown node type: " + nod.Type) + } + } + + // We use .zip here. + // zip : https://github.com/{owner}/{repo}/archive/{sha}.zip + // tarball : https://github.com/{owner}/{repo}/tarball/{sha} + + // Downlaod archive. + p, err := httpGetBytes(client, expand("https://github.com/{owner}/{repo}/archive/{sha}.zip", match), nil) + if err != nil { + return nil, err + } + + shaName := expand("{repo}-{sha}", match) + if nod.Type == "tag" { + shaName = strings.Replace(shaName, "-v", "-", 1) + } + + projectPath := expand("github.com/{owner}/{repo}", match) + installPath := installRepoPath + "/" + projectPath + "." + nod.Value + nod.ImportPath = projectPath + + // Remove old files. + os.RemoveAll(installPath + "/") + // Create destination directory. + os.MkdirAll(installPath+"/", os.ModePerm) + + r, err := zip.NewReader(bytes.NewReader(p), int64(len(p))) + if err != nil { + return nil, err + } + + dirs := make([]string, 0, 5) + // Need to add root path because we cannot get from tarball. + dirs = append(dirs, installPath+"/") + for _, f := range r.File { + absPath := strings.Replace(f.FileInfo().Name(), shaName, installPath, 1) + // Create diretory before create file. + os.MkdirAll(path.Dir(absPath)+"/", os.ModePerm) + + compareDir: + switch { + case strings.HasSuffix(absPath, "/"): // Directory. + // Check if current directory is example. + if !(!cmdFlags["-e"] && strings.Contains(absPath, "example")) { + for _, d := range dirs { + if d == absPath { + break compareDir + } + } + dirs = append(dirs, absPath) + } + case !strings.HasPrefix(f.FileInfo().Name(), "."): + // Get file from archive. + rc, err := f.Open() + if err != nil { + return nil, err + } + + // Write data to file + fw, _ := os.Create(absPath) + if err != nil { + return nil, err + } + + _, err = io.Copy(fw, rc) + // Close files. + rc.Close() + fw.Close() + if err != nil { + return nil, err + } + + // Set modify time. + os.Chtimes(absPath, f.ModTime(), f.ModTime()) + } + } + + var imports []string + + // Check if need to check imports. + if nod.IsGetDeps { + for _, d := range dirs { + importPkgs, err := CheckImports(d, match["importPath"]) + if err != nil { + return nil, err + } + imports = append(imports, importPkgs...) + } + } + + /*fpath := appPath + "repo/tarballs/" + node.ImportPath + "-" + node.Value + ".zip" + // Save tarball. + if autoBackup && !utils.IsExist(fpath) { + os.MkdirAll(path.Dir(fpath)+"/", os.ModePerm) + f, err := os.Create(fpath) + if err != nil { + return nil, err + } + defer f.Close() + _, err = f.Write(p) + }*/ + + return imports, err +} diff --git a/doc/http.go b/doc/http.go new file mode 100644 index 000000000..7b307cd45 --- /dev/null +++ b/doc/http.go @@ -0,0 +1,150 @@ +// Copyright 2013 gopm authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"): you may +// not use this file except in compliance with the License. You may obtain +// a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations +// under the License. + +package doc + +import ( + "encoding/json" + "flag" + "fmt" + "io" + "io/ioutil" + "net" + "net/http" + "time" + + "github.com/astaxie/beego" +) + +var userAgent = "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/29.0.1541.0 Safari/537.36" + +var ( + dialTimeout = flag.Duration("dial_timeout", 10*time.Second, "Timeout for dialing an HTTP connection.") + requestTimeout = flag.Duration("request_timeout", 20*time.Second, "Time out for roundtripping an HTTP request.") +) + +func timeoutDial(network, addr string) (net.Conn, error) { + return net.DialTimeout(network, addr, *dialTimeout) +} + +type transport struct { + t http.Transport +} + +func (t *transport) RoundTrip(req *http.Request) (*http.Response, error) { + timer := time.AfterFunc(*requestTimeout, func() { + t.t.CancelRequest(req) + beego.Warn("Canceled request for %s", req.URL) + }) + defer timer.Stop() + resp, err := t.t.RoundTrip(req) + return resp, err +} + +var ( + httpTransport = &transport{t: http.Transport{Dial: timeoutDial, ResponseHeaderTimeout: *requestTimeout / 2}} + HttpClient = &http.Client{Transport: httpTransport} +) + +// httpGet gets the specified resource. ErrNotFound is returned if the server +// responds with status 404. +func httpGetBytes(client *http.Client, url string, header http.Header) ([]byte, error) { + rc, err := httpGet(client, url, header) + if err != nil { + return nil, err + } + p, err := ioutil.ReadAll(rc) + rc.Close() + return p, err +} + +// httpGet gets the specified resource. ErrNotFound is returned if the +// server responds with status 404. +func httpGet(client *http.Client, url string, header http.Header) (io.ReadCloser, error) { + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return nil, err + } + req.Header.Set("User-Agent", userAgent) + for k, vs := range header { + req.Header[k] = vs + } + resp, err := client.Do(req) + if err != nil { + return nil, &RemoteError{req.URL.Host, err} + } + if resp.StatusCode == 200 { + return resp.Body, nil + } + resp.Body.Close() + if resp.StatusCode == 404 { // 403 can be rate limit error. || resp.StatusCode == 403 { + err = NotFoundError{"Resource not found: " + url} + } else { + err = &RemoteError{req.URL.Host, fmt.Errorf("get %s -> %d", url, resp.StatusCode)} + } + return nil, err +} + +// fetchFiles fetches the source files specified by the rawURL field in parallel. +func fetchFiles(client *http.Client, files []*source, header http.Header) error { + ch := make(chan error, len(files)) + for i := range files { + go func(i int) { + req, err := http.NewRequest("GET", files[i].rawURL, nil) + if err != nil { + ch <- err + return + } + req.Header.Set("User-Agent", userAgent) + for k, vs := range header { + req.Header[k] = vs + } + resp, err := client.Do(req) + if err != nil { + ch <- &RemoteError{req.URL.Host, err} + return + } + defer resp.Body.Close() + if resp.StatusCode != 200 { + ch <- &RemoteError{req.URL.Host, fmt.Errorf("get %s -> %d", req.URL, resp.StatusCode)} + return + } + files[i].data, err = ioutil.ReadAll(resp.Body) + if err != nil { + ch <- &RemoteError{req.URL.Host, err} + return + } + ch <- nil + }(i) + } + for _ = range files { + if err := <-ch; err != nil { + return err + } + } + return nil +} + +func httpGetJSON(client *http.Client, url string, v interface{}) error { + rc, err := httpGet(client, url, nil) + if err != nil { + return err + } + defer rc.Close() + err = json.NewDecoder(rc).Decode(v) + if _, ok := err.(*json.SyntaxError); ok { + err = NotFoundError{"JSON syntax error at " + url} + } + return err +} diff --git a/doc/struct.go b/doc/struct.go new file mode 100644 index 000000000..16bb7a2ec --- /dev/null +++ b/doc/struct.go @@ -0,0 +1,57 @@ +// Copyright 2013 gopm authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"): you may +// not use this file except in compliance with the License. You may obtain +// a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations +// under the License. + +package doc + +import ( + "go/token" + "os" + "time" +) + +const ( + TRUNK = "trunk" + MASTER = "master" + TAG = "tag" + BRANCH = "branch" + COMMIT = "commit" +) + +type Node struct { + ImportPath string + Type string + Value string // Branch, tag or commit. + IsGetDeps bool +} + +// source is source code file. +type source struct { + rawURL string + name string + data []byte +} + +func (s *source) Name() string { return s.name } +func (s *source) Size() int64 { return int64(len(s.data)) } +func (s *source) Mode() os.FileMode { return 0 } +func (s *source) ModTime() time.Time { return time.Time{} } +func (s *source) IsDir() bool { return false } +func (s *source) Sys() interface{} { return nil } + +// walker holds the state used when building the documentation. +type walker struct { + ImportPath string + srcs map[string]*source // Source files. + fset *token.FileSet +} diff --git a/doc/utils.go b/doc/utils.go index bf2f74524..b08fa4f99 100644 --- a/doc/utils.go +++ b/doc/utils.go @@ -16,10 +16,19 @@ package doc import ( "fmt" + "os" + "path" + "regexp" "runtime" "strings" ) +// IsExist returns if a file or directory exists +func IsExist(path string) bool { + _, err := os.Stat(path) + return err == nil || os.IsExist(err) +} + const ( Gray = uint8(iota + 90) Red @@ -86,3 +95,611 @@ func getColorLevel(level string) string { return level } } + +// GetGOPATH returns all paths in GOPATH variable. +func GetGOPATH() []string { + gopath := os.Getenv("GOPATH") + var paths []string + if runtime.GOOS == "windows" { + gopath = strings.Replace(gopath, "\\", "/", -1) + paths = strings.Split(gopath, ";") + } else { + paths = strings.Split(gopath, ":") + } + return paths +} + +// GetGOPATH returns best matched GOPATH. +func GetBestMatchGOPATH(appPath string) string { + paths := GetGOPATH() + for _, p := range paths { + if strings.HasPrefix(p, appPath) { + return strings.Replace(p, "\\", "/", -1) + } + } + return paths[0] +} + +// GetDirsInfo returns os.FileInfo of all sub-directories in root path. +func GetDirsInfo(rootPath string) ([]os.FileInfo, error) { + rootDir, err := os.Open(rootPath) + if err != nil { + return nil, err + } + defer rootDir.Close() + + dirs, err := rootDir.Readdir(0) + if err != nil { + return nil, err + } + + return dirs, err +} + +// CheckIsExistWithVCS returns false if directory only has VCS folder, +// or doesn't exist. +func CheckIsExistWithVCS(path string) bool { + // Check if directory exist. + if !IsExist(path) { + return false + } + + // Check if only has VCS folder. + dirs, err := GetDirsInfo(path) + if err != nil { + ColorLog("[ERRO] CheckIsExistWithVCS -> [ %s ]\n", err) + return false + } + + if len(dirs) > 1 { + return true + } else if len(dirs) == 0 { + return false + } + + switch dirs[0].Name() { + case ".git", ".hg", ".svn": + return false + } + + return true +} + +// CheckIsExistInGOPATH checks if given package import path exists in any path in GOPATH/src, +// and returns corresponding GOPATH. +func CheckIsExistInGOPATH(importPath string) (string, bool) { + paths := GetGOPATH() + for _, p := range paths { + if CheckIsExistWithVCS(p + "/src/" + importPath + "/") { + return p, true + } + } + return "", false +} + +// GetProjectPath returns project path of import path. +func GetProjectPath(importPath string) (projectPath string) { + projectPath = importPath + + // Check project hosting. + switch { + case strings.HasPrefix(importPath, "github.com"): + projectPath = joinPath(importPath, 3) + case strings.HasPrefix(importPath, "code.google.com"): + projectPath = joinPath(importPath, 3) + case strings.HasPrefix(importPath, "bitbucket.org"): + projectPath = joinPath(importPath, 3) + case strings.HasPrefix(importPath, "launchpad.net"): + projectPath = joinPath(importPath, 2) + } + + return projectPath +} + +func joinPath(importPath string, num int) string { + subdirs := strings.Split(importPath, "/") + if len(subdirs) > num { + return strings.Join(subdirs[:num], "/") + } + return importPath +} + +var validTLD = map[string]bool{ + // curl http://data.iana.org/TLD/tlds-alpha-by-domain.txt | sed -e '/#/ d' -e 's/.*/"&": true,/' | tr [:upper:] [:lower:] + ".ac": true, + ".ad": true, + ".ae": true, + ".aero": true, + ".af": true, + ".ag": true, + ".ai": true, + ".al": true, + ".am": true, + ".an": true, + ".ao": true, + ".aq": true, + ".ar": true, + ".arpa": true, + ".as": true, + ".asia": true, + ".at": true, + ".au": true, + ".aw": true, + ".ax": true, + ".az": true, + ".ba": true, + ".bb": true, + ".bd": true, + ".be": true, + ".bf": true, + ".bg": true, + ".bh": true, + ".bi": true, + ".biz": true, + ".bj": true, + ".bm": true, + ".bn": true, + ".bo": true, + ".br": true, + ".bs": true, + ".bt": true, + ".bv": true, + ".bw": true, + ".by": true, + ".bz": true, + ".ca": true, + ".cat": true, + ".cc": true, + ".cd": true, + ".cf": true, + ".cg": true, + ".ch": true, + ".ci": true, + ".ck": true, + ".cl": true, + ".cm": true, + ".cn": true, + ".co": true, + ".com": true, + ".coop": true, + ".cr": true, + ".cu": true, + ".cv": true, + ".cw": true, + ".cx": true, + ".cy": true, + ".cz": true, + ".de": true, + ".dj": true, + ".dk": true, + ".dm": true, + ".do": true, + ".dz": true, + ".ec": true, + ".edu": true, + ".ee": true, + ".eg": true, + ".er": true, + ".es": true, + ".et": true, + ".eu": true, + ".fi": true, + ".fj": true, + ".fk": true, + ".fm": true, + ".fo": true, + ".fr": true, + ".ga": true, + ".gb": true, + ".gd": true, + ".ge": true, + ".gf": true, + ".gg": true, + ".gh": true, + ".gi": true, + ".gl": true, + ".gm": true, + ".gn": true, + ".gov": true, + ".gp": true, + ".gq": true, + ".gr": true, + ".gs": true, + ".gt": true, + ".gu": true, + ".gw": true, + ".gy": true, + ".hk": true, + ".hm": true, + ".hn": true, + ".hr": true, + ".ht": true, + ".hu": true, + ".id": true, + ".ie": true, + ".il": true, + ".im": true, + ".in": true, + ".info": true, + ".int": true, + ".io": true, + ".iq": true, + ".ir": true, + ".is": true, + ".it": true, + ".je": true, + ".jm": true, + ".jo": true, + ".jobs": true, + ".jp": true, + ".ke": true, + ".kg": true, + ".kh": true, + ".ki": true, + ".km": true, + ".kn": true, + ".kp": true, + ".kr": true, + ".kw": true, + ".ky": true, + ".kz": true, + ".la": true, + ".lb": true, + ".lc": true, + ".li": true, + ".lk": true, + ".lr": true, + ".ls": true, + ".lt": true, + ".lu": true, + ".lv": true, + ".ly": true, + ".ma": true, + ".mc": true, + ".md": true, + ".me": true, + ".mg": true, + ".mh": true, + ".mil": true, + ".mk": true, + ".ml": true, + ".mm": true, + ".mn": true, + ".mo": true, + ".mobi": true, + ".mp": true, + ".mq": true, + ".mr": true, + ".ms": true, + ".mt": true, + ".mu": true, + ".museum": true, + ".mv": true, + ".mw": true, + ".mx": true, + ".my": true, + ".mz": true, + ".na": true, + ".name": true, + ".nc": true, + ".ne": true, + ".net": true, + ".nf": true, + ".ng": true, + ".ni": true, + ".nl": true, + ".no": true, + ".np": true, + ".nr": true, + ".nu": true, + ".nz": true, + ".om": true, + ".org": true, + ".pa": true, + ".pe": true, + ".pf": true, + ".pg": true, + ".ph": true, + ".pk": true, + ".pl": true, + ".pm": true, + ".pn": true, + ".post": true, + ".pr": true, + ".pro": true, + ".ps": true, + ".pt": true, + ".pw": true, + ".py": true, + ".qa": true, + ".re": true, + ".ro": true, + ".rs": true, + ".ru": true, + ".rw": true, + ".sa": true, + ".sb": true, + ".sc": true, + ".sd": true, + ".se": true, + ".sg": true, + ".sh": true, + ".si": true, + ".sj": true, + ".sk": true, + ".sl": true, + ".sm": true, + ".sn": true, + ".so": true, + ".sr": true, + ".st": true, + ".su": true, + ".sv": true, + ".sx": true, + ".sy": true, + ".sz": true, + ".tc": true, + ".td": true, + ".tel": true, + ".tf": true, + ".tg": true, + ".th": true, + ".tj": true, + ".tk": true, + ".tl": true, + ".tm": true, + ".tn": true, + ".to": true, + ".tp": true, + ".tr": true, + ".travel": true, + ".tt": true, + ".tv": true, + ".tw": true, + ".tz": true, + ".ua": true, + ".ug": true, + ".uk": true, + ".us": true, + ".uy": true, + ".uz": true, + ".va": true, + ".vc": true, + ".ve": true, + ".vg": true, + ".vi": true, + ".vn": true, + ".vu": true, + ".wf": true, + ".ws": true, + ".xn--0zwm56d": true, + ".xn--11b5bs3a9aj6g": true, + ".xn--3e0b707e": true, + ".xn--45brj9c": true, + ".xn--80akhbyknj4f": true, + ".xn--80ao21a": true, + ".xn--90a3ac": true, + ".xn--9t4b11yi5a": true, + ".xn--clchc0ea0b2g2a9gcd": true, + ".xn--deba0ad": true, + ".xn--fiqs8s": true, + ".xn--fiqz9s": true, + ".xn--fpcrj9c3d": true, + ".xn--fzc2c9e2c": true, + ".xn--g6w251d": true, + ".xn--gecrj9c": true, + ".xn--h2brj9c": true, + ".xn--hgbk6aj7f53bba": true, + ".xn--hlcj6aya9esc7a": true, + ".xn--j6w193g": true, + ".xn--jxalpdlp": true, + ".xn--kgbechtv": true, + ".xn--kprw13d": true, + ".xn--kpry57d": true, + ".xn--lgbbat1ad8j": true, + ".xn--mgb9awbf": true, + ".xn--mgbaam7a8h": true, + ".xn--mgbayh7gpa": true, + ".xn--mgbbh1a71e": true, + ".xn--mgbc0a9azcg": true, + ".xn--mgberp4a5d4ar": true, + ".xn--mgbx4cd0ab": true, + ".xn--o3cw4h": true, + ".xn--ogbpf8fl": true, + ".xn--p1ai": true, + ".xn--pgbs0dh": true, + ".xn--s9brj9c": true, + ".xn--wgbh1c": true, + ".xn--wgbl6a": true, + ".xn--xkc2al3hye2a": true, + ".xn--xkc2dl3a5ee0h": true, + ".xn--yfro4i67o": true, + ".xn--ygbi2ammx": true, + ".xn--zckzah": true, + ".xxx": true, + ".ye": true, + ".yt": true, + ".za": true, + ".zm": true, + ".zw": true, +} + +var ( + validHost = regexp.MustCompile(`^[-a-z0-9]+(?:\.[-a-z0-9]+)+$`) + validPathElement = regexp.MustCompile(`^[-A-Za-z0-9~+][-A-Za-z0-9_.]*$`) +) + +// IsValidRemotePath returns true if importPath is structurally valid for "go get". +func IsValidRemotePath(importPath string) bool { + + parts := strings.Split(importPath, "/") + + if len(parts) <= 1 { + // Import path must contain at least one "/". + return false + } + + if !validTLD[path.Ext(parts[0])] { + return false + } + + if !validHost.MatchString(parts[0]) { + return false + } + for _, part := range parts[1:] { + if !validPathElement.MatchString(part) || part == "testdata" { + return false + } + } + + return true +} + +var standardPath = map[string]bool{ + "builtin": true, + + // go list -f '"{{.ImportPath}}": true,' std | grep -v 'cmd/|exp/' + "cmd/api": true, + "cmd/cgo": true, + "cmd/fix": true, + "cmd/go": true, + "cmd/godoc": true, + "cmd/gofmt": true, + "cmd/vet": true, + "cmd/yacc": true, + "archive/tar": true, + "archive/zip": true, + "bufio": true, + "bytes": true, + "compress/bzip2": true, + "compress/flate": true, + "compress/gzip": true, + "compress/lzw": true, + "compress/zlib": true, + "container/heap": true, + "container/list": true, + "container/ring": true, + "crypto": true, + "crypto/aes": true, + "crypto/cipher": true, + "crypto/des": true, + "crypto/dsa": true, + "crypto/ecdsa": true, + "crypto/elliptic": true, + "crypto/hmac": true, + "crypto/md5": true, + "crypto/rand": true, + "crypto/rc4": true, + "crypto/rsa": true, + "crypto/sha1": true, + "crypto/sha256": true, + "crypto/sha512": true, + "crypto/subtle": true, + "crypto/tls": true, + "crypto/x509": true, + "crypto/x509/pkix": true, + "database/sql": true, + "database/sql/driver": true, + "debug/dwarf": true, + "debug/elf": true, + "debug/gosym": true, + "debug/macho": true, + "debug/pe": true, + "encoding/ascii85": true, + "encoding/asn1": true, + "encoding/base32": true, + "encoding/base64": true, + "encoding/binary": true, + "encoding/csv": true, + "encoding/gob": true, + "encoding/hex": true, + "encoding/json": true, + "encoding/pem": true, + "encoding/xml": true, + "errors": true, + "expvar": true, + "flag": true, + "fmt": true, + "go/ast": true, + "go/build": true, + "go/doc": true, + "go/format": true, + "go/parser": true, + "go/printer": true, + "go/scanner": true, + "go/token": true, + "hash": true, + "hash/adler32": true, + "hash/crc32": true, + "hash/crc64": true, + "hash/fnv": true, + "html": true, + "html/template": true, + "image": true, + "image/color": true, + "image/draw": true, + "image/gif": true, + "image/jpeg": true, + "image/png": true, + "index/suffixarray": true, + "io": true, + "io/ioutil": true, + "log": true, + "log/syslog": true, + "math": true, + "math/big": true, + "math/cmplx": true, + "math/rand": true, + "mime": true, + "mime/multipart": true, + "net": true, + "net/http": true, + "net/http/cgi": true, + "net/http/cookiejar": true, + "net/http/fcgi": true, + "net/http/httptest": true, + "net/http/httputil": true, + "net/http/pprof": true, + "net/mail": true, + "net/rpc": true, + "net/rpc/jsonrpc": true, + "net/smtp": true, + "net/textproto": true, + "net/url": true, + "os": true, + "os/exec": true, + "os/signal": true, + "os/user": true, + "path": true, + "path/filepath": true, + "reflect": true, + "regexp": true, + "regexp/syntax": true, + "runtime": true, + "runtime/cgo": true, + "runtime/debug": true, + "runtime/pprof": true, + "sort": true, + "strconv": true, + "strings": true, + "sync": true, + "sync/atomic": true, + "syscall": true, + "testing": true, + "testing/iotest": true, + "testing/quick": true, + "text/scanner": true, + "text/tabwriter": true, + "text/template": true, + "text/template/parse": true, + "time": true, + "unicode": true, + "unicode/utf16": true, + "unicode/utf8": true, + "unsafe": true, +} + +// IsGoRepoPath returns true if package is from standard library. +func IsGoRepoPath(importPath string) bool { + return standardPath[importPath] +} diff --git a/doc/vcs.go b/doc/vcs.go new file mode 100644 index 000000000..1ab3231d2 --- /dev/null +++ b/doc/vcs.go @@ -0,0 +1,246 @@ +// Copyright 2013 gopm authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"): you may +// not use this file except in compliance with the License. You may obtain +// a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations +// under the License. + +package doc + +import ( + "bytes" + "io/ioutil" + "log" + "os" + "os/exec" + "path" + "regexp" + "strconv" + "strings" +) + +var ( + appPath string + autoBackup bool +) + +func SetAppConfig(path string, backup bool) { + appPath = path + autoBackup = backup +} + +// TODO: specify with command line flag +const repoRoot = "/tmp/gddo" + +var urlTemplates = []struct { + re *regexp.Regexp + template string + lineFmt string +}{ + { + regexp.MustCompile(`^git\.gitorious\.org/(?P[^/]+/[^/]+)$`), + "https://gitorious.org/{repo}/blobs/{tag}/{dir}{0}", + "#line%d", + }, + { + regexp.MustCompile(`^camlistore\.org/r/p/(?P[^/]+)$`), + "http://camlistore.org/code/?p={repo}.git;hb={tag};f={dir}{0}", + "#l%d", + }, +} + +// lookupURLTemplate finds an expand() template, match map and line number +// format for well known repositories. +func lookupURLTemplate(repo, dir, tag string) (string, map[string]string, string) { + if strings.HasPrefix(dir, "/") { + dir = dir[1:] + "/" + } + for _, t := range urlTemplates { + if m := t.re.FindStringSubmatch(repo); m != nil { + match := map[string]string{ + "dir": dir, + "tag": tag, + } + for i, name := range t.re.SubexpNames() { + if name != "" { + match[name] = m[i] + } + } + return t.template, match, t.lineFmt + } + } + return "", nil, "" +} + +type vcsCmd struct { + schemes []string + download func([]string, string, string) (string, string, error) +} + +var vcsCmds = map[string]*vcsCmd{ + "git": &vcsCmd{ + schemes: []string{"http", "https", "git"}, + download: downloadGit, + }, +} + +var lsremoteRe = regexp.MustCompile(`(?m)^([0-9a-f]{40})\s+refs/(?:tags|heads)/(.+)$`) + +func downloadGit(schemes []string, repo, savedEtag string) (string, string, error) { + var p []byte + var scheme string + for i := range schemes { + cmd := exec.Command("git", "ls-remote", "--heads", "--tags", schemes[i]+"://"+repo+".git") + log.Println(strings.Join(cmd.Args, " ")) + var err error + p, err = cmd.Output() + if err == nil { + scheme = schemes[i] + break + } + } + + if scheme == "" { + return "", "", NotFoundError{"VCS not found"} + } + + tags := make(map[string]string) + for _, m := range lsremoteRe.FindAllSubmatch(p, -1) { + tags[string(m[2])] = string(m[1]) + } + + tag, commit, err := bestTag(tags, "master") + if err != nil { + return "", "", err + } + + etag := scheme + "-" + commit + + if etag == savedEtag { + return "", "", errNotModified + } + + dir := path.Join(repoRoot, repo+".git") + p, err = ioutil.ReadFile(path.Join(dir, ".git/HEAD")) + switch { + case err != nil: + if err := os.MkdirAll(dir, 0777); err != nil { + return "", "", err + } + cmd := exec.Command("git", "clone", scheme+"://"+repo, dir) + log.Println(strings.Join(cmd.Args, " ")) + if err := cmd.Run(); err != nil { + return "", "", err + } + case string(bytes.TrimRight(p, "\n")) == commit: + return tag, etag, nil + default: + cmd := exec.Command("git", "fetch") + log.Println(strings.Join(cmd.Args, " ")) + cmd.Dir = dir + if err := cmd.Run(); err != nil { + return "", "", err + } + } + + cmd := exec.Command("git", "checkout", "--detach", "--force", commit) + cmd.Dir = dir + if err := cmd.Run(); err != nil { + return "", "", err + } + + return tag, etag, nil +} + +var defaultTags = map[string]string{"git": "master", "hg": "default"} + +func bestTag(tags map[string]string, defaultTag string) (string, string, error) { + if commit, ok := tags["go1"]; ok { + return "go1", commit, nil + } + if commit, ok := tags[defaultTag]; ok { + return defaultTag, commit, nil + } + return "", "", NotFoundError{"Tag or branch not found."} +} + +// expand replaces {k} in template with match[k] or subs[atoi(k)] if k is not in match. +func expand(template string, match map[string]string, subs ...string) string { + var p []byte + var i int + for { + i = strings.Index(template, "{") + if i < 0 { + break + } + p = append(p, template[:i]...) + template = template[i+1:] + i = strings.Index(template, "}") + if s, ok := match[template[:i]]; ok { + p = append(p, s...) + } else { + j, _ := strconv.Atoi(template[:i]) + p = append(p, subs[j]...) + } + template = template[i+1:] + } + p = append(p, template...) + return string(p) +} + +// checkImports checks package denpendencies. +func CheckImports(absPath, importPath string) (importPkgs []string, err error) { + dir, err := os.Open(absPath) + if err != nil { + return nil, err + } + defer dir.Close() + + // Get file info slice. + fis, err := dir.Readdir(0) + if err != nil { + return nil, err + } + + files := make([]*source, 0, 10) + for _, fi := range fis { + // Only handle files. + if strings.HasSuffix(fi.Name(), ".go") { + f, err := os.Open(absPath + fi.Name()) + if err != nil { + return nil, err + } + + fbytes := make([]byte, fi.Size()) + _, err = f.Read(fbytes) + f.Close() + + if err != nil { + return nil, err + } + + files = append(files, &source{ + name: fi.Name(), + data: fbytes, + }) + } + } + + // Check if has Go source files. + if len(files) > 0 { + w := &walker{ImportPath: importPath} + importPkgs, err = w.build(files) + if err != nil { + return nil, err + } + } + + return importPkgs, err +} diff --git a/doc/walker.go b/doc/walker.go new file mode 100644 index 000000000..0bc597e24 --- /dev/null +++ b/doc/walker.go @@ -0,0 +1,144 @@ +// Copyright 2013 gopm authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"): you may +// not use this file except in compliance with the License. You may obtain +// a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations +// under the License. + +package doc + +import ( + "bytes" + "errors" + "go/ast" + "go/build" + "go/parser" + "go/token" + "io" + "io/ioutil" + "os" + "path" + "runtime" + "strings" +) + +type sliceWriter struct{ p *[]byte } + +func (w sliceWriter) Write(p []byte) (int, error) { + *w.p = append(*w.p, p...) + return len(p), nil +} + +func (w *walker) readDir(dir string) ([]os.FileInfo, error) { + if dir != w.ImportPath { + panic("unexpected") + } + fis := make([]os.FileInfo, 0, len(w.srcs)) + for _, src := range w.srcs { + fis = append(fis, src) + } + return fis, nil +} + +func (w *walker) openFile(path string) (io.ReadCloser, error) { + if strings.HasPrefix(path, w.ImportPath+"/") { + if src, ok := w.srcs[path[len(w.ImportPath)+1:]]; ok { + return ioutil.NopCloser(bytes.NewReader(src.data)), nil + } + } + panic("unexpected") +} + +func simpleImporter(imports map[string]*ast.Object, path string) (*ast.Object, error) { + pkg := imports[path] + if pkg == nil { + // Guess the package name without importing it. Start with the last + // element of the path. + name := path[strings.LastIndex(path, "/")+1:] + + // Trim commonly used prefixes and suffixes containing illegal name + // runes. + name = strings.TrimSuffix(name, ".go") + name = strings.TrimSuffix(name, "-go") + name = strings.TrimPrefix(name, "go.") + name = strings.TrimPrefix(name, "go-") + name = strings.TrimPrefix(name, "biogo.") + + // It's also common for the last element of the path to contain an + // extra "go" prefix, but not always. TODO: examine unresolved ids to + // detect when trimming the "go" prefix is appropriate. + + pkg = ast.NewObj(ast.Pkg, name) + pkg.Data = ast.NewScope(nil) + imports[path] = pkg + } + return pkg, nil +} + +// build gets imports from source files. +func (w *walker) build(srcs []*source) ([]string, error) { + // Add source files to walker, I skipped references here. + w.srcs = make(map[string]*source) + for _, src := range srcs { + w.srcs[src.name] = src + } + + w.fset = token.NewFileSet() + + // Find the package and associated files. + ctxt := build.Context{ + GOOS: runtime.GOOS, + GOARCH: runtime.GOARCH, + CgoEnabled: true, + JoinPath: path.Join, + IsAbsPath: path.IsAbs, + SplitPathList: func(list string) []string { return strings.Split(list, ":") }, + IsDir: func(path string) bool { panic("unexpected") }, + HasSubdir: func(root, dir string) (rel string, ok bool) { panic("unexpected") }, + ReadDir: func(dir string) (fi []os.FileInfo, err error) { return w.readDir(dir) }, + OpenFile: func(path string) (r io.ReadCloser, err error) { return w.openFile(path) }, + Compiler: "gc", + } + + bpkg, err := ctxt.ImportDir(w.ImportPath, 0) + // Continue if there are no Go source files; we still want the directory info. + _, nogo := err.(*build.NoGoError) + if err != nil { + if nogo { + err = nil + } else { + return nil, errors.New("doc.walker.build(): " + err.Error()) + } + } + + // Parse the Go files + + files := make(map[string]*ast.File) + for _, name := range append(bpkg.GoFiles, bpkg.CgoFiles...) { + file, err := parser.ParseFile(w.fset, name, w.srcs[name].data, parser.ParseComments) + if err != nil { + //beego.Error("doc.walker.build():", err) + continue + } + files[name] = file + } + + w.ImportPath = strings.Replace(w.ImportPath, "\\", "/", -1) + var imports []string + for _, v := range bpkg.Imports { + // Skip strandard library. + if !IsGoRepoPath(v) && + (GetProjectPath(v) != GetProjectPath(w.ImportPath)) { + imports = append(imports, v) + } + } + + return imports, err +} diff --git a/gopm.go b/gopm.go index 2f774535a..d46ad6f87 100644 --- a/gopm.go +++ b/gopm.go @@ -19,6 +19,9 @@ import ( "fmt" "io" "os" + "os/exec" + "path" + "path/filepath" "runtime" "strings" "sync" @@ -26,14 +29,15 @@ import ( "unicode" "unicode/utf8" - "./cmd" + "github.com/gpmgo/gopm/cmd" + "github.com/gpmgo/gopm/doc" ) // +build go1.1 // Test that go1.1 tag above is included in builds. main.go refers to this definition. const go11tag = true -const APP_VER = "0.1.0.0813" +const APP_VER = "0.1.0.0818" var ( config map[string]interface{} @@ -44,7 +48,7 @@ var ( var commands = []*cmd.Command{ cmd.CmdGet, cmd.CmdSearch, - cmd.CmdServe, + //cmd.CmdServe, /* cmdBuild, cmdClean, @@ -67,10 +71,46 @@ var commands = []*cmd.Command{ helpTestfunc,*/ } +// getAppPath returns application execute path for current process. +func getAppPath() bool { + // Look up executable in PATH variable. + cmd.AppPath, _ = exec.LookPath(path.Base(os.Args[0])) + // Check if run under $GOPATH/bin. + if !doc.IsExist(cmd.AppPath + "docs/") { + paths := doc.GetGOPATH() + for _, p := range paths { + if doc.IsExist(p + "/src/github.com/gpmgoo/gopm/") { + cmd.AppPath = p + "/src/github.com/gpmgo/gopm/" + break + } + } + } + + if len(cmd.AppPath) == 0 { + doc.ColorLog("[ERRO] Cannot assign 'AppPath'[ %s ]\n", + "Unable to indicate current execute path") + return false + } + + cmd.AppPath = filepath.Dir(cmd.AppPath) + "/" + if runtime.GOOS == "windows" { + // Replace all '\' to '/'. + cmd.AppPath = strings.Replace(cmd.AppPath, "\\", "/", -1) + } + + return true +} + // We don't use init() to initialize // bacause we need to get execute path in runtime. func initialize() bool { runtime.GOMAXPROCS(runtime.NumCPU()) + + // Get application execute path. + if !getAppPath() { + return false + } + return true }