diff --git a/doc/bitbucket.go b/doc/bitbucket.go index 752613c93..5b0e537a0 100644 --- a/doc/bitbucket.go +++ b/doc/bitbucket.go @@ -25,6 +25,8 @@ import ( "path" "regexp" "strings" + + "github.com/Unknwon/com" ) var ( @@ -41,7 +43,7 @@ func getBitbucketDoc(client *http.Client, match map[string]string, installRepoPa var repo struct { Scm string } - if err := httpGetJSON(client, expand("https://api.bitbucket.org/1.0/repositories/{owner}/{repo}", match), &repo); err != nil { + if err := com.HttpGetJSON(client, expand("https://api.bitbucket.org/1.0/repositories/{owner}/{repo}", match), &repo); err != nil { return nil, err } match["vcs"] = repo.Scm @@ -62,7 +64,7 @@ func getBitbucketDoc(client *http.Client, match map[string]string, installRepoPa var nodes map[string]struct { Node string } - if err := httpGetJSON(client, expand("https://api.bitbucket.org/1.0/repositories/{owner}/{repo}/{0}", match, nodeType), &nodes); err != nil { + if err := com.HttpGetJSON(client, expand("https://api.bitbucket.org/1.0/repositories/{owner}/{repo}/{0}", match, nodeType), &nodes); err != nil { return nil, err } for t, n := range nodes { @@ -94,7 +96,7 @@ func getBitbucketDoc(client *http.Client, match map[string]string, installRepoPa // tarball : https://bitbucket.org/{owner}/{repo}/get/{commit}.tar.gz // Downlaod archive. - p, err := HttpGetBytes(client, expand("https://bitbucket.org/{owner}/{repo}/get/{commit}.tar.gz", match), nil) + p, err := com.HttpGetBytes(client, expand("https://bitbucket.org/{owner}/{repo}/get/{commit}.tar.gz", match), nil) if err != nil { return nil, err } diff --git a/doc/error.go b/doc/error.go index 3c6ea8406..56890c961 100644 --- a/doc/error.go +++ b/doc/error.go @@ -23,25 +23,3 @@ var ( errNoMatch = errors.New("no match") errUpdateTimeout = errors.New("update timeout") ) - -type NotFoundError struct { - Message string -} - -func (e NotFoundError) Error() string { - return e.Message -} - -func isNotFound(err error) bool { - _, ok := err.(NotFoundError) - return ok -} - -type RemoteError struct { - Host string - err error -} - -func (e *RemoteError) Error() string { - return e.err.Error() -} diff --git a/doc/github.go b/doc/github.go index fe968335f..7ab20ea8a 100644 --- a/doc/github.go +++ b/doc/github.go @@ -24,6 +24,8 @@ import ( "path" "regexp" "strings" + + "github.com/Unknwon/com" ) var ( @@ -65,7 +67,7 @@ func getGithubDoc(client *http.Client, match map[string]string, installRepoPath // tarball: https://github.com/{owner}/{repo}/tarball/{sha} // Downlaod archive. - p, err := HttpGetBytes(client, expand("https://github.com/{owner}/{repo}/archive/{sha}.zip", match), nil) + p, err := com.HttpGetBytes(client, expand("https://github.com/{owner}/{repo}/archive/{sha}.zip", match), nil) if err != nil { return nil, errors.New("Fail to donwload Github repo -> " + err.Error()) } diff --git a/doc/google.go b/doc/google.go index 97877aa41..e9e3000f7 100644 --- a/doc/google.go +++ b/doc/google.go @@ -19,58 +19,20 @@ import ( "os" "path" "regexp" - "strings" + + "github.com/Unknwon/com" + "github.com/Unknwon/ctw/packer" ) var ( - googleRepoRe = regexp.MustCompile(`id="checkoutcmd">(hg|git|svn)`) - googleRevisionRe = regexp.MustCompile(`

(?:[^ ]+ - )?Revision *([^:]+):`) - googleEtagRe = regexp.MustCompile(`^(hg|git|svn)-`) - googleFileRe = regexp.MustCompile(`
  • [a-z0-9\-]+)(:?\.(?P[a-z0-9\-]+))?(?P/[a-z0-9A-Z_.\-/]+)?$`) + googlePattern = regexp.MustCompile(`^code\.google\.com/p/(?P[a-z0-9\-]+)(:?\.(?P[a-z0-9\-]+))?(?P/[a-z0-9A-Z_.\-/]+)?$`) ) -func setupGoogleMatch(match map[string]string) { - if s := match["subrepo"]; s != "" { - match["dot"] = "." - match["query"] = "?repo=" + s - } else { - match["dot"] = "" - match["query"] = "" - } -} - -func getGoogleVCS(client *http.Client, match map[string]string) error { - // Scrape the HTML project page to find the VCS. - p, err := HttpGetBytes(client, expand("http://code.google.com/p/{repo}/source/checkout", match), nil) - if err != nil { - return err - } - m := googleRepoRe.FindSubmatch(p) - if m == nil { - return NotFoundError{"Could not VCS on Google Code project page."} - } - - match["vcs"] = string(m[1]) - return nil -} - // getGoogleDoc downloads raw files from code.google.com. func getGoogleDoc(client *http.Client, match map[string]string, installRepoPath string, nod *Node, cmdFlags map[string]bool) ([]string, error) { - setupGoogleMatch(match) + packer.SetupGoogleMatch(match) // Check version control. - if m := googleEtagRe.FindStringSubmatch(nod.Value); m != nil { - match["vcs"] = m[1] - } else if err := getGoogleVCS(client, match); err != nil { - return nil, err - } - - rootPath := expand("http://{subrepo}{dot}{repo}.googlecode.com/{vcs}{dir}/", match) - - // Scrape the repo browser to find the project revision and individual Go files. - p, err := HttpGetBytes(client, rootPath+"?r="+nod.Value, nil) - if err != nil { + if err := packer.GetGoogleVCS(client, match); err != nil { return nil, err } @@ -89,143 +51,24 @@ func getGoogleDoc(client *http.Client, match map[string]string, installRepoPath // Remove old files. os.RemoveAll(installPath + "/") - os.MkdirAll(installPath+"/", os.ModePerm) - - // Get source files in root path. - files := make([]*source, 0, 5) - for _, m := range googleFileRe.FindAllSubmatch(p, -1) { - fname := strings.Split(string(m[1]), "?")[0] - if strings.HasPrefix(fname, ".") { - continue - } - - files = append(files, &source{ - name: fname, - rawURL: expand("http://{subrepo}{dot}{repo}.googlecode.com/{vcs}{dir}/{0}", match, fname) + "?r=" + nod.Value, - }) - } - - // Fetch files from VCS. - if err := fetchFiles(client, files, nil); err != nil { + match["tag"] = nod.Value + err := packer.PackToFile(match["importPath"], installPath+".zip", match) + if err != nil { return nil, err } - // Save files. - for _, f := range files { - absPath := installPath + "/" - - // Create diretory before create file. - os.MkdirAll(path.Dir(absPath), os.ModePerm) - - // Write data to file - fw, err := os.Create(absPath + f.name) - if err != nil { - return nil, err - } - - _, err = fw.Write(f.data) - fw.Close() - if err != nil { - return nil, err - } - } - - dirs := make([]string, 0, 3) - // Get subdirectories. - for _, m := range googleDirRe.FindAllSubmatch(p, -1) { - dirName := strings.Split(string(m[1]), "?")[0] - if strings.HasSuffix(dirName, "/") { - dirs = append(dirs, dirName) - } - } - - err = downloadFiles(client, match, rootPath, installPath+"/", nod.Value, dirs) + dirs, err := com.Unzip(installPath+".zip", path.Dir(installPath)) if err != nil { return nil, err } - - var imports []string + os.Remove(installPath + ".zip") + os.Rename(path.Dir(installPath)+"/"+dirs[0], installPath) // Check if need to check imports. if nod.IsGetDeps { - dirs, err := GetDirsInfo(installPath + "/") - if err != nil { - return nil, err - } - - for _, d := range dirs { - if d.IsDir() && !(!cmdFlags["-e"] && strings.Contains(d.Name(), "example")) { - absPath := installPath + "/" + d.Name() + "/" - importPkgs, err := CheckImports(absPath, match["importPath"]) - if err != nil { - return nil, err - } - imports = append(imports, importPkgs...) - } - } + imports := getImports(installPath+"/", match, cmdFlags) + return imports, err } - return imports, err -} - -func downloadFiles(client *http.Client, match map[string]string, rootPath, installPath, commit string, dirs []string) error { - for _, d := range dirs { - p, err := HttpGetBytes(client, rootPath+d+"?r="+commit, nil) - if err != nil { - return err - } - - // Create destination directory. - os.MkdirAll(installPath+d, os.ModePerm) - - // Get source files in current path. - files := make([]*source, 0, 5) - for _, m := range googleFileRe.FindAllSubmatch(p, -1) { - fname := strings.Split(string(m[1]), "?")[0] - files = append(files, &source{ - name: fname, - rawURL: expand("http://{subrepo}{dot}{repo}.googlecode.com/{vcs}{dir}/", match) + d + fname + "?r=" + commit, - }) - } - - // Fetch files from VCS. - if err := fetchFiles(client, files, nil); err != nil { - return err - } - - // Save files. - for _, f := range files { - absPath := installPath + d - - // Create diretory before create file. - os.MkdirAll(path.Dir(absPath), os.ModePerm) - - // Write data to file - fw, err := os.Create(absPath + f.name) - if err != nil { - return err - } - - _, err = fw.Write(f.data) - fw.Close() - if err != nil { - return err - } - } - - subdirs := make([]string, 0, 3) - // Get subdirectories. - for _, m := range googleDirRe.FindAllSubmatch(p, -1) { - dirName := strings.Split(string(m[1]), "?")[0] - if strings.HasSuffix(dirName, "/") { - subdirs = append(subdirs, d+dirName) - } - } - - err = downloadFiles(client, match, rootPath, installPath, commit, subdirs) - if err != nil { - return err - } - } - return nil + return nil, err } diff --git a/doc/http.go b/doc/http.go index d63e31da9..e5f34e05b 100644 --- a/doc/http.go +++ b/doc/http.go @@ -15,20 +15,14 @@ package doc import ( - "encoding/json" "flag" - "fmt" - "io" - "io/ioutil" "net" "net/http" "time" - "github.com/astaxie/beego" + "github.com/Unknwon/com" ) -var userAgent = "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/29.0.1541.0 Safari/537.36" - var ( dialTimeout = flag.Duration("dial_timeout", 10*time.Second, "Timeout for dialing an HTTP connection.") requestTimeout = flag.Duration("request_timeout", 20*time.Second, "Time out for roundtripping an HTTP request.") @@ -45,7 +39,7 @@ type transport struct { func (t *transport) RoundTrip(req *http.Request) (*http.Response, error) { timer := time.AfterFunc(*requestTimeout, func() { t.t.CancelRequest(req) - beego.Warn("Canceled request for %s", req.URL) + com.ColorLog("[WARN] Canceled request for %s, please interrupt the program.\n", req.URL) }) defer timer.Stop() resp, err := t.t.RoundTrip(req) @@ -56,95 +50,3 @@ var ( httpTransport = &transport{t: http.Transport{Dial: timeoutDial, ResponseHeaderTimeout: *requestTimeout / 2}} HttpClient = &http.Client{Transport: httpTransport} ) - -// httpGet gets the specified resource. ErrNotFound is returned if the server -// responds with status 404. -func HttpGetBytes(client *http.Client, url string, header http.Header) ([]byte, error) { - rc, err := httpGet(client, url, header) - if err != nil { - return nil, err - } - p, err := ioutil.ReadAll(rc) - rc.Close() - return p, err -} - -// httpGet gets the specified resource. ErrNotFound is returned if the -// server responds with status 404. -func httpGet(client *http.Client, url string, header http.Header) (io.ReadCloser, error) { - req, err := http.NewRequest("GET", url, nil) - if err != nil { - return nil, err - } - req.Header.Set("User-Agent", userAgent) - for k, vs := range header { - req.Header[k] = vs - } - resp, err := client.Do(req) - if err != nil { - return nil, &RemoteError{req.URL.Host, err} - } - if resp.StatusCode == 200 { - return resp.Body, nil - } - resp.Body.Close() - if resp.StatusCode == 404 { // 403 can be rate limit error. || resp.StatusCode == 403 { - err = NotFoundError{"Resource not found: " + url} - } else { - err = &RemoteError{req.URL.Host, fmt.Errorf("get %s -> %d", url, resp.StatusCode)} - } - return nil, err -} - -// fetchFiles fetches the source files specified by the rawURL field in parallel. -func fetchFiles(client *http.Client, files []*source, header http.Header) error { - ch := make(chan error, len(files)) - for i := range files { - go func(i int) { - req, err := http.NewRequest("GET", files[i].rawURL, nil) - if err != nil { - ch <- err - return - } - req.Header.Set("User-Agent", userAgent) - for k, vs := range header { - req.Header[k] = vs - } - resp, err := client.Do(req) - if err != nil { - ch <- &RemoteError{req.URL.Host, err} - return - } - defer resp.Body.Close() - if resp.StatusCode != 200 { - ch <- &RemoteError{req.URL.Host, fmt.Errorf("get %s -> %d", req.URL, resp.StatusCode)} - return - } - files[i].data, err = ioutil.ReadAll(resp.Body) - if err != nil { - ch <- &RemoteError{req.URL.Host, err} - return - } - ch <- nil - }(i) - } - for _ = range files { - if err := <-ch; err != nil { - return err - } - } - return nil -} - -func httpGetJSON(client *http.Client, url string, v interface{}) error { - rc, err := httpGet(client, url, nil) - if err != nil { - return err - } - defer rc.Close() - err = json.NewDecoder(rc).Decode(v) - if _, ok := err.(*json.SyntaxError); ok { - err = NotFoundError{"JSON syntax error at " + url} - } - return err -} diff --git a/doc/launchpad.go b/doc/launchpad.go index 17fdcc7bb..b7ac63c8d 100644 --- a/doc/launchpad.go +++ b/doc/launchpad.go @@ -21,9 +21,10 @@ import ( "io" "net/http" "os" - //"path" "regexp" "strings" + + "github.com/Unknwon/com" ) var launchpadPattern = regexp.MustCompile(`^launchpad\.net/(?P(?P[a-z0-9A-Z_.\-]+)(?P/[a-z0-9A-Z_.\-]+)?|~[a-z0-9A-Z_.\-]+/(\+junk|[a-z0-9A-Z_.\-]+)/[a-z0-9A-Z_.\-]+)(?P/[a-z0-9A-Z_.\-/]+)*$`) @@ -32,12 +33,13 @@ var launchpadPattern = regexp.MustCompile(`^launchpad\.net/(?P(?P func getLaunchpadDoc(client *http.Client, match map[string]string, installRepoPath string, nod *Node, cmdFlags map[string]bool) ([]string, error) { if match["project"] != "" && match["series"] != "" { - rc, err := httpGet(client, expand("https://code.launchpad.net/{project}{series}/.bzr/branch-format", match), nil) + rc, err := com.HttpGet(client, expand("https://code.launchpad.net/{project}{series}/.bzr/branch-format", match), nil) + _, isNotFound := err.(com.NotFoundError) switch { case err == nil: rc.Close() // The structure of the import path is launchpad.net/{root}/{dir}. - case isNotFound(err): + case isNotFound: // The structure of the import path is is launchpad.net/{project}/{dir}. match["repo"] = match["project"] match["dir"] = expand("{series}{dir}", match) @@ -55,7 +57,7 @@ func getLaunchpadDoc(client *http.Client, match map[string]string, installRepoPa } // Scrape the repo browser to find the project revision and individual Go files. - p, err := HttpGetBytes(client, downloadPath, nil) + p, err := com.HttpGetBytes(client, downloadPath, nil) if err != nil { return nil, err } diff --git a/doc/oschina.go b/doc/oschina.go index 47c075ef3..d6ca10bfe 100644 --- a/doc/oschina.go +++ b/doc/oschina.go @@ -23,6 +23,8 @@ import ( "os" "regexp" "strings" + + "github.com/Unknwon/com" ) var ( @@ -49,7 +51,7 @@ func getOSCDoc(client *http.Client, match map[string]string, installRepoPath str // zip: http://{projectRoot}/repository/archive?ref={sha} // Downlaod archive. - p, err := HttpGetBytes(client, expand("http://git.oschina.net/{owner}/{repo}/repository/archive?ref={sha}", match), nil) + p, err := com.HttpGetBytes(client, expand("http://git.oschina.net/{owner}/{repo}/repository/archive?ref={sha}", match), nil) if err != nil { return nil, errors.New("Fail to donwload OSChina repo -> " + err.Error()) } diff --git a/doc/vcs.go b/doc/vcs.go index c54ef4435..1a7f94926 100644 --- a/doc/vcs.go +++ b/doc/vcs.go @@ -28,6 +28,8 @@ import ( "regexp" "strconv" "strings" + + "github.com/Unknwon/com" ) var ( @@ -112,7 +114,7 @@ func downloadGit(schemes []string, repo, savedEtag string) (string, string, erro } if scheme == "" { - return "", "", NotFoundError{"VCS not found"} + return "", "", com.NotFoundError{"VCS not found"} } tags := make(map[string]string) @@ -172,7 +174,7 @@ func bestTag(tags map[string]string, defaultTag string) (string, string, error) if commit, ok := tags[defaultTag]; ok { return defaultTag, commit, nil } - return "", "", NotFoundError{"Tag or branch not found."} + return "", "", com.NotFoundError{"Tag or branch not found."} } // expand replaces {k} in template with match[k] or subs[atoi(k)] if k is not in match. @@ -199,56 +201,6 @@ func expand(template string, match map[string]string, subs ...string) string { return string(p) } -// checkImports checks package denpendencies. -func CheckImports(absPath, importPath string) (importPkgs []string, err error) { - dir, err := os.Open(absPath) - if err != nil { - return nil, err - } - defer dir.Close() - - // Get file info slice. - fis, err := dir.Readdir(0) - if err != nil { - return nil, err - } - - files := make([]*source, 0, 10) - for _, fi := range fis { - // Only handle files. - if strings.HasSuffix(fi.Name(), ".go") { - f, err := os.Open(absPath + fi.Name()) - if err != nil { - return nil, err - } - - fbytes := make([]byte, fi.Size()) - _, err = f.Read(fbytes) - f.Close() - - if err != nil { - return nil, err - } - - files = append(files, &source{ - name: fi.Name(), - data: fbytes, - }) - } - } - - // Check if has Go source files. - if len(files) > 0 { - w := &walker{ImportPath: importPath} - importPkgs, err = w.build(files) - if err != nil { - return nil, err - } - } - - return importPkgs, err -} - // PureDownload downloads package without version control. func PureDownload(nod *Node, installRepoPath string, flags map[string]bool) ([]string, error) { for _, s := range services { @@ -287,7 +239,7 @@ func getDynamic(client *http.Client, nod *Node, installRepoPath string, flags ma return nil, err } if rootMatch["projectRoot"] != match["projectRoot"] { - return nil, NotFoundError{"Project root mismatch."} + return nil, com.NotFoundError{"Project root mismatch."} } } @@ -312,7 +264,7 @@ func fetchMeta(client *http.Client, importPath string) (map[string]string, error scheme = "http" resp, err = client.Get(scheme + "://" + uri) if err != nil { - return nil, &RemoteError{strings.SplitN(importPath, "/", 2)[0], err} + return nil, &com.RemoteError{strings.SplitN(importPath, "/", 2)[0], err} } } defer resp.Body.Close() @@ -359,7 +311,7 @@ metaScan: continue metaScan } if match != nil { - return nil, NotFoundError{"More than one found at " + scheme + "://" + importPath} + return nil, com.NotFoundError{"More than one found at " + scheme + "://" + importPath} } projectRoot, vcs, repo := f[0], f[1], f[2] @@ -367,7 +319,7 @@ metaScan: repo = strings.TrimSuffix(repo, "."+vcs) i := strings.Index(repo, "://") if i < 0 { - return nil, NotFoundError{"Bad repo URL in ."} + return nil, com.NotFoundError{"Bad repo URL in ."} } proto := repo[:i] repo = repo[i+len("://"):] @@ -390,7 +342,76 @@ metaScan: } } if match == nil { - return nil, NotFoundError{" not found."} + return nil, com.NotFoundError{" not found."} } return match, nil } + +func getImports(rootPath string, match map[string]string, cmdFlags map[string]bool) (imports []string) { + dirs, err := GetDirsInfo(rootPath) + if err != nil { + return nil + } + + for _, d := range dirs { + if d.IsDir() && !(!cmdFlags["-e"] && strings.Contains(d.Name(), "example")) { + absPath := rootPath + d.Name() + "/" + importPkgs, err := CheckImports(absPath, match["importPath"]) + if err != nil { + return nil + } + imports = append(imports, importPkgs...) + } + } + return imports +} + +// checkImports checks package denpendencies. +func CheckImports(absPath, importPath string) (importPkgs []string, err error) { + dir, err := os.Open(absPath) + if err != nil { + return nil, err + } + defer dir.Close() + + // Get file info slice. + fis, err := dir.Readdir(0) + if err != nil { + return nil, err + } + + files := make([]*source, 0, 10) + for _, fi := range fis { + // Only handle files. + if strings.HasSuffix(fi.Name(), ".go") { + f, err := os.Open(absPath + fi.Name()) + if err != nil { + return nil, err + } + + fbytes := make([]byte, fi.Size()) + _, err = f.Read(fbytes) + f.Close() + + if err != nil { + return nil, err + } + + files = append(files, &source{ + name: fi.Name(), + data: fbytes, + }) + } + } + + // Check if has Go source files. + if len(files) > 0 { + w := &walker{ImportPath: importPath} + importPkgs, err = w.build(files) + if err != nil { + return nil, err + } + } + + return importPkgs, err +} diff --git a/gopm.go b/gopm.go index b564be06b..a558f3b58 100644 --- a/gopm.go +++ b/gopm.go @@ -37,7 +37,7 @@ import ( // Test that go1.1 tag above is included in builds. main.go refers to this definition. const go11tag = true -const APP_VER = "0.2.2.0819" +const APP_VER = "0.2.5.0827" var ( config map[string]interface{}