// Copyright 2013 gopm authors. // // Licensed under the Apache License, Version 2.0 (the "License"): you may // not use this file except in compliance with the License. You may obtain // a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the // License for the specific language governing permissions and limitations // under the License. package doc import ( "net/http" "os" "path" "regexp" "strings" ) var ( googleRepoRe = regexp.MustCompile(`id="checkoutcmd">(hg|git|svn)`) googleRevisionRe = regexp.MustCompile(`

(?:[^ ]+ - )?Revision *([^:]+):`) googleEtagRe = regexp.MustCompile(`^(hg|git|svn)-`) googleFileRe = regexp.MustCompile(`
  • [a-z0-9\-]+)(:?\.(?P[a-z0-9\-]+))?(?P/[a-z0-9A-Z_.\-/]+)?$`) ) func setupGoogleMatch(match map[string]string) { if s := match["subrepo"]; s != "" { match["dot"] = "." match["query"] = "?repo=" + s } else { match["dot"] = "" match["query"] = "" } } func getGoogleVCS(client *http.Client, match map[string]string) error { // Scrape the HTML project page to find the VCS. p, err := HttpGetBytes(client, expand("http://code.google.com/p/{repo}/source/checkout", match), nil) if err != nil { return err } m := googleRepoRe.FindSubmatch(p) if m == nil { return NotFoundError{"Could not VCS on Google Code project page."} } match["vcs"] = string(m[1]) return nil } // GetGoogleDoc downloads raw files from code.google.com. func GetGoogleDoc(client *http.Client, match map[string]string, installRepoPath string, nod *Node, cmdFlags map[string]bool) ([]string, error) { setupGoogleMatch(match) // Check version control. if m := googleEtagRe.FindStringSubmatch(nod.Value); m != nil { match["vcs"] = m[1] } else if err := getGoogleVCS(client, match); err != nil { return nil, err } rootPath := expand("http://{subrepo}{dot}{repo}.googlecode.com/{vcs}{dir}/", match) // Scrape the repo browser to find the project revision and individual Go files. p, err := HttpGetBytes(client, rootPath+"?r="+nod.Value, nil) if err != nil { return nil, err } suf := "." + nod.Value if len(suf) == 1 { suf = "" } projectPath := expand("code.google.com/p/{repo}{dot}{subrepo}{dir}", match) installPath := installRepoPath + "/" + projectPath + suf nod.ImportPath = projectPath // Remove old files. os.RemoveAll(installPath + "/") os.MkdirAll(installPath+"/", os.ModePerm) // Get source files in root path. files := make([]*source, 0, 5) for _, m := range googleFileRe.FindAllSubmatch(p, -1) { fname := strings.Split(string(m[1]), "?")[0] if strings.HasPrefix(fname, ".") { continue } files = append(files, &source{ name: fname, rawURL: expand("http://{subrepo}{dot}{repo}.googlecode.com/{vcs}{dir}/{0}", match, fname) + "?r=" + nod.Value, }) } // Fetch files from VCS. if err := fetchFiles(client, files, nil); err != nil { return nil, err } // Save files. for _, f := range files { absPath := installPath + "/" // Create diretory before create file. os.MkdirAll(path.Dir(absPath), os.ModePerm) // Write data to file fw, err := os.Create(absPath + f.name) if err != nil { return nil, err } _, err = fw.Write(f.data) fw.Close() if err != nil { return nil, err } } dirs := make([]string, 0, 3) // Get subdirectories. for _, m := range googleDirRe.FindAllSubmatch(p, -1) { dirName := strings.Split(string(m[1]), "?")[0] if strings.HasSuffix(dirName, "/") { dirs = append(dirs, dirName) } } err = downloadFiles(client, match, rootPath, installPath+"/", nod.Value, dirs) if err != nil { return nil, err } var imports []string // Check if need to check imports. if nod.IsGetDeps { dirs, err := GetDirsInfo(installPath + "/") if err != nil { return nil, err } for _, d := range dirs { if d.IsDir() && !(!cmdFlags["-e"] && strings.Contains(d.Name(), "example")) { absPath := installPath + "/" + d.Name() + "/" importPkgs, err := CheckImports(absPath, match["importPath"]) if err != nil { return nil, err } imports = append(imports, importPkgs...) } } } return imports, err } func downloadFiles(client *http.Client, match map[string]string, rootPath, installPath, commit string, dirs []string) error { for _, d := range dirs { p, err := HttpGetBytes(client, rootPath+d+"?r="+commit, nil) if err != nil { return err } // Create destination directory. os.MkdirAll(installPath+d, os.ModePerm) // Get source files in current path. files := make([]*source, 0, 5) for _, m := range googleFileRe.FindAllSubmatch(p, -1) { fname := strings.Split(string(m[1]), "?")[0] files = append(files, &source{ name: fname, rawURL: expand("http://{subrepo}{dot}{repo}.googlecode.com/{vcs}{dir}/", match) + d + fname + "?r=" + commit, }) } // Fetch files from VCS. if err := fetchFiles(client, files, nil); err != nil { return err } // Save files. for _, f := range files { absPath := installPath + d // Create diretory before create file. os.MkdirAll(path.Dir(absPath), os.ModePerm) // Write data to file fw, err := os.Create(absPath + f.name) if err != nil { return err } _, err = fw.Write(f.data) fw.Close() if err != nil { return err } } subdirs := make([]string, 0, 3) // Get subdirectories. for _, m := range googleDirRe.FindAllSubmatch(p, -1) { dirName := strings.Split(string(m[1]), "?")[0] if strings.HasSuffix(dirName, "/") { subdirs = append(subdirs, d+dirName) } } err = downloadFiles(client, match, rootPath, installPath, commit, subdirs) if err != nil { return err } } return nil }