basic search function

cmd/search.go

@@ -0,0 +1,113 @@
+// Copyright 2013 gopm authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"): you may
+// not use this file except in compliance with the License. You may obtain
+// a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations
+// under the License.
+
+package cmd
+
+import (
+	"../doc"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+)
+
+var CmdSearch = &Command{
+	UsageLine: "search [keyword]",
+	Short:     "search for package",
+	Long: `
+search packages
+
+The search flags are:
+
+	-s
+		start a search service. This must be run before search a package
+
+	-e
+		search extactly, you should input an exactly package name as keyword
+`,
+}
+
+func init() {
+	CmdSearch.Run = runSearch
+	CmdSearch.Flags = map[string]bool{
+		"-s": false,
+	}
+}
+
+func printSearchPrompt(flag string) {
+	switch flag {
+	case "-s":
+		doc.ColorLog("[INFO] You enabled start a service.\n")
+	case "-e":
+		doc.ColorLog("[INFO] You enabled exactly search.\n")
+	}
+}
+
+// search packages
+func runSearch(cmd *Command, args []string) {
+	// Check flags.
+	num := checkFlags(cmd.Flags, args, printSearchPrompt)
+	if num == -1 {
+		return
+	}
+	args = args[num:]
+
+	// Check length of arguments.
+	if len(args) < 1 {
+		doc.ColorLog("[ERROR] Please input package's keyword.\n")
+		return
+	}
+
+	if cmd.Flags["-e"] {
+		search(args[0], true)
+	} else {
+		search(args[0], false)
+	}
+}
+
+/*
+request local or remote search service to find packages according to keyword inputed
+*/
+func search(keyword string, isExactly bool) {
+	url := "http://localhost:8991/search?"
+	if isExactly {
+		url = "http://localhost:8991/searche?"
+	}
+	resp, err := http.Get(url + keyword)
+	if err != nil {
+		doc.ColorLog(err.Error())
+		return
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode == 200 {
+		contents, err := ioutil.ReadAll(resp.Body)
+		if err != nil {
+			doc.ColorLog(err.Error())
+			return
+		}
+
+		pkgs := make([]string, 0)
+		err = json.Unmarshal(contents, &pkgs)
+		if err != nil {
+			doc.ColorLog(err.Error())
+			return
+		}
+		for i, pkg := range pkgs {
+			fmt.Println(i+1, pkg)
+		}
+	} else {
+		doc.ColorLog(resp.Status)
+	}
+}

cmd/serve.go

@@ -0,0 +1,394 @@
+package cmd
+
+import (
+	"../doc"
+	"fmt"
+	"github.com/syndtr/goleveldb/leveldb"
+	"github.com/syndtr/goleveldb/leveldb/errors"
+	"github.com/syndtr/goleveldb/leveldb/opt"
+	"io/ioutil"
+	"net/http"
+	"os"
+	"os/user"
+	"strconv"
+	"strings"
+)
+
+var (
+	dbDir = "~/.gopm/db"
+)
+
+const (
+	STOP = iota
+	LOCALRUN
+	RUNNING
+)
+
+var CmdServe = &Command{
+	UsageLine: "serve [:port]",
+	Short:     "serve for package search",
+	Long: `
+	serve provide a web service to search packages, download packages 
+
+The serve flags are:
+
+	-l
+		only service for localhost ip
+`,
+}
+
+func init() {
+	CmdServe.Run = runServe
+	CmdServe.Flags = map[string]bool{
+		"-l": false,
+	}
+}
+
+func printServePrompt(flag string) {
+	switch flag {
+	case "-l":
+		doc.ColorLog("[INFO] You enabled start a service only localhost.\n")
+	}
+}
+
+// Not implemented
+func autoPort() string {
+	return "8991"
+}
+
+// search packages
+func runServe(cmd *Command, args []string) {
+	// Check flags.
+	num := checkFlags(cmd.Flags, args, printServePrompt)
+	if num == -1 {
+		return
+	}
+	args = args[num:]
+
+	var listen string
+	var port string
+	if cmd.Flags["-l"] {
+		listen += "127.0.0.1:"
+		port = autoPort()
+	} else {
+		listen += "0.0.0.0:"
+		port = "8991"
+	}
+
+	// Check length of arguments.
+	if len(args) >= 1 {
+		port = args[0]
+	}
+
+	startService(listen + port)
+}
+
+func splitWord(word string, res *map[string]bool) {
+	for i, _ := range word {
+		for j, _ := range word[i:] {
+			w := word[i : i+j+1]
+			(*res)[w] = true
+		}
+	}
+	return
+}
+
+func splitPkgName(pkgName string) (res map[string]bool) {
+	//var src string
+	ps := strings.Split(pkgName, "/")
+	if len(ps) > 1 {
+		ps = ps[1:]
+	}
+
+	res = make(map[string]bool, 0)
+	res[strings.Join(ps, "/")] = true
+	for _, w := range ps {
+		splitWord(w, &res)
+	}
+	return
+}
+
+var (
+	ro *opt.ReadOptions  = &opt.ReadOptions{}
+	wo *opt.WriteOptions = &opt.WriteOptions{}
+)
+
+func dbGet(key string) (string, error) {
+	v, err := db.Get([]byte(key), ro)
+	return string(v), err
+}
+
+func dbPut(key string, value string) error {
+	fmt.Println("put ", key, ": ", value)
+	return db.Put([]byte(key), []byte(value), wo)
+}
+
+func batchPut(batch *leveldb.Batch, key string, value string) error {
+	fmt.Println("put ", key, ": ", value)
+	batch.Put([]byte(key), []byte(value))
+	return nil
+}
+
+func addPkg(pkg *Pkg) error {
+	batch := new(leveldb.Batch)
+	strLastId, err := dbGet("lastId")
+	if err != nil {
+		if err == errors.ErrNotFound {
+			strLastId = "0"
+			err = batchPut(batch, "lastId", strLastId)
+		} else {
+			return err
+		}
+	}
+	if err != nil {
+		return err
+	}
+
+	fmt.Println("last id is ", strLastId)
+
+	lastId, err := strconv.ParseInt(strLastId, 0, 64)
+	if err != nil {
+		return err
+	}
+
+	pkgKey := fmt.Sprintf("index:%v", pkg.Name)
+
+	id, err := dbGet(pkgKey)
+	if err != nil {
+		if err == errors.ErrNotFound {
+			id = fmt.Sprintf("%v", lastId+1)
+			fmt.Println(id)
+			err = batchPut(batch, "lastId", id)
+			if err == nil {
+				err = batchPut(batch, pkgKey, id)
+			}
+			if err == nil {
+				err = batchPut(batch, "pkg:"+id, pkg.Name)
+			}
+			total, err := dbGet("total")
+			if err != nil {
+				if err == errors.ErrNotFound {
+					total = "1"
+				} else {
+					return err
+				}
+			} else {
+				totalInt, err := strconv.ParseInt(total, 0, 64)
+				if err != nil {
+					return err
+				}
+				totalInt = totalInt + 1
+				total = fmt.Sprintf("%v", totalInt)
+			}
+
+			err = batchPut(batch, "total", total)
+		} else {
+			return err
+		}
+	}
+
+	if err != nil {
+		return err
+	}
+
+	vers, err := dbGet("ver:" + id)
+	needSplit := (err == errors.ErrNotFound)
+	if err != nil {
+		if err != errors.ErrNotFound {
+			return err
+		}
+	} else {
+		return nil
+	}
+
+	if vers == "" {
+		fmt.Println(pkg)
+		vers = pkg.VerString()
+	} else {
+		if !strings.Contains(vers, pkg.VerString()) {
+			vers = vers + "," + pkg.VerString()
+		} else {
+			return nil
+		}
+	}
+
+	err = batchPut(batch, "ver:"+id, vers)
+	if err != nil {
+		return err
+	}
+
+	if !needSplit {
+		return nil
+	}
+
+	keys := splitPkgName(pkg.Name)
+
+	for key, _ := range keys {
+		err = batchPut(batch, fmt.Sprintf("key:%v:%v", key, id), "")
+		if err != nil {
+			return err
+		}
+	}
+
+	return db.Write(batch, wo)
+}
+
+func rmPkg(pkg *Pkg) {
+
+}
+
+var db *leveldb.DB
+
+// service should be run
+func autoRun() {
+	s, _, _ := runningStatus()
+	if s == STOP {
+		os.StartProcess("gopm", []string{"serve", "-l"}, nil)
+	}
+}
+
+func runningStatus() (int, int, int) {
+	contentByte, err := ioutil.ReadFile("~/.gopm/var/pid")
+	if err != nil {
+		return STOP, 0, 0
+	}
+	content := string(contentByte)
+	if len(content) < 0 || !strings.Contains(content, ",") {
+		return STOP, 0, 0
+	}
+	cs := strings.Split(string(content), ",")
+	if len(cs) != 3 {
+		return STOP, 0, 0
+	}
+	status, err := strconv.Atoi(cs[0])
+	if err != nil {
+		return STOP, 0, 0
+	}
+	if status < STOP || status > RUNNING {
+		return STOP, 0, 0
+	}
+	pid, err := strconv.Atoi(cs[1])
+	if err != nil {
+		return STOP, 0, 0
+	}
+
+	_, err = os.FindProcess(pid)
+	if err != nil {
+		return STOP, 0, 0
+	}
+
+	port, err := strconv.Atoi(cs[2])
+	if err != nil {
+		return STOP, 0, 0
+	}
+
+	return status, pid, port
+}
+
+func startService(listen string) {
+	// check the pre serve's type
+	curUser, err := user.Current()
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+
+	dbDir = strings.Replace(dbDir, "~", curUser.HomeDir, -1)
+
+	db, err = leveldb.OpenFile(dbDir, &opt.Options{Flag: opt.OFCreateIfMissing})
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+	defer db.Close()
+
+	// these handlers should only access by localhost
+	http.HandleFunc("/add", addHandler)
+	http.HandleFunc("/rm", rmHandler)
+
+	// these handlers can be accessed according listen's ip
+	http.HandleFunc("/search", searchHandler)
+	http.HandleFunc("/searche", searcheHandler)
+	http.ListenAndServe(listen, nil)
+}
+
+func searchHandler(w http.ResponseWriter, r *http.Request) {
+	r.ParseForm()
+	ids := make(map[string]bool)
+	for key, _ := range r.Form {
+		iter := db.NewIterator(ro)
+		rkey := fmt.Sprintf("key:%v:", key)
+		if iter.Seek([]byte(rkey)) {
+			k := iter.Key()
+			if !strings.HasPrefix(string(k), rkey) {
+				break
+			} else {
+				ids[string(k)] = true
+			}
+		}
+		for iter.Next() {
+			k := iter.Key()
+			if !strings.HasPrefix(string(k), rkey) {
+				break
+			}
+			ids[string(k)] = true
+		}
+	}
+
+	pkgs := make([]string, 0)
+
+	for id, _ := range ids {
+		idkeys := strings.SplitN(id, ":", -1)
+		rId := idkeys[len(idkeys)-1]
+		fmt.Println(rId)
+		pkg, err := dbGet(fmt.Sprintf("pkg:%v", rId))
+		if err != nil {
+			doc.ColorLog(err.Error())
+			continue
+		}
+		pkgs = append(pkgs, pkg)
+	}
+
+	w.Write([]byte("[\"" + strings.Join(pkgs, "\", \"") + "\"]"))
+}
+
+func searcheHandler(w http.ResponseWriter, r *http.Request) {
+	//if r.Method == "POST" {
+	r.ParseForm()
+	pkgs := make([]string, 0)
+	for key, _ := range r.Form {
+		_, err := dbGet("index:" + key)
+
+		if err != nil {
+			doc.ColorLog(err.Error())
+			continue
+		}
+
+		pkgs = append(pkgs, key)
+	}
+
+	w.Write([]byte("[\"" + strings.Join(pkgs, "\", \"") + "\"]"))
+	//}
+}
+
+func addHandler(w http.ResponseWriter, r *http.Request) {
+	//if r.Method == "POST" {
+	r.ParseForm()
+	for key, _ := range r.Form {
+		fmt.Println(key)
+		pkg := NewPkg(key, "")
+		if pkg != nil {
+			err := addPkg(pkg)
+			if err != nil {
+				fmt.Println(err)
+			}
+		} else {
+			fmt.Println(key)
+		}
+	}
+	//}
+}
+
+func rmHandler(w http.ResponseWriter, r *http.Request) {
+
+}

cmd/service.go

@@ -93,6 +93,10 @@ func NewPkg(pkgName string, ver string) *Pkg {
 		verId = vers[1]
 	}
 
+	if len(vers) == 1 {
+		vers[0] = TRUNK
+	}
+
 	service := getService(pkgName)
 	if service == nil {
 		return nil

docs/features_CN.md

@@ -50,6 +50,15 @@ http://gopm.io
 [repos]
 ~/.gopm/repos
 
+#数据库说明
+包信息数据采用goleveldb，这是一个key/value数据库。数据存放规则如下：
+"lastId" : "{lastId}"   			lastId中存放最大的Id，Id为自增
+
+"index:{packageName}": "{id}"  index:中存放的是包名，value中存放的是这个包的不同版本的id，不同版本用逗号分隔
+
+“ver:{id}” : "{verString1}, {verString2}"  某个包版本对应的内容
+
+“key:{keyword}:{id}” : ""   关键词及其对应的版本
 
 <a id="30" name="30"></a>
 #各命令的目标和作用

gopm.go

@@ -43,26 +43,28 @@ var (
 // The order here is the order in which they are printed by 'gopm help'.
 var commands = []*cmd.Command{
 	cmd.CmdGet,
-	/*cmd.CmdGen,
-	cmdBuild,
-	cmdClean,
-	cmdDoc,
-	cmdEnv,
-	cmdFix,
-	cmdFmt,
-	cmdInstall,
-	cmdList,
-	cmdRun,
-	cmdTest,
-	cmdTool,
-	cmdVersion,
-	cmdVet,
-
-	helpGopath,
-	helpPackages,
-	helpRemote,
-	helpTestflag,
-	helpTestfunc,*/
+	cmd.CmdSearch,
+	cmd.CmdServe,
+	/*
+		cmdBuild,
+		cmdClean,
+		cmdDoc,
+		cmdEnv,
+		cmdFix,
+		cmdFmt,
+		cmdInstall,
+		cmdList,
+		cmdRun,
+		cmdTest,
+		cmdTool,
+		cmdVersion,
+		cmdVet,
+
+		helpGopath,
+		helpPackages,
+		helpRemote,
+		helpTestflag,
+		helpTestfunc,*/
 }
 
 // We don't use init() to initialize