diff options
Diffstat (limited to 'tools/convertdb')
-rw-r--r-- | tools/convertdb/Makefile | 13 | ||||
-rw-r--r-- | tools/convertdb/convertdb.go | 116 | ||||
-rw-r--r-- | tools/convertdb/couch.go | 403 |
3 files changed, 0 insertions, 532 deletions
diff --git a/tools/convertdb/Makefile b/tools/convertdb/Makefile deleted file mode 100644 index eba288ec1d5b..000000000000 --- a/tools/convertdb/Makefile +++ /dev/null @@ -1,13 +0,0 @@ -all: convertdb - -convertdb: couch.8 convertdb.8 - 8l -o convertdb convertdb.8 - -convertdb.8: convertdb.go - 8g convertdb.go - -couch.8: couch.go - 8g couch.go - -clean: - rm -rf *.8 convertdb diff --git a/tools/convertdb/convertdb.go b/tools/convertdb/convertdb.go deleted file mode 100644 index f7b94176b422..000000000000 --- a/tools/convertdb/convertdb.go +++ /dev/null @@ -1,116 +0,0 @@ -package main - -import ( - "strconv" - "fmt" - "io/ioutil" - "json" - "./couch" - "os" - "time" -) - -//old -type OldComment struct { - Author string - Text string - Date string -} - -type OldEntry struct { - Id string - Title string - Author string - Text string - Mtext string - Comments []OldComment -} - -//old -type Comment struct { - Author string `json:"cauthor"` - Text string `json:"ctext"` - Date int64 `json:"cdate"` -} - -type Entry struct { - Id string `json:"_id"` - Year int `json:"year"` - Month int `json:"month"` - Day int `json:"day"` - Lang string `json:"lang"` - Title string `json:"title"` - Author string `json:"author"` - Text string `json:"text"` - Mtext string `json:"mtext"` - Comments []Comment `json:"comments"` -} - -func main() { - getAllByYear("2011", 8, 12) - getAllByYear("2012", 1, 2) -} - -func getAllByYear(year string, minm, maxm int){ - db, _ := couch.NewDatabase("127.0.0.1", "5984", "tazblog") - for i:=minm;i<=maxm;i++{ - dirList, err := ioutil.ReadDir(fmt.Sprintf("data/%s/%02d/", year, i)) - if err != nil { - fmt.Println(err.String()) - os.Exit(1) - } - for d:=len(dirList)-1; d>-1; d--{ - content, cErr := ioutil.ReadFile(fmt.Sprintf("data/%s/%02d/%s", year, i, dirList[d].Name)) - if cErr != nil { - fmt.Println(cErr) - os.Exit(1) - } - var oEntry OldEntry - jErr := json.Unmarshal(content, &oEntry) - if jErr != nil { - fmt.Println(jErr.String()) - os.Exit(1) - } - nEntry := convertEntry(oEntry, fmt.Sprintf("data/%s/%02d/%s", year, i, dirList[d].Name)) - eId, _, err := db.Insert(nEntry) - if err != nil { - fmt.Println(err.String()) - os.Exit(1) - } - fmt.Println("Inserted " + eId) - } - } -} - -func convertEntry(oEntry OldEntry, p string) Entry{ - var nEntry Entry - nComments := make([]Comment, len(oEntry.Comments)) - for i:=0;i<len(oEntry.Comments);i++{ - nComments[i].Author = oEntry.Comments[i].Author - nComments[i].Text = oEntry.Comments[i].Text - nComments[i].Date = parseDumbTime(oEntry.Comments[i].Date) - } - - nEntry.Id = oEntry.Id[3:] - nEntry.Year, _ = strconv.Atoi(p[5:9]) - nEntry.Month, _ = strconv.Atoi(p[10:12]) - nEntry.Day, _ = strconv.Atoi(p[13:15]) - nEntry.Title = oEntry.Title - nEntry.Author = oEntry.Author - nEntry.Mtext = oEntry.Mtext - nEntry.Text = oEntry.Text - nEntry.Comments = nComments - nEntry.Lang = "DE" - - return nEntry -} - -func parseDumbTime(ct string) int64 { - x, err := time.Parse("[Am 02.01.2006 um 15:04 Uhr]", ct) - if err != nil { - fmt.Println(err.String()) - os.Exit(1) - } - - return x.Seconds() -} \ No newline at end of file diff --git a/tools/convertdb/couch.go b/tools/convertdb/couch.go deleted file mode 100644 index 764eb49a6099..000000000000 --- a/tools/convertdb/couch.go +++ /dev/null @@ -1,403 +0,0 @@ -// -*- tab-width: 4 -*- -package couch - -import ( - "bytes" - "fmt" - "os" - "json" - "http" - "net" - "io/ioutil" - "url" -) - -var def_hdrs = map[string][]string{} - -type buffer struct { - b *bytes.Buffer -} - -func (b *buffer) Read(out []byte) (int, os.Error) { - return b.b.Read(out) -} - -func (b *buffer) Close() os.Error { return nil } - -// Converts given URL to string containing the body of the response. -func url_to_buf(u string) []byte { - if r, err := http.Get(u); err == nil { - b, err := ioutil.ReadAll(r.Body) - r.Body.Close() - if err == nil { - return b - } - } - return make([]byte, 0) -} - -type IdAndRev struct { - Id string `json:"_id"` - Rev string `json:"_rev"` -} - -// Sends a query to CouchDB and parses the response back. -// method: the name of the HTTP method (POST, PUT,...) -// url: the URL to interact with -// headers: additional headers to pass to the request -// in: body of the request -// out: a structure to fill in with the returned JSON document -func (p Database) interact(method, u string, headers map[string][]string, in []byte, out interface{}) (int, os.Error) { - fullHeaders := map[string][]string{} - for k, v := range headers { - fullHeaders[k] = v - } - bodyLength := 0 - if in != nil { - bodyLength = len(in) - fullHeaders["Content-Type"] = []string{"application/json"} - } - req := http.Request{ - Method: method, - ProtoMajor: 1, - ProtoMinor: 1, - Close: true, - ContentLength: int64(bodyLength), - Header: fullHeaders, - } - req.TransferEncoding = []string{"chunked"} - req.URL, _ = url.Parse(u) - if in != nil { - req.Body = &buffer{bytes.NewBuffer(in)} - } - conn, err := net.Dial("tcp", fmt.Sprintf("%s:%s", p.Host, p.Port)) - if err != nil { - return 0, err - } - http_conn := http.NewClientConn(conn, nil) - defer http_conn.Close() - if err := http_conn.Write(&req); err != nil { - return 0, err - } - r, err := http_conn.Read(&req) - if err != nil { - return 0, err - } - if r.StatusCode < 200 || r.StatusCode >= 300 { - b := []byte{} - r.Body.Read(b) - return r.StatusCode, os.NewError("server said: " + r.Status) - } - decoder := json.NewDecoder(r.Body) - if err = decoder.Decode(out); err != nil { - return 0, err - } - r.Body.Close() - return r.StatusCode, nil -} - -type Database struct { - Host string - Port string - Name string -} - -func (p Database) BaseURL() string { - return fmt.Sprintf("http://%s:%s", p.Host, p.Port) -} - -func (p Database) DBURL() string { - return fmt.Sprintf("%s/%s", p.BaseURL(), p.Name) -} - -// Test whether CouchDB is running (ignores Database.Name) -func (p Database) Running() bool { - u := fmt.Sprintf("%s/%s", p.BaseURL(), "_all_dbs") - s := url_to_buf(u) - if len(s) > 0 { - return true - } - return false -} - -type database_info struct { - Db_name string - // other stuff too, ignore for now -} - -// Test whether specified database exists in specified CouchDB instance -func (p Database) Exists() bool { - di := new(database_info) - if err := json.Unmarshal(url_to_buf(p.DBURL()), di); err != nil { - return false - } - if di.Db_name != p.Name { - return false - } - return true -} - -func (p Database) create_database() os.Error { - ir := response{} - if _, err := p.interact("PUT", p.DBURL(), def_hdrs, nil, &ir); err != nil { - return err - } - if !ir.Ok { - return os.NewError("Create database operation returned not-OK") - } - return nil -} - -// Deletes the given database and all documents -func (p Database) DeleteDatabase() os.Error { - ir := response{} - if _, err := p.interact("DELETE", p.DBURL(), def_hdrs, nil, &ir); err != nil { - return err - } - if !ir.Ok { - return os.NewError("Delete database operation returned not-OK") - } - return nil -} - -func NewDatabase(host, port, name string) (Database, os.Error) { - db := Database{host, port, name} - if !db.Running() { - return db, os.NewError("CouchDB not running") - } - if !db.Exists() { - if err := db.create_database(); err != nil { - return db, err - } - } - return db, nil -} - -// Strip _id and _rev from d, returning them separately if they exist -func clean_JSON(d interface{}) (json_buf []byte, id, rev string, err os.Error) { - json_buf, err = json.Marshal(d) - if err != nil { - return - } - m := map[string]interface{}{} - err = json.Unmarshal(json_buf, &m) - if err != nil { - return - } - id_rev := new(IdAndRev) - err = json.Unmarshal(json_buf, &id_rev) - if err != nil { - return - } - if _, ok := m["_id"]; ok { - id = id_rev.Id - m["_id"] = nil, false - } - if _, ok := m["_rev"]; ok { - rev = id_rev.Rev - m["_rev"] = nil, false - } - json_buf, err = json.Marshal(m) - return -} - -type response struct { - Ok bool - Id string - Rev string - Error string - Reason string -} - -// Inserts document to CouchDB, returning id and rev on success. -// Document may specify both "_id" and "_rev" fields (will overwrite existing) -// or just "_id" (will use that id, but not overwrite existing) -// or neither (will use autogenerated id) -func (p Database) Insert(d interface{}) (string, string, os.Error) { - json_buf, id, rev, err := clean_JSON(d) - if err != nil { - return "", "", err - } - if id != "" && rev != "" { - new_rev, err2 := p.Edit(d) - return id, new_rev, err2 - } else if id != "" { - return p.insert_with(json_buf, id) - } else if id == "" { - return p.insert(json_buf) - } - return "", "", os.NewError("invalid Document") -} - -// Private implementation of simple autogenerated-id insert -func (p Database) insert(json_buf []byte) (string, string, os.Error) { - ir := response{} - if _, err := p.interact("POST", p.DBURL(), def_hdrs, json_buf, &ir); err != nil { - return "", "", err - } - if !ir.Ok { - return "", "", os.NewError(fmt.Sprintf("%s: %s", ir.Error, ir.Reason)) - } - return ir.Id, ir.Rev, nil -} - -// Inserts the given document (shouldn't contain "_id" or "_rev" tagged fields) -// using the passed 'id' as the _id. Will fail if the id already exists. -func (p Database) InsertWith(d interface{}, id string) (string, string, os.Error) { - json_buf, err := json.Marshal(d) - if err != nil { - return "", "", err - } - return p.insert_with(json_buf, id) -} - -// Private implementation of insert with given id -func (p Database) insert_with(json_buf []byte, id string) (string, string, os.Error) { - u := fmt.Sprintf("%s/%s", p.DBURL(), url.QueryEscape(id)) - ir := response{} - if _, err := p.interact("PUT", u, def_hdrs, json_buf, &ir); err != nil { - return "", "", err - } - if !ir.Ok { - return "", "", os.NewError(fmt.Sprintf("%s: %s", ir.Error, ir.Reason)) - } - return ir.Id, ir.Rev, nil -} - -// Edits the given document, returning the new revision. -// d must contain "_id" and "_rev" tagged fields. -func (p Database) Edit(d interface{}) (string, os.Error) { - json_buf, err := json.Marshal(d) - if err != nil { - return "", err - } - id_rev := new(IdAndRev) - err = json.Unmarshal(json_buf, id_rev) - if err != nil { - return "", err - } - if id_rev.Id == "" { - return "", os.NewError("Id not specified in interface") - } - if id_rev.Rev == "" { - return "", os.NewError("Rev not specified in interface (try InsertWith)") - } - u := fmt.Sprintf("%s/%s", p.DBURL(), url.QueryEscape(id_rev.Id)) - ir := response{} - if _, err = p.interact("PUT", u, def_hdrs, json_buf, &ir); err != nil { - return "", err - } - return ir.Rev, nil -} - -// Edits the given document, returning the new revision. -// d should not contain "_id" or "_rev" tagged fields. If it does, they will -// be overwritten with the passed values. -func (p Database) EditWith(d interface{}, id, rev string) (string, os.Error) { - if id == "" || rev == "" { - return "", os.NewError("EditWith: must specify both id and rev") - } - json_buf, err := json.Marshal(d) - if err != nil { - return "", err - } - m := map[string]interface{}{} - err = json.Unmarshal(json_buf, &m) - if err != nil { - return "", err - } - m["_id"] = id - m["_rev"] = rev - return p.Edit(m) -} - -// Unmarshals the document matching id to the given interface, returning rev. -func (p Database) Retrieve(id string, d interface{}) (string, os.Error) { - if id == "" { - return "", os.NewError("no id specified") - } - json_buf := url_to_buf(fmt.Sprintf("%s/%s", p.DBURL(), id)) - id_rev := new(IdAndRev) - if err := json.Unmarshal(json_buf, &id_rev); err != nil { - return "", err - } - if id_rev.Id != id { - return "", os.NewError("invalid id specified") - } - return id_rev.Rev, json.Unmarshal(json_buf, d) -} - -// Deletes document given by id and rev. -func (p Database) Delete(id, rev string) os.Error { - headers := map[string][]string{ - "If-Match": []string{rev}, - } - u := fmt.Sprintf("%s/%s", p.DBURL(), id) - ir := response{} - if _, err := p.interact("DELETE", u, headers, nil, &ir); err != nil { - return err - } - if !ir.Ok { - return os.NewError(fmt.Sprintf("%s: %s", ir.Error, ir.Reason)) - } - return nil -} - -type Row struct { - Id *string -} - -type keyed_view_response struct { - Total_rows uint64 - Offset uint64 - Rows []Row -} - -// Return array of document ids as returned by the given view/options combo. -// view should be eg. "_design/my_foo/_view/my_bar" -// options should be eg. { "limit": 10, "key": "baz" } -func (p Database) QueryIds(view string, options map[string]interface{}) ([]string, os.Error) { - kvr := new(keyed_view_response) - - if err := p.Query(view, options, kvr); err != nil { - fmt.Println("Query error: " + err.String()) - return make([]string, 0), err - } - ids := make([]string, len(kvr.Rows)) - i := 0 - for _, row := range kvr.Rows { - if row.Id != nil { - ids[i] = *row.Id - i++ - } - } - return ids[:i], nil -} - -func (p Database) Query(view string, options map[string]interface{}, results interface{}) os.Error { - if view == "" { - return os.NewError("empty view") - } - - var parameters string - for k, v := range options { - switch t := v.(type) { - case string: - parameters += fmt.Sprintf(`%s=%s&`, k, url.QueryEscape(t)) - case int: - parameters += fmt.Sprintf(`%s=%d&`, k, t) - case bool: - parameters += fmt.Sprintf(`%s=%v&`, k, t) - default: - // TODO more types are supported - panic(fmt.Sprintf("unsupported value-type %T in Query", t)) - } - } - full_url := fmt.Sprintf("%s/%s?%s", p.DBURL(), view, parameters) - json_buf := url_to_buf(full_url) - - if err := json.Unmarshal(json_buf, results); err != nil { - return err - } - return nil -} |