diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 57612c3..0000000 --- a/.travis.yml +++ /dev/null @@ -1,10 +0,0 @@ -language: go - -go: - - 1.2 - - tip - -notifications: - flowdock: - secure: fZrcf9rlh2IrQrlch1sHkn3YI7SKvjGnAl/zyV5D6NROe1Bbr6d3QRMuCXWWdhJHzjKmXk5rIzbqJhUc0PNF7YjxGNKSzqWMQ56KcvN1k8DzlqxpqkcA3Jbs6fXCWo2fssRtZ7hj/wOP1f5n6cc7kzHDt9dgaYJ6nO2fqNPJiTc= - diff --git a/README.md b/README.md index f26ada9..484d7a8 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,10 @@ -raft-mdb [![Build Status](https://travis-ci.org/hashicorp/raft-mdb.png)](https://travis-ci.org/hashicorp/raft-mdb) +raft-mdb ======== This repository provides the `raftmdb` package. The package exports the `MDBStore` which is an implementation of both a LogStore and StableStore. -It is meant to be used as a backend for the `raft` [package here](https://github.com/hashicorp/raft). +It is meant to be used as a backend for the `raft` [package here](github.com/hashicorp/raft). This implementation uses [LMDB](http://symas.com/mdb/). LMDB has a number of advantages to other embedded databases includes transactions, MVCC, @@ -20,3 +20,4 @@ Documentation The documentation for this package can be found on [Godoc](http://godoc.org/github.com/hashicorp/raft-mdb) here. +based on hashicorp commit version: 70e1c88f4b6fb06fc94cc02109243160a443609d \ No newline at end of file diff --git a/bench_test.go b/bench_test.go deleted file mode 100644 index e4b6367..0000000 --- a/bench_test.go +++ /dev/null @@ -1,87 +0,0 @@ -package raftmdb - -import ( - "github.com/hashicorp/raft/bench" - "os" - "testing" -) - -func BenchmarkMDBStore_FirstIndex(b *testing.B) { - dir, store := MDBTestStore(b) - defer store.Close() - defer os.RemoveAll(dir) - - raftbench.FirstIndex(b, store) -} - -func BenchmarkMDBStore_LastIndex(b *testing.B) { - dir, store := MDBTestStore(b) - defer store.Close() - defer os.RemoveAll(dir) - - raftbench.LastIndex(b, store) -} - -func BenchmarkMDBStore_GetLog(b *testing.B) { - dir, store := MDBTestStore(b) - defer store.Close() - defer os.RemoveAll(dir) - - raftbench.GetLog(b, store) -} - -func BenchmarkMDBStore_StoreLog(b *testing.B) { - dir, store := MDBTestStore(b) - defer store.Close() - defer os.RemoveAll(dir) - - raftbench.StoreLog(b, store) -} - -func BenchmarkMDBStore_StoreLogs(b *testing.B) { - dir, store := MDBTestStore(b) - defer store.Close() - defer os.RemoveAll(dir) - - raftbench.StoreLogs(b, store) -} - -func BenchmarkMDBStore_DeleteRange(b *testing.B) { - dir, store := MDBTestStore(b) - defer store.Close() - defer os.RemoveAll(dir) - - raftbench.DeleteRange(b, store) -} - -func BenchmarkMDBStore_Set(b *testing.B) { - dir, store := MDBTestStore(b) - defer store.Close() - defer os.RemoveAll(dir) - - raftbench.Set(b, store) -} - -func BenchmarkMDBStore_Get(b *testing.B) { - dir, store := MDBTestStore(b) - defer store.Close() - defer os.RemoveAll(dir) - - raftbench.Get(b, store) -} - -func BenchmarkMDBStore_SetUint64(b *testing.B) { - dir, store := MDBTestStore(b) - defer store.Close() - defer os.RemoveAll(dir) - - raftbench.SetUint64(b, store) -} - -func BenchmarkMDBStore_GetUint64(b *testing.B) { - dir, store := MDBTestStore(b) - defer store.Close() - defer os.RemoveAll(dir) - - raftbench.GetUint64(b, store) -} diff --git a/mdb_store.go b/mdb_store.go index 8baddbd..5505c80 100644 --- a/mdb_store.go +++ b/mdb_store.go @@ -2,10 +2,12 @@ package raftmdb import ( "fmt" - "github.com/armon/gomdb" - "github.com/hashicorp/raft" "os" "path/filepath" + + "github.com/armon/gomdb" + "github.com/stackengine/raft" + "github.com/ugorji/go/codec" ) const ( @@ -23,19 +25,20 @@ type MDBStore struct { env *mdb.Env path string maxSize uint64 + mh *codec.MsgpackHandle } // NewMDBStore returns a new MDBStore and potential // error. Requres a base directory from which to operate. // Uses the default maximum size. -func NewMDBStore(base string) (*MDBStore, error) { - return NewMDBStoreWithSize(base, 0) +func NewMDBStore(mh *codec.MsgpackHandle, base string) (*MDBStore, error) { + return NewMDBStoreWithSize(mh, base, 0) } // NewMDBStore returns a new MDBStore and potential // error. Requres a base directory from which to operate, // and a maximum size. If maxSize is not 0, a default value is used. -func NewMDBStoreWithSize(base string, maxSize uint64) (*MDBStore, error) { +func NewMDBStoreWithSize(mh *codec.MsgpackHandle, base string, maxSize uint64) (*MDBStore, error) { // Get the paths path := filepath.Join(base, mdbPath) if err := os.MkdirAll(path, 0755); err != nil { @@ -58,6 +61,7 @@ func NewMDBStoreWithSize(base string, maxSize uint64) (*MDBStore, error) { env: env, path: path, maxSize: maxSize, + mh: mh, } // Initialize the db @@ -194,7 +198,7 @@ func (m *MDBStore) GetLog(index uint64, logOut *raft.Log) error { } // Convert the value to a log - return decodeMsgPack(val, logOut) + return decodeMsgPack(m.mh, val, logOut) } // Stores a log entry @@ -213,7 +217,7 @@ func (m *MDBStore) StoreLogs(logs []*raft.Log) error { for _, log := range logs { // Convert to an on-disk format key := uint64ToBytes(log.Index) - val, err := encodeMsgPack(log) + val, err := encodeMsgPack(m.mh, log) if err != nil { tx.Abort() return err @@ -251,7 +255,7 @@ DELETE: return tx.Commit() } -// innerDeleteRange does a single pass to delete the indexes (inclusively) +// innerDeleteRange does a singel pass to delete the indexes (inclusively) func (m *MDBStore) innerDeleteRange(tx *mdb.Txn, dbis []mdb.DBI, minIdx, maxIdx uint64) (num int, err error) { // Open a cursor cursor, err := tx.CursorOpen(dbis[0]) @@ -275,7 +279,7 @@ func (m *MDBStore) innerDeleteRange(tx *mdb.Txn, dbis []mdb.DBI, minIdx, maxIdx } else { key, _, err = cursor.Get(nil, mdb.NEXT) } - if err == mdb.NotFound || len(key) == 0 { + if err == mdb.NotFound { break } else if err != nil { return num, err @@ -300,6 +304,19 @@ func (m *MDBStore) innerDeleteRange(tx *mdb.Txn, dbis []mdb.DBI, minIdx, maxIdx return num, nil } +func (m *MDBStore) Drop() error { + tx, dbis, err := m.startTxn(false, dbConf) + if err != nil { + return err + } + + if err := tx.Drop(dbis[0], 0); err != nil { + tx.Abort() + return err + } + return tx.Commit() +} + // Set a K/V pair func (m *MDBStore) Set(key []byte, val []byte) error { // Start write txn diff --git a/mdb_store_test.go b/mdb_store_test.go index b8adff5..0d16553 100644 --- a/mdb_store_test.go +++ b/mdb_store_test.go @@ -2,13 +2,17 @@ package raftmdb import ( "bytes" - "github.com/hashicorp/raft" "io/ioutil" "os" "testing" + + "github.com/stackengine/raft" + "github.com/ugorji/go/codec" ) -func MDBTestStore(t testing.TB) (string, *MDBStore) { +var Mh = &codec.MsgpackHandle{RawToString: true, WriteExt: true} + +func MDBTestStore(t *testing.T) (string, *MDBStore) { // Create a test dir dir, err := ioutil.TempDir("", "raft") if err != nil { @@ -16,7 +20,7 @@ func MDBTestStore(t testing.TB) (string, *MDBStore) { } // New level - store, err := NewMDBStore(dir) + store, err := NewMDBStore(Mh, dir) if err != nil { t.Fatalf("err: %v ", err) } @@ -41,7 +45,7 @@ func TestMDB_SetGet(t *testing.T) { defer os.RemoveAll(dir) // New level - l, err := NewMDBStore(dir) + l, err := NewMDBStore(Mh, dir) if err != nil { t.Fatalf("err: %v ", err) } @@ -69,6 +73,47 @@ func TestMDB_SetGet(t *testing.T) { } } +func TestMDB_Drop(t *testing.T) { + // Create a test dir + dir, err := ioutil.TempDir("", "raft") + if err != nil { + t.Fatalf("err: %v ", err) + } + defer os.RemoveAll(dir) + + // New level + l, err := NewMDBStore(Mh, dir) + if err != nil { + t.Fatalf("err: %v ", err) + } + defer l.Close() + + key := []byte("foobar") + val := []byte("this is a test value") + if err := l.Set(key, val); err != nil { + t.Fatalf("err: %v ", err) + } + + out, err := l.Get(key) + if err != nil { + t.Fatalf("err: %v ", err) + } + + if bytes.Compare(val, out) != 0 { + t.Fatalf("did not get result back: %v %v", val, out) + } + + // now drop the db + err = l.Drop() + if err != nil { + t.Fatalf("err: %v ", err) + } + _, err = l.Get(key) + if err.Error() != "not found" { + t.Fatalf("err: %v ", err) + } +} + func TestMDB_SetGetUint64(t *testing.T) { // Create a test dir dir, err := ioutil.TempDir("", "raft") @@ -78,7 +123,7 @@ func TestMDB_SetGetUint64(t *testing.T) { defer os.RemoveAll(dir) // New level - l, err := NewMDBStore(dir) + l, err := NewMDBStore(Mh, dir) if err != nil { t.Fatalf("err: %v ", err) } @@ -123,7 +168,7 @@ func TestMDB_Logs(t *testing.T) { defer os.RemoveAll(dir) // New level - l, err := NewMDBStore(dir) + l, err := NewMDBStore(Mh, dir) if err != nil { t.Fatalf("err: %v ", err) } @@ -218,8 +263,8 @@ func TestMDB_Logs(t *testing.T) { // Verify they are all deleted for i := 5; i <= 20; i++ { - if err := l.GetLog(uint64(i), &out); err != raft.ErrLogNotFound { - t.Fatalf("err: %v ", err) + if err := l.GetLog(uint64(i), &out); err.Error() != "log not found" { + t.Fatalf("err: %#v expecting: %#v", err, raft.ErrLogNotFound) } } diff --git a/util.go b/util.go index 06a4641..032c13b 100644 --- a/util.go +++ b/util.go @@ -3,22 +3,21 @@ package raftmdb import ( "bytes" "encoding/binary" - "github.com/hashicorp/go-msgpack/codec" + + "github.com/ugorji/go/codec" ) // Decode reverses the encode operation on a byte slice input -func decodeMsgPack(buf []byte, out interface{}) error { +func decodeMsgPack(mh *codec.MsgpackHandle, buf []byte, out interface{}) error { r := bytes.NewBuffer(buf) - hd := codec.MsgpackHandle{} - dec := codec.NewDecoder(r, &hd) + dec := codec.NewDecoder(r, mh) return dec.Decode(out) } // Encode writes an encoded object to a new bytes buffer -func encodeMsgPack(in interface{}) (*bytes.Buffer, error) { +func encodeMsgPack(mh *codec.MsgpackHandle, in interface{}) (*bytes.Buffer, error) { buf := bytes.NewBuffer(nil) - hd := codec.MsgpackHandle{} - enc := codec.NewEncoder(buf, &hd) + enc := codec.NewEncoder(buf, mh) err := enc.Encode(in) return buf, err }