restructure db accessors

server/app/setup.go

@@ -2,7 +2,7 @@ package app
 
 import (
 	"mapserver/colormapping"
-	"mapserver/db"
+	"mapserver/db/sqlite"
 	"mapserver/eventbus"
 	"mapserver/mapblockaccessor"
 	"mapserver/mapblockrenderer"
@@ -30,7 +30,7 @@ func Setup(p params.ParamsType, cfg *Config) *App {
 
 	switch a.Worldconfig.Backend {
 	case worldconfig.BACKEND_SQLITE3:
-		a.Blockdb, err = db.NewSqliteAccessor("map.sqlite")
+		a.Blockdb, err = sqlite.New("map.sqlite")
 		if err != nil {
 			panic(err)
 		}

server/db/postgres/postgres.go

@@ -0,0 +1,133 @@
+package postgres
+
+import (
+	"database/sql"
+	_ "github.com/mattn/go-sqlite3"
+	"mapserver/coords"
+	"mapserver/db"
+)
+
+
+type PostgresAccessor struct {
+	db       *sql.DB
+}
+
+func (db *PostgresAccessor) Migrate() error {
+	return nil
+}
+
+func convertRows(pos int64, data []byte, mtime int64) db.Block {
+	c := coords.PlainToCoord(pos)
+	return db.Block{Pos: c, Data: data, Mtime: mtime}
+}
+
+
+func (this *PostgresAccessor) FindBlocksByMtime(gtmtime int64, limit int) ([]db.Block, error) {
+	blocks := make([]db.Block, 0)
+
+	rows, err := this.db.Query(getBlocksByMtimeQuery, gtmtime, limit)
+	if err != nil {
+		return nil, err
+	}
+
+	defer rows.Close()
+
+	for rows.Next() {
+		var pos int64
+		var data []byte
+		var mtime int64
+
+		err = rows.Scan(&pos, &data, &mtime)
+		if err != nil {
+			return nil, err
+		}
+
+		mb := convertRows(pos, data, mtime)
+		blocks = append(blocks, mb)
+	}
+
+	return blocks, nil
+}
+
+func (this *PostgresAccessor) FindLegacyBlocksByPos(lastpos coords.MapBlockCoords, limit int) ([]db.Block, error) {
+	blocks := make([]db.Block, 0)
+	pc := coords.CoordToPlain(lastpos)
+
+	rows, err := this.db.Query(getLastBlockQuery, pc, limit)
+	if err != nil {
+		return nil, err
+	}
+
+	defer rows.Close()
+
+	for rows.Next() {
+		var pos int64
+		var data []byte
+		var mtime int64
+
+		err = rows.Scan(&pos, &data, &mtime)
+		if err != nil {
+			return nil, err
+		}
+
+		mb := convertRows(pos, data, mtime)
+		blocks = append(blocks, mb)
+	}
+
+	return blocks, nil
+}
+
+
+func (this *PostgresAccessor) CountBlocks(frommtime, tomtime int64) (int, error) {
+	rows, err := this.db.Query(countBlocksQuery, frommtime, tomtime)
+	if err != nil {
+		return 0, err
+	}
+
+	defer rows.Close()
+
+	if rows.Next() {
+		var count int64
+
+		err = rows.Scan(&count)
+		if err != nil {
+			return 0, err
+		}
+
+		return int(count), nil
+	}
+
+	return 0, nil
+}
+
+
+func (this *PostgresAccessor) GetBlock(pos coords.MapBlockCoords) (*db.Block, error) {
+	ppos := coords.CoordToPlain(pos)
+
+	rows, err := this.db.Query(getBlockQuery, ppos)
+	if err != nil {
+		return nil, err
+	}
+
+	defer rows.Close()
+
+	if rows.Next() {
+		var pos int64
+		var data []byte
+		var mtime int64
+
+		err = rows.Scan(&pos, &data, &mtime)
+		if err != nil {
+			return nil, err
+		}
+
+		mb := convertRows(pos, data, mtime)
+		return &mb, nil
+	}
+
+	return nil, nil
+}
+
+func NewPostgresAccessor(filename string) (*PostgresAccessor, error) {
+	return nil, nil
+}

server/db/postgres/sql.go

@@ -0,0 +1,42 @@
+package postgres
+
+const migrateScript = `
+alter table blocks add mtime integer default 0;
+create index blocks_mtime on blocks(mtime);
+
+CREATE TRIGGER update_blocks_mtime_insert after insert on blocks for each row
+begin
+update blocks set mtime = strftime('%s', 'now') where pos = new.pos;
+end;
+
+CREATE TRIGGER update_blocks_mtime_update after update on blocks for each row
+begin
+update blocks set mtime = strftime('%s', 'now') where pos = old.pos;
+end;
+`
+
+const getBlocksByMtimeQuery = `
+select pos,data,mtime
+from blocks b
+where b.mtime > ?
+order by b.mtime asc
+limit ?
+`
+
+const getLastBlockQuery = `
+select pos,data,mtime
+from blocks b
+where b.mtime = 0
+and b.pos > ?
+order by b.pos asc, b.mtime asc
+limit ?
+`
+
+const countBlocksQuery = `
+select count(*) from blocks b where b.mtime >= ? and b.mtime <= ?
+`
+
+
+const getBlockQuery = `
+select pos,data,mtime from blocks b where b.pos = ?
+`

server/db/sqlite/logger.go

@@ -0,0 +1,11 @@
+package sqlite
+
+import (
+	"github.com/sirupsen/logrus"
+)
+
+var log *logrus.Entry
+
+func init() {
+	log = logrus.WithFields(logrus.Fields{"prefix": "sqlite-db"})
+}

server/db/sqlite/sql.go

@@ -0,0 +1,42 @@
+package sqlite
+
+const migrateScript = `
+alter table blocks add mtime integer default 0;
+create index blocks_mtime on blocks(mtime);
+
+CREATE TRIGGER update_blocks_mtime_insert after insert on blocks for each row
+begin
+update blocks set mtime = strftime('%s', 'now') where pos = new.pos;
+end;
+
+CREATE TRIGGER update_blocks_mtime_update after update on blocks for each row
+begin
+update blocks set mtime = strftime('%s', 'now') where pos = old.pos;
+end;
+`
+
+const getBlocksByMtimeQuery = `
+select pos,data,mtime
+from blocks b
+where b.mtime > ?
+order by b.mtime asc
+limit ?
+`
+
+const getLastBlockQuery = `
+select pos,data,mtime
+from blocks b
+where b.mtime = 0
+and b.pos > ?
+order by b.pos asc, b.mtime asc
+limit ?
+`
+
+const countBlocksQuery = `
+select count(*) from blocks b where b.mtime >= ? and b.mtime <= ?
+`
+
+
+const getBlockQuery = `
+select pos,data,mtime from blocks b where b.pos = ?
+`

server/db/sqlite/sqlite.go

@@ -1,10 +1,11 @@
-package db
+package sqlite
 
 import (
 	"database/sql"
 	_ "github.com/mattn/go-sqlite3"
 	"github.com/sirupsen/logrus"
 	"mapserver/coords"
+	"mapserver/db"
 	"time"
 )
 
@@ -13,24 +14,6 @@ sqlite extract: https://stackoverflow.com/questions/15448373/how-to-dump-a-file-
 sqlite3 my.db "SELECT writefile('object0.gz', MyBlob) FROM MyTable WHERE id = 1"
 */
 
-const migrateScript = `
-alter table blocks add mtime integer default 0;
-create index blocks_mtime on blocks(mtime);
-
-CREATE TRIGGER update_blocks_mtime_insert after insert on blocks for each row
-begin
-update blocks set mtime = strftime('%s', 'now') where pos = new.pos;
-end;
-
-CREATE TRIGGER update_blocks_mtime_update after update on blocks for each row
-begin
-update blocks set mtime = strftime('%s', 'now') where pos = old.pos;
-end;
-`
-
-//TODO: initial run: https://stackoverflow.com/questions/14468586/efficient-paging-in-sqlite-with-millions-of-records
-//TODO: postgres test
-
 type Sqlite3Accessor struct {
 	db       *sql.DB
 	filename string
@@ -67,23 +50,16 @@ func (db *Sqlite3Accessor) Migrate() error {
 	return nil
 }
 
-func convertRows(pos int64, data []byte, mtime int64) Block {
+func convertRows(pos int64, data []byte, mtime int64) db.Block {
 	c := coords.PlainToCoord(pos)
-	return Block{Pos: c, Data: data, Mtime: mtime}
+	return db.Block{Pos: c, Data: data, Mtime: mtime}
 }
 
-const getBlocksByMtimeQuery = `
-select pos,data,mtime
-from blocks b
-where b.mtime > ?
-order by b.mtime asc
-limit ?
-`
 
-func (db *Sqlite3Accessor) FindBlocksByMtime(gtmtime int64, limit int) ([]Block, error) {
+func (this *Sqlite3Accessor) FindBlocksByMtime(gtmtime int64, limit int) ([]db.Block, error) {
-	blocks := make([]Block, 0)
+	blocks := make([]db.Block, 0)
 
-	rows, err := db.db.Query(getBlocksByMtimeQuery, gtmtime, limit)
+	rows, err := this.db.Query(getBlocksByMtimeQuery, gtmtime, limit)
 	if err != nil {
 		return nil, err
 	}
@@ -107,20 +83,12 @@ func (db *Sqlite3Accessor) FindBlocksByMtime(gtmtime int64, limit int) ([]Block,
 	return blocks, nil
 }
 
-const getLastBlockQuery = `
-select pos,data,mtime
-from blocks b
-where b.mtime = 0
-and b.pos > ?
-order by b.pos asc, b.mtime asc
-limit ?
-`
 
-func (db *Sqlite3Accessor) FindLegacyBlocksByPos(lastpos coords.MapBlockCoords, limit int) ([]Block, error) {
+func (this *Sqlite3Accessor) FindLegacyBlocksByPos(lastpos coords.MapBlockCoords, limit int) ([]db.Block, error) {
-	blocks := make([]Block, 0)
+	blocks := make([]db.Block, 0)
 	pc := coords.CoordToPlain(lastpos)
 
-	rows, err := db.db.Query(getLastBlockQuery, pc, limit)
+	rows, err := this.db.Query(getLastBlockQuery, pc, limit)
 	if err != nil {
 		return nil, err
 	}
@@ -144,9 +112,7 @@ func (db *Sqlite3Accessor) FindLegacyBlocksByPos(lastpos coords.MapBlockCoords,
 	return blocks, nil
 }
 
-const countBlocksQuery = `
+
-select count(*) from blocks b where b.mtime >= ? and b.mtime <= ?
-`
 
 func (db *Sqlite3Accessor) CountBlocks(frommtime, tomtime int64) (int, error) {
 	rows, err := db.db.Query(countBlocksQuery, frommtime, tomtime)
@@ -170,11 +136,7 @@ func (db *Sqlite3Accessor) CountBlocks(frommtime, tomtime int64) (int, error) {
 	return 0, nil
 }
 
-const getBlockQuery = `
+func (db *Sqlite3Accessor) GetBlock(pos coords.MapBlockCoords) (*db.Block, error) {
-select pos,data,mtime from blocks b where b.pos = ?
-`
-
-func (db *Sqlite3Accessor) GetBlock(pos coords.MapBlockCoords) (*Block, error) {
 	ppos := coords.CoordToPlain(pos)
 
 	rows, err := db.db.Query(getBlockQuery, ppos)
@@ -201,7 +163,7 @@ func (db *Sqlite3Accessor) GetBlock(pos coords.MapBlockCoords) (*Block, error) {
 	return nil, nil
 }
 
-func NewSqliteAccessor(filename string) (*Sqlite3Accessor, error) {
+func New(filename string) (*Sqlite3Accessor, error) {
 	db, err := sql.Open("sqlite3", filename+"?mode=ro&_timeout=2000")
 	if err != nil {
 		return nil, err

server/db/sqlite/sqlite_test.go

@@ -1,4 +1,4 @@
-package db
+package sqlite
 
 import (
 	_ "github.com/mattn/go-sqlite3"
@@ -17,7 +17,7 @@ func TestMigrateEmpty(t *testing.T) {
 	defer os.Remove(tmpfile.Name())
 
 	testutils.CreateEmptyDatabase(tmpfile.Name())
-	a, err := NewSqliteAccessor(tmpfile.Name())
+	a, err := New(tmpfile.Name())
 	if err != nil {
 		panic(err)
 	}
@@ -35,7 +35,7 @@ func TestMigrate(t *testing.T) {
 	defer os.Remove(tmpfile.Name())
 
 	testutils.CreateEmptyDatabase(tmpfile.Name())
-	a, err := NewSqliteAccessor(tmpfile.Name())
+	a, err := New(tmpfile.Name())
 	if err != nil {
 		panic(err)
 	}
@@ -53,7 +53,7 @@ func TestMigrateAndQuery(t *testing.T) {
 	defer os.Remove(tmpfile.Name())
 
 	testutils.CreateTestDatabase(tmpfile.Name())
-	a, err := NewSqliteAccessor(tmpfile.Name())
+	a, err := New(tmpfile.Name())
 	if err != nil {
 		panic(err)
 	}
@@ -83,7 +83,7 @@ func TestMigrateAndQueryCount(t *testing.T) {
 	defer os.Remove(tmpfile.Name())
 
 	testutils.CreateTestDatabase(tmpfile.Name())
-	a, err := NewSqliteAccessor(tmpfile.Name())
+	a, err := New(tmpfile.Name())
 	if err != nil {
 		panic(err)
 	}

server/mapblockaccessor/mapblockaccessor_test.go

@@ -4,7 +4,7 @@ import (
 	"github.com/sirupsen/logrus"
 	"io/ioutil"
 	"mapserver/coords"
-	"mapserver/db"
+	"mapserver/db/sqlite"
 	"mapserver/testutils"
 	"os"
 	"testing"
@@ -20,7 +20,7 @@ func TestSimpleAccess(t *testing.T) {
 	defer os.Remove(tmpfile.Name())
 	testutils.CreateTestDatabase(tmpfile.Name())
 
-	a, err := db.NewSqliteAccessor(tmpfile.Name())
+	a, err := sqlite.New(tmpfile.Name())
 	if err != nil {
 		panic(err)
 	}

server/mapblockrenderer/renderer_test.go

@@ -6,7 +6,7 @@ import (
 	"io/ioutil"
 	"mapserver/colormapping"
 	"mapserver/coords"
-	"mapserver/db"
+	"mapserver/db/sqlite"
 	"mapserver/layer"
 	"mapserver/mapblockaccessor"
 	"mapserver/testutils"
@@ -33,7 +33,7 @@ func TestSimpleRender(t *testing.T) {
 	defer os.Remove(tmpfile.Name())
 	testutils.CreateTestDatabase(tmpfile.Name())
 
-	a, err := db.NewSqliteAccessor(tmpfile.Name())
+	a, err := sqlite.New(tmpfile.Name())
 	if err != nil {
 		panic(err)
 	}

server/tilerenderer/renderer_test.go

@@ -5,7 +5,7 @@ import (
 	"io/ioutil"
 	"mapserver/colormapping"
 	"mapserver/coords"
-	"mapserver/db"
+	"mapserver/db/sqlite"
 	"mapserver/layer"
 	"mapserver/mapblockaccessor"
 	"mapserver/mapblockrenderer"
@@ -28,7 +28,7 @@ func TestTileRender(t *testing.T) {
 	defer os.Remove(tmpfile.Name())
 	testutils.CreateTestDatabase(tmpfile.Name())
 
-	a, err := db.NewSqliteAccessor(tmpfile.Name())
+	a, err := sqlite.New(tmpfile.Name())
 	if err != nil {
 		panic(err)
 	}