orbitdb/orbit-db
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Use LevelDB as the local persistence cache

Browse Source
This commit is contained in:
haad 2017-12-06 13:08:31 +01:00
parent 63f3b4d73c
commit 6df41f4fad
12 changed files with 494 additions and 473 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
benchmarks/browser/benchmark-add.html
View File

@ -12,7 +12,7 @@
<pre id="output"></pre>
<script type="text/javascript" src="../../dist/orbitdb.min.js" charset="utf-8"></script>
<script type="text/javascript" src="../../node_modules/ipfs/dist/index.min.js" charset="utf-8"></script>
<script type="text/javascript" src="../../node_modules/ipfs/dist/index.js" charset="utf-8"></script>
<script type="text/javascript">
let ipfs
@ -33,7 +33,7 @@
let run = (() => {
ipfs = new Ipfs({
repo: '/orbitdb/benchmarks/browser/benchmark-add',
repo: '/orbitdb/benchmarks/browser/benchmark-add/0.27.0',
start: false,
EXPERIMENTAL: {
pubsub: true,

6
conf/webpack.config.js
View File

@ -15,6 +15,7 @@ module.exports = {
devtool: 'none',
externals: {
fs: '{}',
mkdirp: '{}',
},
node: {
console: false,
@ -32,7 +33,10 @@ module.exports = {
modules: [
'node_modules',
path.resolve(__dirname, '../node_modules')
]
],
alias: {
leveldown: 'level-js',
},
},
resolveLoader: {
modules: [

6
conf/webpack.debug.config.js
View File

@ -14,6 +14,7 @@ module.exports = {
devtool: 'none',
externals: {
fs: '{}',
mkdirp: '{}',
},
node: {
console: false,
@ -25,7 +26,10 @@ module.exports = {
modules: [
'node_modules',
path.resolve(__dirname, '../node_modules')
]
],
alias: {
leveldown: 'level-js',
},
},
resolveLoader: {
modules: [

4
conf/webpack.example.config.js
View File

@ -24,6 +24,7 @@ module.exports = {
},
externals: {
fs: '{}',
mkdirp: '{}',
},
plugins: [
new webpack.DefinePlugin({
@ -38,6 +39,9 @@ module.exports = {
'node_modules',
path.resolve(__dirname, '../node_modules')
],
alias: {
leveldown: 'level-js',
},
},
resolveLoader: {
modules: [

37
dist/es5/OrbitDB.js vendored
View File

@ -178,10 +178,12 @@ var OrbitDB = function () {
await accessController.load(options.accessControllerAddress);
}
var cache = await this._loadCache(this.directory, address);
var opts = (0, _assign2.default)({ replicate: true }, options, {
accessController: accessController,
keystore: this.keystore,
cache: this._cache
cache: cache
});
var store = new Store(this._ipfs, this.id, address, opts);
@ -268,6 +270,8 @@ var OrbitDB = function () {
value: async function create(name, type) {
var options = arguments.length > 2 && arguments[2] !== undefined ? arguments[2] : {};
logger.debug('create()');
if (!OrbitDB.isValidType(type)) throw new Error('Invalid database type \'' + type + '\'');
// The directory to look databases from can be passed in as an option
@ -305,13 +309,13 @@ var OrbitDB = function () {
);var dbAddress = OrbitDBAddress.parse(path.join('/orbitdb', manifestHash, name)
// // Load local cache
);var haveManifest = await this._loadCache(directory, dbAddress).then(function (cache) {
return cache.get(dbAddress.toString());
}).then(function (localData) {
return localData && localData.manifest;
);var haveDB = await this._loadCache(directory, dbAddress).then(function (cache) {
return cache ? cache.get(path.join(dbAddress.toString(), '_manifest')) : null;
}).then(function (data) {
return data !== undefined && data !== null;
});
if (haveManifest && !options.overwrite) throw new Error('Database \'' + dbAddress + '\' already exists!');
if (haveDB && !options.overwrite) throw new Error('Database \'' + dbAddress + '\' already exists!');
// Save the database locally
await this._saveDBManifest(directory, dbAddress);
@ -336,6 +340,7 @@ var OrbitDB = function () {
value: async function open(address) {
var options = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : {};
logger.debug('open()');
options = (0, _assign2.default)({ localOnly: false, create: false }, options);
logger.debug('Open database \'' + address + '\''
@ -361,9 +366,9 @@ var OrbitDB = function () {
// Check if we have the database
);var haveDB = await this._loadCache(directory, dbAddress).then(function (cache) {
return cache.get(dbAddress.toString());
}).then(function (localData) {
return localData && localData.manifest;
return cache ? cache.get(path.join(dbAddress.toString(), '_manifest')) : null;
}).then(function (data) {
return data !== undefined && data !== null;
});
logger.debug((haveDB ? 'Found' : 'Didn\'t find') + (' database \'' + dbAddress + '\'')
@ -398,11 +403,11 @@ var OrbitDB = function () {
}, {
key: '_saveDBManifest',
value: async function _saveDBManifest(directory, dbAddress) {
var cache = await this._loadCache(directory, dbAddress);
var localData = (0, _assign2.default)({}, cache.get(dbAddress.toString()), {
manifest: dbAddress.root
});
await cache.set(dbAddress.toString(), localData);
var cache = await this._loadCache(directory, dbAddress
// let localData = Object.assign({}, cache.get(dbAddress.toString()), {
// manifest: dbAddress.root
// })
);await cache.set(path.join(dbAddress.toString(), '_manifest'), dbAddress.root);
logger.debug('Saved manifest to IPFS as \'' + dbAddress.root + '\'');
}
}, {
@ -410,9 +415,7 @@ var OrbitDB = function () {
value: async function _loadCache(directory, dbAddress) {
var cache = void 0;
try {
var cacheFilePath = path.join(dbAddress.root, dbAddress.path);
cache = new Cache(path.join(directory), cacheFilePath);
await cache.load();
cache = await Cache.load(directory, dbAddress);
} catch (e) {
logger.warn("Couldn't load Cache:", e);
}

2
dist/orbitdb.min.js vendored
View File

File diff suppressed because one or more lines are too long

3
examples/browser/example.js
View File

@ -46,7 +46,8 @@ const main = (IPFS, ORBITDB) => {
// Create IPFS instance
const ipfs = new Ipfs({
// repo: '/orbitdb/examples/browser/ipfs' + new Date().getTime(),
repo: '/orbitdb/examples/browser/new/ipfs',
repo: '/orbitdb/examples/browser/new/ipfs/0.27.0',
start: false,
EXPERIMENTAL: {
pubsub: true,
},

837
package-lock.json generated
View File

File diff suppressed because it is too large Load Diff

6
package.json
View File

@ -16,16 +16,18 @@
},
"main": "src/OrbitDB.js",
"dependencies": {
"level": "^2.1.0",
"logplease": "^1.2.14",
"multihashes": "^0.4.12",
"orbit-db-cache": "~0.0.7",
"orbit-db-cache": "orbitdb/orbit-db-cache#feat/leveldb",
"orbit-db-counterstore": "~1.0.3",
"orbit-db-docstore": "~1.0.1",
"orbit-db-eventstore": "~1.0.0",
"orbit-db-feedstore": "~1.0.0",
"orbit-db-keystore": "~0.0.2",
"orbit-db-kvstore": "~1.0.0",
"orbit-db-pubsub": "~0.3.6"
"orbit-db-pubsub": "~0.3.6",
"orbit-db-store": "orbitdb/orbit-db-store#feat/leveldb"
},
"devDependencies": {
"babel-core": "^6.26.0",

31
src/OrbitDB.js
View File

@ -102,10 +102,12 @@ class OrbitDB {
await accessController.load(options.accessControllerAddress)
}
const cache = await this._loadCache(this.directory, address)
const opts = Object.assign({ replicate: true }, options, {
accessController: accessController,
keystore: this.keystore,
cache: this._cache,
cache: cache,
})
const store = new Store(this._ipfs, this.id, address, opts)
@ -177,6 +179,8 @@ class OrbitDB {
}
*/
async create (name, type, options = {}) {
logger.debug(`create()`)
if (!OrbitDB.isValidType(type))
throw new Error(`Invalid database type '${type}'`)
@ -214,11 +218,11 @@ class OrbitDB {
const dbAddress = OrbitDBAddress.parse(path.join('/orbitdb', manifestHash, name))
// // Load local cache
const haveManifest = await this._loadCache(directory, dbAddress)
.then(cache => cache.get(dbAddress.toString()))
.then(localData => localData && localData.manifest)
const haveDB = await this._loadCache(directory, dbAddress)
.then(cache => cache ? cache.get(path.join(dbAddress.toString(), '_manifest')) : null)
.then(data => data !== undefined && data !== null)
if (haveManifest && !options.overwrite)
if (haveDB && !options.overwrite)
throw new Error(`Database '${dbAddress}' already exists!`)
// Save the database locally
@ -240,6 +244,7 @@ class OrbitDB {
}
*/
async open (address, options = {}) {
logger.debug(`open()`)
options = Object.assign({ localOnly: false, create: false }, options)
logger.debug(`Open database '${address}'`)
@ -265,8 +270,8 @@ class OrbitDB {
// Check if we have the database
const haveDB = await this._loadCache(directory, dbAddress)
.then(cache => cache.get(dbAddress.toString()))
.then(localData => localData && localData.manifest)
.then(cache => cache ? cache.get(path.join(dbAddress.toString(), '_manifest')) : null)
.then(data => data !== undefined && data !== null)
logger.debug((haveDB ? 'Found' : 'Didn\'t find') + ` database '${dbAddress}'`)
@ -299,19 +304,17 @@ class OrbitDB {
// Save the database locally
async _saveDBManifest (directory, dbAddress) {
const cache = await this._loadCache(directory, dbAddress)
let localData = Object.assign({}, cache.get(dbAddress.toString()), {
manifest: dbAddress.root
})
await cache.set(dbAddress.toString(), localData)
// let localData = Object.assign({}, cache.get(dbAddress.toString()), {
// manifest: dbAddress.root
// })
await cache.set(path.join(dbAddress.toString(), '_manifest'), dbAddress.root)
logger.debug(`Saved manifest to IPFS as '${dbAddress.root}'`)
}
async _loadCache (directory, dbAddress) {
let cache
try {
const cacheFilePath = path.join(dbAddress.root, dbAddress.path)
cache = new Cache(path.join(directory), cacheFilePath)
await cache.load()
cache = await Cache.load(directory, dbAddress)
} catch (e) {
logger.warn("Couldn't load Cache:", e)
}

2
test/counterdb.test.js
View File

@ -109,7 +109,7 @@ describe('CounterStore', function() {
assert.equal(counter1.value, 30)
assert.equal(counter2.value, 30)
resolve()
}, 1000)
}, 5000)
})
})
})

29
test/create-open.test.js
View File

@ -5,6 +5,8 @@ const fs = require('fs')
const path = require('path')
const rmrf = require('rimraf')
const mapSeries = require('p-map-series')
const levelup = require('levelup')
const leveldown = require('leveldown')
const OrbitDB = require('../src/OrbitDB')
const OrbitDBAddress = require('../src/orbit-db-address')
const config = require('./utils/config')
@ -68,6 +70,7 @@ describe('orbit-db - Create & Open', function() {
assert.equal(err, `Error: Database '${db.address}' already exists!`)
})
it('throws an error if database type doesn\'t match', async () => {
let err, log, kv
try {
@ -83,7 +86,8 @@ describe('orbit-db - Create & Open', function() {
describe('Success', function() {
before(async () => {
db = await orbitdb.create('second', 'feed')
localDataPath = path.join(dbPath, db.address.root, db.address.path + '.orbitdb')
localDataPath = path.join(dbPath, db.address.root, db.address.path)
await db.close()
})
it('creates a feed database', async () => {
@ -97,14 +101,27 @@ describe('orbit-db - Create & Open', function() {
})
it('saves the database locally', async () => {
console.log(localDataPath)
assert.equal(fs.existsSync(localDataPath), true)
})
it('saves database manifest reference locally', async () => {
const buffer = JSON.parse(fs.readFileSync(localDataPath))
const data = buffer[db.address.toString()]
assert.equal(data.manifest, db.address.root)
assert.equal(db.address.path, 'second')
const manifestHash = db.address.root
const address = db.address.toString()
levelup(leveldown(localDataPath), (err, db) => {
if (err) {
assert.equal(err, null)
}
db.get(address + '/_manifest', (err, value) => {
if (err) {
assert.equal(err, null)
}
const data = JSON.parse(value || '{}')
assert.equal(data, manifestHash)
})
})
})
it('saves database manifest file locally', async () => {
@ -120,7 +137,7 @@ describe('orbit-db - Create & Open', function() {
it('can pass local database directory as an option', async () => {
const dir = './orbitdb/tests/another-feed'
db = await orbitdb.create('third', 'feed', { directory: dir })
localDataPath = path.join(dir, db.address.root, db.address.path + '.orbitdb')
localDataPath = path.join(dir, db.address.root, db.address.path)
assert.equal(fs.existsSync(localDataPath), true)
})