Use LevelDB as the local persistence cache

benchmarks/browser/benchmark-add.html

@@ -12,7 +12,7 @@
     <pre id="output"></pre>
 
     <script type="text/javascript" src="../../dist/orbitdb.min.js" charset="utf-8"></script>
-    <script type="text/javascript" src="../../node_modules/ipfs/dist/index.min.js" charset="utf-8"></script>
+    <script type="text/javascript" src="../../node_modules/ipfs/dist/index.js" charset="utf-8"></script>
 
     <script type="text/javascript">
       let ipfs
@@ -33,7 +33,7 @@
 
       let run = (() => {
         ipfs = new Ipfs({ 
-          repo: '/orbitdb/benchmarks/browser/benchmark-add',
+          repo: '/orbitdb/benchmarks/browser/benchmark-add/0.27.0',
           start: false,
           EXPERIMENTAL: {
             pubsub: true,

conf/webpack.config.js

@@ -15,6 +15,7 @@ module.exports = {
   devtool: 'none',
   externals: {
     fs: '{}',
+    mkdirp: '{}',
   },
   node: {
     console: false,
@@ -32,7 +33,10 @@ module.exports = {
     modules: [
       'node_modules',
       path.resolve(__dirname, '../node_modules')
-    ]
+    ],
+    alias: {
+      leveldown: 'level-js',
+    },
   },
   resolveLoader: {
     modules: [

conf/webpack.debug.config.js

@@ -14,6 +14,7 @@ module.exports = {
   devtool: 'none',
   externals: {
     fs: '{}',
+    mkdirp: '{}',
   },
   node: {
     console: false,
@@ -25,7 +26,10 @@ module.exports = {
     modules: [
       'node_modules',
       path.resolve(__dirname, '../node_modules')
-    ]
+    ],
+    alias: {
+      leveldown: 'level-js',
+    },
   },
   resolveLoader: {
     modules: [

conf/webpack.example.config.js

@@ -24,6 +24,7 @@ module.exports = {
   },
   externals: {
     fs: '{}',
+    mkdirp: '{}',
   },
   plugins: [
     new webpack.DefinePlugin({
@@ -38,6 +39,9 @@ module.exports = {
       'node_modules',
       path.resolve(__dirname, '../node_modules')
     ],
+    alias: {
+      leveldown: 'level-js',
+    },
   },
   resolveLoader: {
     modules: [

dist/es5/OrbitDB.js

@@ -178,10 +178,12 @@ var OrbitDB = function () {
         await accessController.load(options.accessControllerAddress);
       }
 
+      var cache = await this._loadCache(this.directory, address);
+
       var opts = (0, _assign2.default)({ replicate: true }, options, {
         accessController: accessController,
         keystore: this.keystore,
-        cache: this._cache
+        cache: cache
       });
 
       var store = new Store(this._ipfs, this.id, address, opts);
@@ -268,6 +270,8 @@ var OrbitDB = function () {
     value: async function create(name, type) {
       var options = arguments.length > 2 && arguments[2] !== undefined ? arguments[2] : {};
 
+      logger.debug('create()');
+
       if (!OrbitDB.isValidType(type)) throw new Error('Invalid database type \'' + type + '\'');
 
       // The directory to look databases from can be passed in as an option
@@ -305,13 +309,13 @@ var OrbitDB = function () {
       );var dbAddress = OrbitDBAddress.parse(path.join('/orbitdb', manifestHash, name)
 
       // // Load local cache
-      );var haveManifest = await this._loadCache(directory, dbAddress).then(function (cache) {
+      );var haveDB = await this._loadCache(directory, dbAddress).then(function (cache) {
-        return cache.get(dbAddress.toString());
+        return cache ? cache.get(path.join(dbAddress.toString(), '_manifest')) : null;
-      }).then(function (localData) {
+      }).then(function (data) {
-        return localData && localData.manifest;
+        return data !== undefined && data !== null;
       });
 
-      if (haveManifest && !options.overwrite) throw new Error('Database \'' + dbAddress + '\' already exists!');
+      if (haveDB && !options.overwrite) throw new Error('Database \'' + dbAddress + '\' already exists!');
 
       // Save the database locally
       await this._saveDBManifest(directory, dbAddress);
@@ -336,6 +340,7 @@ var OrbitDB = function () {
     value: async function open(address) {
       var options = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : {};
 
+      logger.debug('open()');
       options = (0, _assign2.default)({ localOnly: false, create: false }, options);
       logger.debug('Open database \'' + address + '\''
 
@@ -361,9 +366,9 @@ var OrbitDB = function () {
 
       // Check if we have the database
       );var haveDB = await this._loadCache(directory, dbAddress).then(function (cache) {
-        return cache.get(dbAddress.toString());
+        return cache ? cache.get(path.join(dbAddress.toString(), '_manifest')) : null;
-      }).then(function (localData) {
+      }).then(function (data) {
-        return localData && localData.manifest;
+        return data !== undefined && data !== null;
       });
 
       logger.debug((haveDB ? 'Found' : 'Didn\'t find') + (' database \'' + dbAddress + '\'')
@@ -398,11 +403,11 @@ var OrbitDB = function () {
   }, {
     key: '_saveDBManifest',
     value: async function _saveDBManifest(directory, dbAddress) {
-      var cache = await this._loadCache(directory, dbAddress);
+      var cache = await this._loadCache(directory, dbAddress
-      var localData = (0, _assign2.default)({}, cache.get(dbAddress.toString()), {
+      // let localData = Object.assign({}, cache.get(dbAddress.toString()), {
-        manifest: dbAddress.root
+      //   manifest: dbAddress.root
-      });
+      // })
-      await cache.set(dbAddress.toString(), localData);
+      );await cache.set(path.join(dbAddress.toString(), '_manifest'), dbAddress.root);
       logger.debug('Saved manifest to IPFS as \'' + dbAddress.root + '\'');
     }
   }, {
@@ -410,9 +415,7 @@ var OrbitDB = function () {
     value: async function _loadCache(directory, dbAddress) {
       var cache = void 0;
       try {
-        var cacheFilePath = path.join(dbAddress.root, dbAddress.path);
+        cache = await Cache.load(directory, dbAddress);
-        cache = new Cache(path.join(directory), cacheFilePath);
-        await cache.load();
       } catch (e) {
         logger.warn("Couldn't load Cache:", e);
       }

examples/browser/example.js

@@ -46,7 +46,8 @@ const main = (IPFS, ORBITDB) => {
   // Create IPFS instance
   const ipfs = new Ipfs({
     // repo: '/orbitdb/examples/browser/ipfs' + new Date().getTime(),
-    repo: '/orbitdb/examples/browser/new/ipfs',
+    repo: '/orbitdb/examples/browser/new/ipfs/0.27.0',
+    start: false,
     EXPERIMENTAL: {
       pubsub: true,
     },

package.json

@@ -16,16 +16,18 @@
   },
   "main": "src/OrbitDB.js",
   "dependencies": {
+    "level": "^2.1.0",
     "logplease": "^1.2.14",
     "multihashes": "^0.4.12",
-    "orbit-db-cache": "~0.0.7",
+    "orbit-db-cache": "orbitdb/orbit-db-cache#feat/leveldb",
     "orbit-db-counterstore": "~1.0.3",
     "orbit-db-docstore": "~1.0.1",
     "orbit-db-eventstore": "~1.0.0",
     "orbit-db-feedstore": "~1.0.0",
     "orbit-db-keystore": "~0.0.2",
     "orbit-db-kvstore": "~1.0.0",
-    "orbit-db-pubsub": "~0.3.6"
+    "orbit-db-pubsub": "~0.3.6",
+    "orbit-db-store": "orbitdb/orbit-db-store#feat/leveldb"
   },
   "devDependencies": {
     "babel-core": "^6.26.0",

src/OrbitDB.js

@@ -102,10 +102,12 @@ class OrbitDB {
       await accessController.load(options.accessControllerAddress)
     }
 
+    const cache = await this._loadCache(this.directory, address)
+
     const opts = Object.assign({ replicate: true }, options, { 
       accessController: accessController, 
       keystore: this.keystore,
-      cache: this._cache,
+      cache: cache,
     })
 
     const store = new Store(this._ipfs, this.id, address, opts)
@@ -177,6 +179,8 @@ class OrbitDB {
     }
   */
   async create (name, type, options = {}) {
+    logger.debug(`create()`)
+
     if (!OrbitDB.isValidType(type))
       throw new Error(`Invalid database type '${type}'`)
 
@@ -214,11 +218,11 @@ class OrbitDB {
     const dbAddress = OrbitDBAddress.parse(path.join('/orbitdb', manifestHash, name))
 
     // // Load local cache
-    const haveManifest = await this._loadCache(directory, dbAddress)
+    const haveDB = await this._loadCache(directory, dbAddress)
-      .then(cache => cache.get(dbAddress.toString()))
+      .then(cache => cache ? cache.get(path.join(dbAddress.toString(), '_manifest')) : null)
-      .then(localData => localData && localData.manifest)
+      .then(data => data !== undefined && data !== null)
 
-    if (haveManifest && !options.overwrite)
+    if (haveDB && !options.overwrite)
       throw new Error(`Database '${dbAddress}' already exists!`)
 
     // Save the database locally
@@ -240,6 +244,7 @@ class OrbitDB {
       }
    */
   async open (address, options = {}) {
+    logger.debug(`open()`)
     options = Object.assign({ localOnly: false, create: false }, options)
     logger.debug(`Open database '${address}'`)
 
@@ -265,8 +270,8 @@ class OrbitDB {
 
     // Check if we have the database
     const haveDB = await this._loadCache(directory, dbAddress)
-      .then(cache => cache.get(dbAddress.toString()))
+      .then(cache => cache ? cache.get(path.join(dbAddress.toString(), '_manifest')) : null)
-      .then(localData => localData && localData.manifest)
+      .then(data => data !== undefined && data !== null)
 
     logger.debug((haveDB ? 'Found' : 'Didn\'t find') + ` database '${dbAddress}'`)
 
@@ -299,19 +304,17 @@ class OrbitDB {
   // Save the database locally
   async _saveDBManifest (directory, dbAddress) {
     const cache = await this._loadCache(directory, dbAddress)
-    let localData = Object.assign({}, cache.get(dbAddress.toString()), {
+    // let localData = Object.assign({}, cache.get(dbAddress.toString()), {
-      manifest: dbAddress.root
+    //   manifest: dbAddress.root
-    })
+    // })
-    await cache.set(dbAddress.toString(), localData)
+    await cache.set(path.join(dbAddress.toString(), '_manifest'), dbAddress.root)
     logger.debug(`Saved manifest to IPFS as '${dbAddress.root}'`)
   }
 
   async _loadCache (directory, dbAddress) {
     let cache
     try {
-      const cacheFilePath = path.join(dbAddress.root, dbAddress.path)
+      cache = await Cache.load(directory, dbAddress)
-      cache = new Cache(path.join(directory), cacheFilePath)
-      await cache.load()
     } catch (e) {
       logger.warn("Couldn't load Cache:", e)
     }

test/counterdb.test.js

@@ -109,7 +109,7 @@ describe('CounterStore', function() {
           assert.equal(counter1.value, 30)
           assert.equal(counter2.value, 30)
           resolve()
-        }, 1000)
+        }, 5000)
       })
     })
   })

test/create-open.test.js

@@ -5,6 +5,8 @@ const fs = require('fs')
 const path = require('path')
 const rmrf = require('rimraf')
 const mapSeries = require('p-map-series')
+const levelup = require('levelup')
+const leveldown = require('leveldown')
 const OrbitDB = require('../src/OrbitDB')
 const OrbitDBAddress = require('../src/orbit-db-address')
 const config = require('./utils/config')
@@ -68,6 +70,7 @@ describe('orbit-db - Create & Open', function() {
         assert.equal(err, `Error: Database '${db.address}' already exists!`)
       })
 
+
       it('throws an error if database type doesn\'t match', async () => {
         let err, log, kv
         try {
@@ -83,7 +86,8 @@ describe('orbit-db - Create & Open', function() {
     describe('Success', function() {
       before(async () => {
         db = await orbitdb.create('second', 'feed')
-        localDataPath = path.join(dbPath, db.address.root, db.address.path + '.orbitdb')
+        localDataPath = path.join(dbPath, db.address.root, db.address.path)
+        await db.close()
       })
 
       it('creates a feed database', async () => {
@@ -97,14 +101,27 @@ describe('orbit-db - Create & Open', function() {
       })
 
       it('saves the database locally', async () => {
+        console.log(localDataPath)
         assert.equal(fs.existsSync(localDataPath), true)
       })
 
       it('saves database manifest reference locally', async () => {
-        const buffer = JSON.parse(fs.readFileSync(localDataPath))
+        const manifestHash = db.address.root
-        const data = buffer[db.address.toString()]
+        const address = db.address.toString()
-        assert.equal(data.manifest, db.address.root)
+        levelup(leveldown(localDataPath), (err, db) => {
-        assert.equal(db.address.path, 'second')
+          if (err) {
+            assert.equal(err, null)
+          }
+
+          db.get(address + '/_manifest', (err, value) => {
+            if (err) {
+              assert.equal(err, null)
+            }
+
+            const data = JSON.parse(value || '{}')
+            assert.equal(data, manifestHash)
+          })
+        })
       })
 
       it('saves database manifest file locally', async () => {
@@ -120,7 +137,7 @@ describe('orbit-db - Create & Open', function() {
       it('can pass local database directory as an option', async () => {
         const dir = './orbitdb/tests/another-feed'
         db = await orbitdb.create('third', 'feed', { directory: dir })
-        localDataPath = path.join(dir, db.address.root, db.address.path + '.orbitdb')
+        localDataPath = path.join(dir, db.address.root, db.address.path)
         assert.equal(fs.existsSync(localDataPath), true)
       })