use clusters for base sequences

ref #10

.gitignore

@@ -1,6 +1,8 @@
 .env
 .DS_Store
 db/*.sqlite3
+db/**/*.geojson
+testData/**/*.geojson
 db/migrations/.migrate
 testData/
 .python_version
@@ -9,3 +11,4 @@ node_modules/
 yarn-error.log
 coverage/
 .nyc_output/
+yarn.lock

baseSequence/index.js

@@ -0,0 +1,106 @@
+'use strict'
+
+const d3_dispatch = require('d3-dispatch').dispatch;
+// const geojsonvt = require('geojson-vt');
+const superCluster = require('supercluster');
+
+const singleton = Symbol();
+const singletonEnforcer = Symbol();
+
+const Database = require('../db')
+const databaseLocation = require('../config')[process.env.ENVIRONMENT || 'develop'].db;
+const baseLocation = require('../config')[process.env.ENVIRONMENT || 'develop'].base;
+const writeFile = require('fs-extra').writeFile;
+const readFile = require('fs-extra').readFile;
+const exists = require('fs-extra').exists;
+
+class BaseSequence {
+    constructor(enforcer) {
+        if (enforcer !== singletonEnforcer) {
+            throw new Error('Cannot construct singleton');
+        };
+    }
+    static getInstance() {
+        if (!this[singleton]) {
+            this[singleton] = new BaseSequence(singletonEnforcer);
+        }
+        return this[singleton];
+    }
+
+    get tileLoc() {
+        return this._tileLoc;
+    }
+
+    get tileIndex() { 
+        return this._tileIndex;
+    }
+
+    async rebuild() {
+        try {
+            Database.connect(databaseLocation);
+            const sequences = await Database.getAllImages();
+            await writeFile(baseLocation, JSON.stringify(sequences));
+            Database.close(); 
+            return;
+        } catch (err) {
+            throw err;
+        }
+    }
+
+    async init(tileLoc) {
+        this._tileLoc = tileLoc || baseLocation;
+        this._tileIndex = superCluster({ radius: 40, maxZoom: 16 });
+        if (await exists(this._tileLoc)) {
+            await this.retile()
+        }
+        const that = this;
+
+        this._dispatcher = d3_dispatch('rebuild', 'tile');
+        this._dispatcher.on('rebuild', async () => {
+            try {
+                await that.rebuild();
+                await that.retile(); 
+                return;
+            } catch (err) {
+                throw err;
+            }
+        })
+
+        this._dispatcher.on('tile', (_) => {
+            that.tile.apply(that, arguments);
+        })
+    }
+
+    async retile() {
+        try {
+            const geojson = await readFile(this._tileLoc).then((res) => JSON.parse(res.toString('utf-8')));
+            this._tileIndex.load(geojson.features);
+            return;
+        } catch (err) {
+            throw err;
+        }
+    }
+
+    // https://wiki.openstreetmap.org/wiki/Slippy_map_tilenames#Lon..2Flat._to_tile_numbers_2
+    fromPoint(zoom,lon,lat) {
+        const tileX = (Math.floor((lon+180)/360*Math.pow(2,zoom)))
+        const tileY = (Math.floor((1-Math.log(Math.tan(lat*Math.PI/180) + 1/Math.cos(lat*Math.PI/180))/Math.PI)/2 *Math.pow(2,zoom)))
+        return [zoom,tileX,tileY]
+    }
+
+    clusters(bbox, zoom) {
+        return this._tileIndex.getClusters(bbox, zoom);
+    }
+
+    getTiles(z,x,y) {
+        return this._tileIndex.getTile.apply(this._tileIndex, this.fromPoint(z,x,y));
+    }
+
+    call(_) {
+        this._dispatcher.call.apply(this._dispatcher, arguments);
+    }
+
+}
+
+module.exports = BaseSequence;
+

config.js

@@ -3,11 +3,13 @@
 module.exports = {
     test: {
         db: './db/test-posm-paths.sqlite3',
+        base: './db/test-base-layer.geojson',
         host: 'localhost',
         injectDefaults: { simulate: { error: false }}
     },
     develop: {
         db: './db/posm-paths.sqlite3',
+        base: './db/test-base-layer.geojson',
         host: 'localhost',
         injectDefaults: { simulate: { error: false }}
     }

db/index.js

@@ -92,9 +92,19 @@ class Database {
             .then((res) => res)
             .catch((err) => { throw err; });
     }
+    getAllImages() {
+        const sql = `
+            SELECT AsGeoJSON(loc) as geometry, id, seqId
+            FROM Images;
+        `
+        return this
+            .selectSpatial(sql)
+            .then(res => Formatter.toFeatureCollection(res))
+            .catch(err => { throw err; });
+    }
     getSequence(id) {
         const sql = `
-            SELECT AsGeoJSON(loc), id, seqId
+            SELECT AsGeoJSON(loc) as geometry, id, seqId
             FROM Images
             WHERE seqId='${id}';
         `;

formatter/index.js

@@ -17,8 +17,8 @@ class Formatter {
             const features = rows.map(row => {
                 const base = { type: 'Feature' }
                 let properties = {};
-                const geometry = JSON.parse(row['AsGeoJSON(loc)']);
-                delete row['AsGeoJSON(loc)'];
+                const geometry = JSON.parse(row['geometry']);
+                delete row['geometry'];
                 Object.entries(row).forEach(entry => properties = Object.assign(properties, { [entry[0]] : entry[1] }));
                 return Object.assign(base, { geometry: geometry, properties: properties });
             })
@@ -31,7 +31,8 @@ class Formatter {
     }
     whereAnd(config) {
         const conditionals = Object.entries(config).map(entry => {
-            const colKey = entry[0], values = flatten([`'${entry[1]}'`]).join(',');
+            const colKey = entry[0];
+            const values = flatten([`'${entry[1]}'`]).join(',');
             return `${colKey} IN (${values})`; 
         }).join(' AND ');
         return `WHERE ${conditionals}`;

handlers/base/get.js

@@ -0,0 +1,24 @@
+'use strict';
+
+const Boom = require('boom');
+const BaseSequence = require('../../baseSequence');
+const baseSequence = BaseSequence.getInstance();
+
+module.exports = async (r, h) => {
+	try {
+
+        const bbox = r.query.bbox.split(',').map(c => Number(c));
+        const zoom = Number(r.query.zoom);
+
+        const clusters = baseSequence.clusters(bbox, zoom);
+
+        if (!clusters.length) {
+            return Boom.badRequest('No images in the database!')
+        }
+
+        return h.response(clusters).code(200)
+    } catch (error) {
+        console.error(error);
+        throw error;
+    }
+}

handlers/base/index.js

@@ -0,0 +1,3 @@
+module.exports = {
+    get: require('./get')
+}

handlers/sequence/post.js

@@ -4,6 +4,9 @@ const Boom = require('boom');
 const Sequence = require('../../adapters/sequence');
 const Database = require('../../db');
 const databaseLocation = require('../../config')[process.env.ENVIRONMENT || 'develop'].db;
+const BaseSequence = require('../../baseSequence');
+const baseSequence = BaseSequence.getInstance();
+
 
 Promise = require('bluebird');
 
@@ -22,6 +25,9 @@ module.exports = async (r, h) => {
 		Database.connect(databaseLocation);
 		await Promise.each(sequences, (sequence) => Database.addSequence(sequence).catch(err => { throw err; }))
 		Database.close();
+
+		baseSequence.call('rebuild');
+
 		return h.response({ upload: 'successful' }).code(200);
 
 	} catch (error) {

package.json

@@ -6,7 +6,6 @@
   "author": "Max Grossman <[email protected]>",
   "license": "MIT",
   "scripts": {
-    "test": "npm run migrate:clean && npm run migrate:up:seed && npm run test:unit",
     "test:unit": "mocha test/**/*.js --exit",
     "test:db": "mocha test/db.js",
     "test:migrate": "npm run migrate:up:seed && npm run test:db",
@@ -20,10 +19,12 @@
   "dependencies": {
     "bluebird": "^3.5.1",
     "chai": "^4.1.2",
+    "d3-dispatch": "^1.0.5",
     "dayjs": "^1.5.16",
     "dotenv": "^5.0.1",
     "exiftool-vendored": "^4.22.1",
     "fs-extra": "^5.0.0",
+    "geojson-vt": "^3.2.0",
     "hapi": "^17.4.0",
     "inert": "^5.1.0",
     "joi": "^13.2.0",
@@ -32,6 +33,7 @@
     "mocha": "^5.1.1",
     "spatialite": "^0.1.0",
     "sqlite3": "https://github.com/mapbox/node-sqlite3/tarball/master",
+    "supercluster": "^4.1.1",
     "taskboard": "^2.0.6"
   },
   "devDependencies": {

routes/base/get.js

@@ -0,0 +1,9 @@
+'use strict';
+
+module.exports = {
+    method: 'GET',
+    path: '/base',
+    config: {
+        handler: require('../../handlers/base').get
+    }
+}

routes/base/index.js

@@ -0,0 +1,3 @@
+module.exports = {
+    get: require('./get')
+}

routes/index.js

@@ -5,5 +5,6 @@ const flatten = require('../helpers').flatten;
 module.exports = flatten([
 	require('./user').post,
 	require('./sequence').get,
-	require('./sequence').post
-]);
+	require('./sequence').post,
+	require('./base').get
+])

test/baseSequence/adapter.js

@@ -0,0 +1,42 @@
+const expect = require('chai').expect;  
+
+const baseSequence = require('../../baseSequence').getInstance();
+const nyc = process.cwd() + '/testData/nyc.geojson';
+
+describe('baseSequence', () => {
+    describe('init', () => {
+        it('#generates sets up event emitter and tile source for provided or default tiles', async () => {
+            await baseSequence.init(nyc);
+            expect(baseSequence.tileLoc).to.eql(nyc);
+            return;
+        })
+    })
+    describe('#fromPoint', () => {
+        it('returns tile given xyz', async () => {
+            await baseSequence.init(nyc)
+            const tiles = baseSequence.fromPoint(13,-73.9751,40.6268)
+            expect(tiles).to.deep.equal([13, 2412, 3082]); 
+            return;
+        });
+    })
+
+    describe('#clusters', () => {
+        it('returns clusters for given bbox and zoom', async () => {
+            await baseSequence.init(nyc);
+            const bbox = [-74.007491,40.685416,-73.917197,40.733562]
+            const clusters = baseSequence.clusters(bbox, 15);
+            expect(clusters.length).to.eql(1217);
+            // expect(clusters[0].point_count).to.eql(4);
+        })
+    })
+    describe('#getTiles', () => {
+        it('returns tiles found in tile matching given point & zoom', () => {
+            const tiles = baseSequence.getTiles(13,-73.9751,40.6268)
+            expect(tiles.features.length).not.to.be.null;
+        });
+        it('returns null if no tiles found', () => {
+            const tiles = baseSequence.getTiles(13, 0, 0);
+            expect(tiles).to.be.null;
+        });
+    });
+});

test/baseSequence/handler.js

@@ -0,0 +1,38 @@
+'use strict';
+
+const chai = require('chai');
+const expect = chai.expect;
+
+const flatten = require('../../helpers').flatten;
+
+const server = require('../server');
+const mergeDefaults = require('../helpers').mergeDefaults;
+const routes =  flatten([ 
+    require('../../routes/base').get
+]);
+
+const nyc = process.cwd() + '/testData/nyc.geojson';
+const baseSequence = require('../../baseSequence').getInstance();
+
+before(async () => {
+    await server.liftOff(routes)
+    await baseSequence.init(nyc);
+})
+describe('get', () => {
+    it ('replies 200 and sequence images when given valid uuid', async () => {
+
+
+        const baseSequenceRequest = mergeDefaults({
+            method: 'GET',
+            url: '/base?zoom=16&bbox=-74.007491,40.685416,-73.917197,40.733562'
+        })
+
+        const r = await server.inject(baseSequenceRequest);
+        const statusCode = r.statusCode;
+        const clusters = JSON.parse(r.payload);
+
+        expect(statusCode).to.eql(200);
+        expect(clusters.length).to.not.be.null;
+
+    });
+})

test/baseSequence/index.js

@@ -0,0 +1,6 @@
+'use strict';
+
+module.exports = {
+    handlerSpec: require('./handler'),
+    adapterSpec: require('./adapter')
+}