Merge pull request #257 from SEED-platform/develop

Release 1.1.5

.gitignore

@@ -45,4 +45,5 @@ src/
 /seed/static/vendors/bower_components/fine-uploader/
 docs/build/
 .project
-.pydevproject
+.pydevproject
+dump.rdb

BE/settings/dev.py

@@ -49,7 +49,7 @@
     LOGGING = {
         'version': 1,
         'disable_existing_loggers': True,
-        # set up some log message handers to chose from
+        # set up some log message handlers to choose from
         'handlers': {
             'sentry': {
                 'level': 'ERROR',

README-osx.md

@@ -4,18 +4,27 @@ These instructions are for installing and running SEED on Mac OSX in development
 
 ## Prerequisites
 
-These instructions assume you have/use [Macports](https://www.macports.org/).
+These instructions assume you have/use [Macports](https://www.macports.org/). The workflow has been testing with homebrew as well, but is not directly supported. You system should have the following dependencies already installed:
 
-Although you _could_ install Python packages globally, the easiest way to install Python packages is with [virtualenv](https://virtualenv.pypa.io/en/latest/) and [virtualenvwrapper](https://virtualenvwrapper.readthedocs.org/en/latest/). Setting these up first will help avoid polluting your base Python installation and make it much easier to switch between different versions of the code.
+* git (`port install git` or `brew install git`)
+* Mercurial (`port install hg` or `brew install mercurial`)
 
-Once you have these installed, creating and entering a new virtualenv called "``seed``" for SEED development is as easy as:
+(Recommended)
 
-    mkvirtualenv --python=python2.7 seed
-
+* [virtualenv](https://virtualenv.pypa.io/en/latest/) and [virtualenvwrapper](https://virtualenvwrapper.readthedocs.org/en/latest/).
+    * Although you _could_ install Python packages globally, this is the easiest way to install Python packages. Setting these up first will help avoid polluting your base Python installation and make it much easier to switch between different versions of the code.
+
+        pip install virtualenv
+        pip install virtualenvwrapper
+
+    * Follow instructions on [virtualenvwrapper](https://virtualenvwrapper.readthedocs.org/en/latest/) to setup your environment.
+    * Once you have these installed, creating and entering a new virtualenv called "``seed``" for SEED development is as easy as:
+
+        mkvirtualenv --python=python2.7 seed
 
 ## Install PostgreSQL 9.4
 
-Perform the following commands as 'root'
+Perform the following commands as 'root' if using port
 
     sudo su - root
 
@@ -25,7 +34,12 @@ Install Postgres 9.4
     # init db
     mkdir -p /opt/local/var/db/postgresql94/defaultdb
     chown postgres:postgres /opt/local/var/db/postgresql94/defaultdb
-
+
+    # homebrew
+    brew install postgres
+    # follow the post install instructions to add to launchagents or call manually with `postgres -D /usr/local/var/postgres`
+    # Skip the remaining Postgres instructions
+
 Finish initializing the DB
 
     sudo su postgres -c '/opt/local/lib/postgresql94/bin/initdb -D /opt/local/var/db/postgresql94/defaultdb'
@@ -44,11 +58,10 @@ Start Postgres
 
 Switch to postgres user
 
-      sudo su - postgres
-      PATH=$PATH:/opt/local/lib/postgresql94/bin/
+    sudo su - postgres
+    PATH=$PATH:/opt/local/lib/postgresql94/bin/
       
-Configure Postgresql. Replace 'seeddb', 'seeduser' with desired db/user.
-seedpass
+Configure PostgreSQL. Replace 'seeddb', 'seeduser' with desired db/user. By default use password `seedpass` when prompted
 
     createdb seeddb
     createuser -P seeduser
@@ -60,9 +73,11 @@ Now exit any root environments, becoming just yourself (even though it's not tha
 
 Run these commands as your normal user id.
 
-Change to a virtualenv (using virtualenvwrapper) or do the following as a superuser. A virtualenv is usually better for development.
+Change to a virtualenv (using virtualenvwrapper) or do the following as a superuser. A virtualenv is usually better for development. Set the virtualenv to seed.
+
+    workon seed
 
-Make sure PostgreSQL command line scripts are in your PATH
+Make sure PostgreSQL command line scripts are in your PATH (if using port)
 
     export PATH=$PATH:/opt/local/lib/postgresql94/bin
 
@@ -85,13 +100,11 @@ Install library with `setup.py`
 
 First, install [npm](https://www.npmjs.com/) if you haven't already. You can do this by installing from [nodejs.org](http://nodejs.org/), or use Macports:
 
+    # port
     sudo port install npm
 
-### Install libraries
-
-Then run, from the top-level, a script to install the JS libraries:
-
-    ./bin/install_javascript_dependencies.sh
+    # homebrew
+    brew install npm
 
 ## Configure Django and its back-end DBs
 
@@ -150,7 +163,7 @@ If you want to do any API testing (and of course you do!), you will
 need to add an API KEY for this user.
 You can do this in postgresql directly:
 
-    psql94 seeddb seeduser
+    psql seeddb seeduser
     seeddb=> update landing_seeduser set api_key='DEADBEEF' where id=1;
 
 The 'secret' key DEADBEEF is hard-coded into the test scripts.
@@ -159,8 +172,13 @@ The 'secret' key DEADBEEF is hard-coded into the test scripts.
 
 You need to manually install Redis for Celery to work.
 
+    # port
     sudo port install redis
 
+    # homebrew
+    brew install redis
+    # follow the post install instructions to add to launchagents or call manually with `redis-server`
+
 ### Install Javascript dependencies
 
 The JS dependencies are installed using node.js package management (npm), with

green_button/xml_importer.py

@@ -119,7 +119,7 @@ def interval_data(reading_xml_data):
     :returns: dictionary representing a time series reading with keys
     'cost', 'value', 'start_time', and 'duration'.
     """
-    cost = reading_xml_data['cost']
+    cost = reading_xml_data.get('cost')
     value = reading_xml_data['value']
 
     time_period = reading_xml_data['timePeriod']
@@ -155,8 +155,8 @@ def meter_data(raw_meter_meta):
     # this function currently assumes those types are present and does
     # not check for any other types
 
-    currency = params_data['currency']
-    power_of_ten_multiplier = params_data['powerOfTenMultiplier']
+    currency = params_data.get('currency')
+    power_of_ten_multiplier = params_data.get('powerOfTenMultiplier')
     uom = params_data['uom']
 
     result = {

seed/decorators.py

@@ -22,21 +22,22 @@ def _get_lock_key(func_name, import_file_pk):
 
 
 def get_prog_key(func_name, import_file_pk):
+    """Return the progress key for the cache"""
     return _get_cache_key(
         PROGRESS_CACHE_PREFIX.format(func_name), import_file_pk
     )
 
 
 def increment_cache(key, increment):
     """Increment cache by value increment, never exceed 100."""
-    value = cache.get(key) or 0.0
-    value = float(value)
+    value = cache.get(key) or {'status': 'parsing', 'progress': 0.0}
+    value = float(value['progress'])
     if value + increment >= 100.0:
         value = 100.0
     else:
         value += increment
 
-    cache.set(key, value)
+    cache.set(key, {'status': 'parsing', 'progress': value})
 
 
 def lock_and_track(fn, *args, **kwargs):

seed/models.py

@@ -152,15 +152,16 @@ def get_ancestors(building):
        source_type {
            2: ASSESSED_BS,
            3: PORTFOLIO_BS,
-           4: COMPOSITE_BS
+           4: COMPOSITE_BS,
+           6: GREEN_BUTTON_BS
        }
 
        :param building: BuildingSnapshot inst.
        :returns: list of BuildingSnapshot inst., ancestors of building
     """
     ancestors = []
-    parents = building.parents.filter(source_type__in=[2, 3, 4])
-    ancestors.extend(parents.filter(source_type__in=[2, 3]))
+    parents = building.parents.filter(source_type__in=[2, 3, 4, 6])
+    ancestors.extend(parents.filter(source_type__in=[2, 3, 6]))
     for p in parents:
         ancestors.extend(get_ancestors(p))
     return ancestors

seed/search.py

@@ -8,6 +8,7 @@
 # python
 import operator
 import json
+import re
 
 # django
 from django.db.models import Q
@@ -184,49 +185,90 @@ def is_column(k, columns):
         sanitized = strip_suffixes(k, ['__lt', '__gt', '__lte', '__gte'])
         if sanitized in columns:
             return True
-        else:
-            return False
+        return False
+
+    def is_string_query(q):
+        return isinstance(q, basestring)
+
+    def is_exact_match(q):
+        if is_string_query(q):
+            return re.match(r"""^(["'])(.+)\1$""", q)
+        return False
+
+    def is_empty_match(q):
+        if is_string_query(q):
+            return re.match(r"""^(["'])\1$""", q)
+        return False
 
-    query_dict = {}
+    # Build query as Q objects so we can AND and OR.
+    query_filters = Q()
     for k, v in other_params.iteritems():
         in_columns = is_column(k, db_columns)
-        if in_columns and k != 'q' and v is not None and v != '':
-            if ('__lt' in k or
+        if in_columns and k != 'q' and v:
+
+            # Is this query surrounded by matching quotes?
+            exact_match = is_exact_match(v)
+            empty_match = is_empty_match(v)
+
+            if exact_match:
+                query_filters &= Q(**{"%s__exact" % k: exact_match.group(2)})
+            elif empty_match:
+                query_filters &= Q(**{"%s__exact" % k: ''}) | Q(**{"%s__isnull" % k: True})
+            elif ('__lt' in k or
                 '__lte' in k or
                 '__gt' in k or
                 '__gte' in k or
                 '__isnull' in k or
                k == 'import_file_id' or k == 'source_type'):
-                query_dict["%s" % k] = v
+                query_filters &= Q(**{"%s" % k: v})
             else:
-                query_dict["%s__icontains" % k] = v
+                query_filters &= Q(**{"%s__icontains" % k: v})
 
-    queryset = queryset.filter(**query_dict)
+    queryset = queryset.filter(query_filters)
 
     # handle extra_data with json_query
     for k, v in other_params.iteritems():
-        if (not is_column(k, db_columns)) and k != 'q' and v != '':
-            if k.endswith(('__gt', '__gte')):
+        if (not is_column(k, db_columns)) and k != 'q' and v:
+
+            # Is this query surrounded by matching quotes?
+            exact_match = is_exact_match(v)
+            empty_match = is_empty_match(v)
+
+
+            # If querying for empty matches, do a hack-y 'contains' query on
+            # the json field to check if the field exists. We're checking 
+            # existence because empty mapped fields are not saved in the
+            # extra_data json field if they contain no data.
+            #
+            # When we bump to Django 1.7, we can switch to the newer
+            # django-pgjson package, and use the new "HAS" operator syntax.
+            # - nicholasserra
+            if empty_match:
+                queryset = queryset.exclude(extra_data__contains='"%s":' % k)
+                continue
+
+            conditions = {
+                'value': v
+            }
+
+            if exact_match:
+                conditions['value'] = exact_match.group(2)
+                conditions['key_cast'] = 'text'
+            elif k.endswith(('__gt', '__gte')):
                 k = strip_suffixes(k, ['__gt', '__gte'])
-                cond = '>'
-                key_cast = 'float'
+                conditions['cond'] = '>'
+                conditions['key_cast'] = 'float'
             elif k.endswith(('__lt', '__lte')):
                 k = strip_suffixes(k, ['__lt', '__lte'])
-                cond = '<'
-                key_cast = 'float'
+                conditions['cond'] = '<'
+                conditions['key_cast'] = 'float'
             else:
-                cond = 'LIKE'
-                key_cast = 'text'
-                v = "%{0}%".format(v)
-            case_insensitive = key_cast == 'text'
-
-            queryset = queryset.json_query(
-                k,
-                cond=cond,
-                key_cast=key_cast,
-                value=v,
-                case_insensitive=case_insensitive,
-            )
+                conditions['cond'] = 'LIKE'
+                conditions['key_cast'] = 'text'
+                conditions['value'] = "%{0}%".format(v)
+                conditions['case_insensitive'] = True
+
+            queryset = queryset.json_query(k, **conditions)
 
     return queryset
 

seed/static/seed/css/mapping.css

@@ -7,9 +7,9 @@
     margin: 10px;
 }
 .fa-arrows {
-	   padding-right: 7px;
-	   color: #444;
-   }
+    padding-right: 7px;
+    color: #444;
+}
 .tree-handle {
     padding: 10px;
     background: #428bca;

seed/static/seed/js/controllers/admin_controller.js

@@ -162,6 +162,8 @@ angular.module('BE.seed.controller.admin', [])
                 function(data){  //success fn
                   org.remove_message = "success";
                   get_organizations();
+                }, function(data){  //failure fn
+                  // Do nothing
                 },
                 org  // progress bar obj
             );

seed/static/seed/js/controllers/building_detail_controller.js

@@ -32,6 +32,15 @@ angular.module('BE.seed.controller.building_detail', [])
     $scope.building_copy = {};
     $scope.data_columns = [];
     $scope.audit_logs = audit_payload.audit_logs;
+    $scope.green_button_filenames = [];
+
+    // gather green button filenames
+    building_payload.imported_buildings.forEach(function(e) {
+        if (e.source_type == 6) { // GREEN_BUTTON_BS
+            $scope.green_button_filenames.push(e.import_file_name);
+        }
+    });
+
     // set the tab
     $scope.section = $location.hash();
     /**

seed/static/seed/js/controllers/building_list_controller.js

@@ -66,6 +66,7 @@ angular.module('BE.seed.controller.building_list', [])
      */
     var get_columns = function() {
         $scope.assessor_fields = all_columns.fields;
+        $scope.search.init_storage();
         $scope.columns = $scope.search.generate_columns(
             all_columns.fields,
             default_columns.columns,