From 28189bd9d45b44e670b16f878d736091b4e0eb91 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alice=20Zo=C3=AB=20Bevan=E2=80=93McGregor?= Date: Wed, 10 May 2017 04:14:37 -0400 Subject: [PATCH] Traits, basic documents, utility fields. (#26) Traits added: * Collection * Derived * Expires * Identified * Localized * Published * Queryable Major unrelated additions: * String stripping and case conversion. Fixes #33. * Markdown field. Fixes #34. * Path field. Fixes #35. Minor additions and changes: * Utility function: `utcnow` * Improved docstring coverage and referential integrity. Work on #16. * Generalized programmers' representations. * Improved minifying plugin name lookup for `PluginReference` fields. * Correct regression in the ability to manipulate field attributes via class attribute access. * Collation support. * Various project metadata updates including pre-commit and testing: * Updated CPython, Pypy, and MongoDB versions on Travis. * Skip slow (and unreliable) capped collection tests on Pypy. * Step debugger integration. * Adjustments to query fragment merging. * Switch test logging capture plugin to eliminate warnings. * Simplification for HasKinds field types to only support a single referenced kind. * Corrected Reference field behaviours. * Parametric adjustments for extended `push` options. * Dead code removal. * Updated example for latest usage patterns. * Array and Embed default value handling to reduce boilerplate lambdas, tests, trait updates. * Utilize MutableMapping subclasses to work around an issue on Pypy. Fixes #32, reference #31. --- .pre-commit-config.yaml | 60 ++-- .travis.yml | 20 +- Makefile | 3 +- example/readme.py | 49 ++- marrow/mongo/__init__.py | 4 +- marrow/mongo/core/document.py | 256 ++++---------- marrow/mongo/core/field/__init__.py | 9 +- marrow/mongo/core/field/base.py | 26 +- marrow/mongo/core/field/complex.py | 189 +++++----- marrow/mongo/core/field/md.py | 40 +++ marrow/mongo/core/field/path.py | 16 + marrow/mongo/core/index.py | 18 +- marrow/mongo/core/trait/__init__.py | 0 marrow/mongo/core/trait/collection.py | 221 ++++++++++++ marrow/mongo/core/trait/derived.py | 25 ++ marrow/mongo/core/trait/expires.py | 44 +++ marrow/mongo/core/trait/identified.py | 38 ++ marrow/mongo/core/trait/localized.py | 64 ++++ marrow/mongo/core/trait/published.py | 49 +++ marrow/mongo/core/trait/queryable.py | 310 ++++++++++++++++ marrow/mongo/param/project.py | 2 +- marrow/mongo/param/sort.py | 8 +- marrow/mongo/param/update.py | 75 +++- marrow/mongo/query/ops.py | 40 ++- marrow/mongo/query/query.py | 45 ++- marrow/mongo/util/__init__.py | 34 +- marrow/mongo/util/capped.py | 13 +- setup.py | 38 +- test/core/document/test_projection.py | 10 +- test/core/document/test_serialization.py | 57 ++- test/core/document/test_validation.py | 9 +- test/core/test_index.py | 19 +- test/core/test_util.py | 37 +- test/field/common.py | 19 + test/field/test_array.py | 25 ++ test/field/test_base.py | 150 -------- test/field/test_binary.py | 15 + test/field/test_boolean.py | 38 ++ test/field/test_complex.py | 331 ------------------ test/field/test_date.py | 10 + test/field/test_decimal.py | 32 ++ test/field/test_double.py | 21 ++ test/field/test_embed.py | 64 ++++ test/field/{test_core.py => test_field.py} | 2 +- test/field/test_integer.py | 21 ++ test/field/test_long.py | 14 + test/field/test_md.py | 36 ++ test/field/test_number.py | 77 +--- test/field/test_objectid.py | 55 +++ test/field/test_path.py | 23 ++ test/field/test_plugin_explicit.py | 19 + test/field/test_plugin_namespace.py | 40 +++ test/field/test_reference.py | 43 +++ test/field/test_reference_cached.py | 103 ++++++ test/field/test_reference_concrete.py | 49 +++ test/field/test_regex.py | 10 + test/field/test_string.py | 40 +++ test/field/test_timestamp.py | 10 + test/field/test_ttl.py | 42 +++ test/param/test_project.py | 5 +- test/param/test_update.py | 20 ++ test/query/test_ops.py | 8 + test/query/test_q.py | 4 +- .../test_collection.py} | 47 ++- test/trait/test_derived.py | 27 ++ test/trait/test_expires.py | 49 +++ test/trait/test_identified.py | 37 ++ test/trait/test_localized.py | 57 +++ test/trait/test_published.py | 72 ++++ test/trait/test_queryable.py | 94 +++++ test/util/test_capped.py | 13 +- 71 files changed, 2582 insertions(+), 968 deletions(-) create mode 100644 marrow/mongo/core/field/md.py create mode 100644 marrow/mongo/core/field/path.py create mode 100644 marrow/mongo/core/trait/__init__.py create mode 100644 marrow/mongo/core/trait/collection.py create mode 100644 marrow/mongo/core/trait/derived.py create mode 100644 marrow/mongo/core/trait/expires.py create mode 100644 marrow/mongo/core/trait/identified.py create mode 100644 marrow/mongo/core/trait/localized.py create mode 100644 marrow/mongo/core/trait/published.py create mode 100644 marrow/mongo/core/trait/queryable.py create mode 100644 test/field/common.py create mode 100644 test/field/test_array.py delete mode 100644 test/field/test_base.py create mode 100644 test/field/test_binary.py create mode 100644 test/field/test_boolean.py delete mode 100644 test/field/test_complex.py create mode 100644 test/field/test_date.py create mode 100644 test/field/test_decimal.py create mode 100644 test/field/test_double.py create mode 100644 test/field/test_embed.py rename test/field/{test_core.py => test_field.py} (99%) create mode 100644 test/field/test_integer.py create mode 100644 test/field/test_long.py create mode 100644 test/field/test_md.py create mode 100644 test/field/test_objectid.py create mode 100644 test/field/test_path.py create mode 100644 test/field/test_plugin_explicit.py create mode 100644 test/field/test_plugin_namespace.py create mode 100644 test/field/test_reference.py create mode 100644 test/field/test_reference_cached.py create mode 100644 test/field/test_reference_concrete.py create mode 100644 test/field/test_regex.py create mode 100644 test/field/test_string.py create mode 100644 test/field/test_timestamp.py create mode 100644 test/field/test_ttl.py rename test/{core/document/test_binding.py => trait/test_collection.py} (61%) create mode 100644 test/trait/test_derived.py create mode 100644 test/trait/test_expires.py create mode 100644 test/trait/test_identified.py create mode 100644 test/trait/test_localized.py create mode 100644 test/trait/test_published.py create mode 100644 test/trait/test_queryable.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5b75bf33..48143ba9 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,30 +1,30 @@ -- repo: https://github.com/pre-commit/pre-commit-hooks.git - sha: c8a1c91c762b8e24fdc5a33455ec10662f523328 - hooks: - - id: check-added-large-files - - id: check-ast - - id: check-byte-order-marker - - id: check-docstring-first - - id: check-merge-conflict - - id: check-symlinks - - id: debug-statements - - id: detect-private-key - - id: end-of-file-fixer - - id: check-json - - id: check-xml - - id: check-yaml -- repo: https://github.com/Lucas-C/pre-commit-hooks-safety - sha: v1.0.9 - hooks: - - id: python-safety-dependencies-check -- repo: https://github.com/Lucas-C/pre-commit-hooks-bandit - sha: v1.0.1 - hooks: - - id: python-bandit-vulnerability-check -- repo: local - hooks: - - id: py.test - name: py.test - language: system - entry: sh -c py.test - files: '' +- repo: https://github.com/pre-commit/pre-commit-hooks.git + sha: 46251c9523506b68419aefdf5ff6ff2fbc4506a4 + hooks: + - id: check-added-large-files + - id: check-ast + - id: check-byte-order-marker + - id: check-docstring-first + - id: check-merge-conflict + - id: check-symlinks + - id: debug-statements + - id: detect-private-key + - id: end-of-file-fixer + - id: check-json + - id: check-xml + - id: check-yaml +- repo: https://github.com/Lucas-C/pre-commit-hooks-safety + sha: v1.1.0 + hooks: + - id: python-safety-dependencies-check +- repo: https://github.com/Lucas-C/pre-commit-hooks-bandit + sha: v1.0.3 + hooks: + - id: python-bandit-vulnerability-check +- repo: local + hooks: + - id: py.test + name: py.test + language: system + entry: sh -c py.test + files: '' diff --git a/.travis.yml b/.travis.yml index 4388e8d2..2847ab76 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,15 +2,6 @@ language: python sudo: false cache: pip -addons: - apt: - sources: - - mongodb-upstart - - mongodb-3.2-precise - packages: - - mongodb-org-server - - mongodb-org-shell - branches: except: - /^[^/]+/.+$/ @@ -19,13 +10,20 @@ branches: python: - "2.7" - - "pypy" + - "pypy-5.4.1" - "3.3" - "3.4" - "3.5" + - "3.6" + +before_install: + - wget http://fastdl.mongodb.org/linux/mongodb-linux-x86_64-3.4.1.tgz -O /tmp/mongodb.tgz + - tar -xvf /tmp/mongodb.tgz + - mkdir /tmp/data + - ${PWD}/mongodb-linux-x86_64-3.4.1/bin/mongod --dbpath /tmp/data --bind_ip 127.0.0.1 --noauth &> /dev/null & install: - - travis_retry pip install --upgrade setuptools pip pytest pytest-cov codecov 'setuptools_scm>=1.9' + - travis_retry pip install --upgrade setuptools pip pytest pytest-cov codecov 'setuptools_scm>=1.9' cffi - pip install -e '.[development,logger]' script: diff --git a/Makefile b/Makefile index 3878ebf6..5a85c7de 100644 --- a/Makefile +++ b/Makefile @@ -17,7 +17,8 @@ veryclean: clean rm -rvf *.egg-info .packaging test: develop - ./setup.py test + @clear + @pytest release: ./setup.py register sdist bdist_wheel upload ${RELEASE_OPTIONS} diff --git a/example/readme.py b/example/readme.py index 708ed9b4..ce3cabd3 100644 --- a/example/readme.py +++ b/example/readme.py @@ -1,35 +1,56 @@ # encoding: utf-8 +# Imports. + from pymongo import MongoClient -from marrow.mongo import Document, Index +from marrow.mongo import Index from marrow.mongo.field import Array, Number, ObjectId, String +from marrow.mongo.trait import Queryable + + +# Connect to the test database on the local machine. + +db = MongoClient().test -collection = MongoClient().test.collection -collection.drop() +# Define an Account object model and associated metadata. -class Account(Document): +class Account(Queryable): + __collection__ = 'collection' + username = String(required=True) name = String() locale = String(default='en-CA-u-tz-cator-cu-CAD', assign=True) age = Number() - id = ObjectId('_id', assign=True) - tag = Array(String(), default=lambda: [], assign=True) + tag = Array(String(), assign=True) _username = Index('username', unique=True) +# Bind it to a database, then (re-)create the collection including indexes. + +collection = Account.bind(db).create_collection(drop=True) + + +# Create a new record; this record is unsaved... + alice = Account('amcgregor', "Alice Bevan-McGregor", age=27) -print(alice.id) # Already has an ID. -print(alice.id.generation_time) # This even includes the creation time. -Account._username.create_index(collection) +print(alice.id) # It already has an ID, however! +print(alice.id.generation_time) # This includes the creation time. + + +# Inserting it will add it to the storage back-end; the MongoDB database. + +result = alice.insert_one() + +assert result.acknowledged and result.inserted_id == alice + + +# Now you can query Account objects for one with an appropriate username. -result = collection.insert_one(alice) -assert result.acknowledged and result.inserted_id == alice.id +record = Account.find_one(username='amcgregor') -record = collection.find_one(Account.username == 'amcgregor') -record = Account.from_mongo(record) -print(record.name) # Alice Bevan-McGregor +print(record.name) # Alice Bevan-McGregor; it's already an Account instance. diff --git a/marrow/mongo/__init__.py b/marrow/mongo/__init__.py index 7110462b..b3fa74d7 100644 --- a/marrow/mongo/__init__.py +++ b/marrow/mongo/__init__.py @@ -7,10 +7,12 @@ from .core import Document, Field, Index, __version__ # noqa from .query import Q, Ops, Filter, Update from .param import F, P, S, U -from .util import Registry +from .util import Registry, utcnow document = sys.modules['marrow.mongo.document'] = Registry('marrow.mongo.document') field = sys.modules['marrow.mongo.field'] = Registry('marrow.mongo.field') +trait = sys.modules['marrow.mongo.trait'] = Registry('marrow.mongo.trait') + __all__ = [ 'Document', diff --git a/marrow/mongo/core/document.py b/marrow/mongo/core/document.py index 09bef3f4..e694e6ea 100644 --- a/marrow/mongo/core/document.py +++ b/marrow/mongo/core/document.py @@ -4,19 +4,13 @@ from collections import MutableMapping -from bson.binary import STANDARD -from bson.codec_options import CodecOptions +from bson import ObjectId from bson.json_util import dumps, loads -from bson.tz_util import utc -from pymongo.collection import Collection -from pymongo.database import Database -from pymongo.read_concern import ReadConcern -from pymongo.read_preferences import ReadPreference -from pymongo.write_concern import WriteConcern from ...package.loader import load +from ...package.canonical import name as named from ...schema import Attributes, Container -from ...schema.compat import odict +from ...schema.compat import str, unicode, odict from ..util import SENTINEL from .field import Field from .index import Index @@ -29,34 +23,20 @@ class Document(Container): This is the top-level class your own document schemas should subclass. They may also subclass eachother; field declaration order is preserved, with subclass fields coming after those provided by the parent class(es). Any - fields redefined will have their original position preserved. + fields redefined will have their original position preserved. Fields may be disabled in subclasses by assigning + `None` to the attribute within the subclass; this is not recommended, though supported to ease testing. - Documents may be bound to a PyMongo `Collection` instance, allowing for easier identification of where a document - has been loaded from, and to allow reloading and loading of previously un-projected data. This is not meant to - implement a full Active Record pattern; no `save` method or similar is provided. (You are free to add one - yourself, of course!) + Traits, Document sub-classes to be used as mix-ins, are provided to augment and specialize behaviour. """ # Note: These may be dynamic based on content; always access from an instance where possible. __store__ = odict # For fields, this may be a bson type like Binary, or Code. __foreign__ = {'object'} # The representation for the database side of things, ref: $type - __type_store__ = '_cls' # The pseudo-field to store embedded document class references as. - - __bound__ = False # Has this class been "attached" to a live MongoDB connection? - __collection__ = None # The name of the collection to "attach" to using bind(). - __read_preference__ = ReadPreference.PRIMARY # Default read preference to assign when binding. - __read_concern__ = ReadConcern() # Default read concern. - __write_concern__ = WriteConcern(w=1) # Default write concern. - __capped__ = False # The size of the capped collection to create in bytes. - __capped_count__ = None # The optional number of records to limit the capped collection to. - __engine__ = None # Override the default storage engine (and configuration) as a mapping of `{name: options}`. - __validate__ = 'off' # Control validation strictness: off, strict, or moderate. - __collation__ = None # A pymongo.collation.Collation object to control collation during creation. - - __projection__ = None # The set of fields used during projection, to identify fields which are not loaded. - __validator__ = None # The MongoDB Validation document matching these records. + __type_store__ = None # The pseudo-field to store embedded document class references as. + __pk__ = None # The primary key of the document, to make searchable if embedded, or the name of the '_id' field. + __fields__ = Attributes(only=Field) # An ordered mapping of field names to their respective Field instance. - __fields__.__sequence__ = 10000 + __fields__.__sequence__ = 10000 # TODO: project=False __indexes__ = Attributes(only=Index) # An ordered mapping of index names to their respective Index instance. __indexes__.__sequence__ = 10000 @@ -75,168 +55,32 @@ def __init__(self, *args, **kw): if prepare_defaults: self._prepare_defaults() - @classmethod - def _get_default_projection(cls): - """Construct the default projection document.""" - - projected = [] # The fields explicitly requested for inclusion. - neutral = [] # Fields returning neutral (None) status. - omitted = False # Have any fields been explicitly omitted? - - for name, field in cls.__fields__.items(): - if field.project is None: - neutral.append(name) - elif field.project: - projected.append(name) - else: - omitted = True - - if not projected and not omitted: - # No preferences specified. - return None - - elif not projected and omitted: - # No positive inclusions given, but negative ones were. - projected = neutral - - return {name: True for name in projected} - - @classmethod - def __attributed__(cls): - """Executed after each new subclass is constructed.""" - - cls.__projection__ = cls._get_default_projection() - def _prepare_defaults(self): """Trigger assignment of default values.""" for name, field in self.__fields__.items(): if field.assign: - getattr(self, name) - - @classmethod - def bind(cls, db=None, collection=None): - """Bind a copy of the collection to the class, modified per our class' settings.""" - - if db is collection is None: - raise ValueError("Must bind to either a database or explicit collection.") - - collection = cls.get_collection(db or collection) - - cls.__bound__ = True - cls._collection = collection - - return cls - - # Database Operations - - @classmethod - def _collection_configuration(cls, creation=False): - config = { - 'codec_options': CodecOptions( - document_class = cls.__store__, - tz_aware = True, - uuid_representation = STANDARD, - tzinfo = utc, - ), - 'read_preference': cls.__read_preference__, - 'read_concern': cls.__read_concern__, - 'write_concern': cls.__write_concern__, - } - - if not creation: - return config - - if cls.__capped__: - config['size'] = cls.__capped__ - config['capped'] = True - - if cls.__capped_count__: - config['max'] = cls.__capped_count__ - - if cls.__engine__: - config['storageEngine'] = cls.__engine__ - - if cls.__validate__ != 'off': - config['validator'] = cls.__validator__ - config['validationLevel'] = 'strict' if cls.__validate__ is True else cls.__validate__ - - if cls.__collation__ is not None: - config['collation'] = cls.__collation__ - - return config - - @classmethod - def create_collection(cls, target, recreate=False, indexes=True): - """Ensure the collection identified by this document class exists, creating it if not. - - http://api.mongodb.com/python/current/api/pymongo/database.html#pymongo.database.Database.create_collection - """ - - if isinstance(target, Collection): - name = target.name - target = target.database - else: - name = cls.__collection__ - - if recreate: - target.drop_collection(name) - - collection = target.create_collection(name, **cls._collection_configuration(True)) - - if indexes: - cls.create_indexes(collection) - - return collection - - @classmethod - def get_collection(cls, target): - """Retrieve a properly configured collection object as configured by this document class. - - If given an existing collection, will instead call `collection.with_options`. - - http://api.mongodb.com/python/current/api/pymongo/database.html#pymongo.database.Database.get_collection - """ - - if isinstance(target, Collection): - return target.with_options(**cls._collection_configuration()) - - elif isinstance(target, Database): - return target.get_collection(cls.__collection__, **cls._collection_configuration()) - - raise TypeError("Can not retrieve collection from: " + repr(target)) - - @classmethod - def create_indexes(cls, target, recreate=False): - """Iterate all known indexes and construct them.""" - - results = [] - collection = cls.get_collection(target) - - if recreate: - collection.drop_indexes() - - for index in cls.__indexes__.values(): - results.append(index.create_index(collection)) - - return results + getattr(self, name) # An attempt to retrieve the value of an assignable field will assign it. # Data Conversion and Casting @classmethod - def from_mongo(cls, doc, projected=None): + def from_mongo(cls, doc): """Convert data coming in from the MongoDB wire driver into a Document instance.""" - if isinstance(doc, Document): + if doc is None: # To support simplified iterative use, None should return None. + return None + + if isinstance(doc, Document): # No need to perform processing on existing Document instances. return doc - if cls.__type_store__ in doc: # Instantiate any specific class mentioned in the data. + if cls.__type_store__ and cls.__type_store__ in doc: # Instantiate specific class mentioned in the data. cls = load(doc[cls.__type_store__], 'marrow.mongo.document') - instance = cls(_prepare_defaults=False) - instance.__data__ = instance.__store__(doc) - instance._prepare_defaults() # pylint:disable=protected-access - instance.__loaded__ = set(projected) if projected else None + # Prepare a new instance in such a way that changes to the instance will be reflected in the originating doc. + instance = cls(_prepare_defaults=False) # Construct an instance, but delay default value processing. + instance.__data__ = doc # I am Popeye of Borg (pattern); you will be askimilgrated. + instance._prepare_defaults() # pylint:disable=protected-access -- deferred default value processing. return instance @@ -264,6 +108,59 @@ def as_rest(self): return self # We're sufficiently dictionary-like to pass muster. + # Python Magic Methods + + def __repr__(self, *args, **kw): + """A generic programmers' representation of documents. + + We add a little non-standard protocol on top of Python's own `__repr__`, allowing passing of additional + positional or keyword paramaters for inclusion in the result. This allows subclasses to define additional + information not based on simple field presence. + """ + + parts = [] + + if self.__pk__: + pk = getattr(self, self.__pk__, None) + + if isinstance(pk, ObjectId): + pk = unicode(pk) + elif isinstance(pk, (str, unicode)): + pass + else: + pk = repr(pk) + + parts.append(pk) + + parts.extend(args) + + for name, field in self.__fields__.items(): + if name == self.__pk__: + continue + + if field.repr is not None: + if callable(field.repr): + if not field.repr(self, field): + continue + else: + if not field.repr: + continue + + value = getattr(self, name, None) + + if value: + parts.append(name + "=" + repr(value)) + + for k in kw: + parts.append(k + "=" + repr(kw[k])) + + if self.__type_store__: + cls = self.get(self.__type_store__, named(self.__class__)) + else: + cls = self.__class__.__name__ + + return "{0}({1})".format(cls, ", ".join(parts)) + # Mapping Protocol def __getitem__(self, name): @@ -301,10 +198,7 @@ def items(self): return self.__data__.items() - def iteritems(self): - """Python 2 interation, as per items.""" - - return self.__data__.items() + iteritems = items # Python 2 interation, as per items. def values(self): """Iterate the values within the backing store.""" diff --git a/marrow/mongo/core/field/__init__.py b/marrow/mongo/core/field/__init__.py index 9fcaf8cd..d9d1604e 100644 --- a/marrow/mongo/core/field/__init__.py +++ b/marrow/mongo/core/field/__init__.py @@ -31,7 +31,7 @@ def native(self, value, context): # pylint:disable=signature-differs return value -@adjust_attribute_sequence(1000, 'transformer', 'validator', 'translated', 'assign', 'project', 'read', 'write') +@adjust_attribute_sequence(1000, 'transformer', 'validator', 'positional', 'assign', 'repr', 'project', 'read', 'write', 'sort') class Field(Attribute): # Possible values for operators include any literal $-prefixed operator, or one of: # * #rel -- allow/prevent all relative comparison such as $gt, $gte, $lt, etc. @@ -40,7 +40,7 @@ class Field(Attribute): # * #geo -- allow/prevent geographic query operators __allowed_operators__ = set() __disallowed_operators__ = set() - __document__ = None # If we're assigned to a Document, this gets populated with a weak reference proxy. + __document__ = None # The Document subclass the field originates from. __foreign__ = {} __acl__ = [] # Overall document access control list. @@ -57,11 +57,12 @@ class Field(Attribute): transformer = Attribute(default=FieldTransform()) # A Transformer class to use when loading/saving values. validator = Attribute(default=Validator()) # The Validator class to use when validating values. - translated = Attribute(default=False) # If truthy this field should be stored in the per-language subdocument. + positional = Attribute(default=True) # If True, will be accepted positionally. assign = Attribute(default=False) # If truthy attempt to access and store resulting variable when instantiated. # Security Properties + repr = Attribute(default=True) # Should this field be included in the programmers' representation? project = Attribute(default=None) # Predicate to indicate inclusion in the default projection. read = Attribute(default=True) # Read predicate, either a boolean, callable, or web.security ACL predicate. write = Attribute(default=True) # Write predicate, either a boolean, callable, or web.security ACL predicate. @@ -141,7 +142,7 @@ def __get__(self, obj, cls=None): # If this is class attribute (and not instance attribute) access, we return a Queryable interface. if obj is None: - return Q(self.__document__, self) + return Q(cls, self) result = super(Field, self).__get__(obj, cls) diff --git a/marrow/mongo/core/field/base.py b/marrow/mongo/core/field/base.py index 2f8cd11f..6979aedd 100644 --- a/marrow/mongo/core/field/base.py +++ b/marrow/mongo/core/field/base.py @@ -9,6 +9,7 @@ from bson import ObjectId as OID from . import Field +from ...util import utcnow from ....schema import Attribute from ....schema.compat import unicode @@ -17,8 +18,27 @@ class String(Field): __foreign__ = 'string' __disallowed_operators__ = {'#array'} + strip = Attribute(default=False) + case = Attribute(default=None) + def to_foreign(self, obj, name, value): # pylint:disable=unused-argument - return unicode(value) + value = unicode(value) + + if self.strip is True: + value = value.strip() + elif self.strip: + value = value.strip(self.strip) + + if self.case in (1, True, 'u', 'upper'): + value = value.upper() + + elif self.case in (-1, False, 'l', 'lower'): + value = value.lower() + + elif self.case == 'title': + value = value.title() + + return value class Binary(Field): @@ -95,13 +115,13 @@ class TTL(Date): def to_foreign(self, obj, name, value): # pylint:disable=unused-argument if isinstance(value, timedelta): - return datetime.utcnow() + value + return utcnow() + value if isinstance(value, datetime): return value if isinstance(value, Number): - return datetime.utcnow() + timedelta(days=value) + return utcnow() + timedelta(days=value) raise ValueError("Invalid TTL value: " + repr(value)) diff --git a/marrow/mongo/core/field/complex.py b/marrow/mongo/core/field/complex.py index 9bab2062..b219ec53 100644 --- a/marrow/mongo/core/field/complex.py +++ b/marrow/mongo/core/field/complex.py @@ -17,80 +17,102 @@ from ....schema.compat import odict, str, unicode -class _HasKinds(Field): +class _HasKind(Field): """A mix-in to provide an easily definable singular or plural set of document types.""" kind = Attribute(default=None) # One or more foreign model references, a string, Document subclass, or set of. - def __init__(self, *kinds, **kw): - if kinds: - kw['kind'] = kinds + def __init__(self, *args, **kw): + if args: + kw['kind'], args = args[0], args[1:] - super(_HasKinds, self).__init__(**kw) + super(_HasKind, self).__init__(*args, **kw) - @property - def kinds(self): - values = self.kind + def __fixup__(self, document): + super(_HasKind, self).__fixup__(document) - if isinstance(values, (str, unicode)) or not isinstance(values, Iterable): - values = (values, ) + kind = self.kind - for value in values: - if isinstance(value, (str, unicode)): - value = load(value, 'marrow.mongo.document') - - yield value + if not kind: + return + + if isinstance(kind, Field): + print("updating child field", self.__name__, repr(kind)) + kind.__name__ = self.__name__ + kind.__document__ = proxy(document) + kind.__fixup__(document) # Chain this down to embedded fields. + + def _kind(self, document=None): + kind = self.kind + + if isinstance(kind, (str, unicode)): + if kind.startswith('.'): + # This allows the reference to be dynamic. + kind = traverse(document or self.__document__, kind[1:]) + + if not isinstance(kind, (str, unicode)): + return kind + else: + kind = load(kind, 'marrow.mongo.document') + + return kind class _CastingKind(Field): def to_native(self, obj, name, value): # pylint:disable=unused-argument """Transform the MongoDB value into a Marrow Mongo value.""" - if not isinstance(value, Document): - - kinds = list(self.kinds) + from marrow.mongo.trait import Derived + + kind = self._kind(obj.__class__) + + if isinstance(value, Document): + if __debug__ and kind and issubclass(kind, Document) and not isinstance(value, kind): + raise ValueError("Not an instance of " + kind.__name__ + " or a sub-class: " + repr(value)) - if len(kinds) == 1: - if hasattr(kinds[0], 'transformer'): - value = kinds[0].transformer.native(value, (kinds[0], obj)) - else: - value = kinds[0].from_mongo(value) - else: - value = Document.from_mongo(value) # TODO: Pass in allowed classes. + return value - return value + if isinstance(kind, Field): + return kind.transformer.native(value, (kind, obj)) + + return (kind or Derived).from_mongo(value) def to_foreign(self, obj, name, value): # pylint:disable=unused-argument """Transform to a MongoDB-safe value.""" - kinds = list(self.kinds) + kind = self._kind(obj.__class__) - if not isinstance(value, Document): - if len(kinds) != 1: - raise ValueError("Ambigouous assignment, assign an instance of: " + \ - ", ".join(repr(kind) for kind in kinds)) - - kind = kinds[0] - - # Attempt to figure out what to do with the value. - if isinstance(kind, Field): - kind.__name__ = self.__name__ - return kind.transformer.foreign(value, (kind, obj)) + if isinstance(value, Document): + if __debug__ and kind and issubclass(kind, Document) and not isinstance(value, kind): + raise ValueError("Not an instance of " + kind.__name__ + " or a sub-class: " + repr(value)) - value = kind(**value) + return value - if isinstance(value, Document) and value.__type_store__ not in value and len(kinds) != 1: - value[value.__type_store__] = canon(value.__class__) # Automatically add the tracking field. + if isinstance(kind, Field): + return kind.transformer.foreign(value, (kind, obj)) + + if kind: + value = kind(**value) return value -class Array(_HasKinds, _CastingKind, Field): +class Array(_HasKind, _CastingKind, Field): __foreign__ = 'array' - __allowed_operators__ = {'#array', '$elemMatch'} + __allowed_operators__ = {'#array', '$elemMatch', '$eq'} class List(list): - pass + """Placeholder list shadow class to identify already-cast arrays.""" + + @classmethod + def new(cls): + return cls() + + def __init__(self, *args, **kw): + if kw.get('assign', False): + kw.setdefault('default', self.List.new) + + super(Array, self).__init__(*args, **kw) def to_native(self, obj, name, value): """Transform the MongoDB value into a Marrow Mongo value.""" @@ -109,29 +131,19 @@ def to_foreign(self, obj, name, value): return self.List(super(Array, self).to_foreign(obj, name, i) for i in value) -class Embed(_HasKinds, _CastingKind, Field): +class Embed(_HasKind, _CastingKind, Field): __foreign__ = 'object' __allowed_operators__ = {'#document'} - def to_native(self, obj, name, value): - """Transform the MongoDB value into a Marrow Mongo value.""" - - if isinstance(value, Document): - return value - - result = super(Embed, self).to_native(obj, name, value) - obj.__data__[self.__name__] = result - - return result - - def to_foreign(self, obj, name, value): - """Transform to a MongoDB-safe value.""" + def __init__(self, *args, **kw): + if args: + kw['kind'], args = args[0], args[1:] + kw.setdefault('default', lambda: self._kind()()) - result = super(Embed, self).to_foreign(obj, name, value) - return result + super(Embed, self).__init__(*args, **kw) -class Reference(_HasKinds, Field): +class Reference(_HasKind, Field): concrete = Attribute(default=False) # If truthy, will store a DBRef instead of ObjectId. cache = Attribute(default=None) # Attributes to preserve from the referenced object at the reference level. @@ -203,6 +215,8 @@ def to_foreign(self, obj, name, value): # pylint:disable=unused-argument if self.cache: return self._populate_cache(value) + identifier = value + # First, we handle the typcial Document object case. if isinstance(value, Document): identifier = value.__data__.get('_id', None) @@ -213,18 +227,18 @@ def to_foreign(self, obj, name, value): # pylint:disable=unused-argument try: identifier = OID(value) except InvalidId: - identifier = value - - kinds = list(self.kinds) + pass - if not isinstance(value, Document) and len(kinds) > 1: - raise ValueError("Passed an identifier (not a Document instance) when multiple document kinds registered.") + kind = self._kind(obj.__class__) if self.concrete: - if isinstance(value, Document): + if isinstance(value, Document) and value.__collection__: return DBRef(value.__collection__, identifier) - return DBRef(kinds[0].__collection__, identifier) + if kind and kind.__collection__: + return DBRef(kind.__collection__, identifier) + + raise ValueError("Could not infer collection name.") return identifier @@ -266,32 +280,36 @@ def to_foreign(self, obj, name, value): # pylint:disable=unused-argument namespace = self.namespace except AttributeError: namespace = None - if __debug__: - names = plugins = {} - else: - if __debug__: - names = {i.name: i.load() for i in iter_entry_points(namespace)} - plugins = {j: i for i, j in names.items()} try: explicit = self.explicit except AttributeError: explicit = not bool(namespace) - if isinstance(value, (str, unicode)): - if ':' in value: - if not explicit: - raise ValueError("Explicit object references not allowed.") + if not isinstance(value, (str, unicode)): + value = canon(value) + + if namespace and ':' in value: # Try to reduce to a known plugin short name. + for point in iter_entry_points(namespace): + qualname = point.module_name + + if point.attrs: + qualname += ':' + '.'.join(point.attrs) - elif __debug__ and namespace and value not in names: - raise ValueError('Unknown plugin "' + value + '" for namespace "' + namespace + '".') + if qualname == value: + value = point.name + break + + if ':' in value: + if not explicit: + raise ValueError("Explicit object references not allowed.") return value - if __debug__ and namespace and not explicit and value not in plugins: - raise ValueError(repr(value) + ' object is not a known plugin for namespace "' + namespace + '".') + if namespace and value not in (i.name for i in iter_entry_points(namespace)): + raise ValueError('Unknown plugin "' + value + '" for namespace "' + namespace + '".') - return plugins.get(value, canon(value)) + return value class Alias(Attribute): @@ -312,11 +330,12 @@ class Point(Document): path = Attribute() - def __init__(self, path): - super(Alias, self).__init__(path=path) + def __init__(self, path, **kw): + super(Alias, self).__init__(path=path, **kw) def __fixup__(self, document): """Called after an instance of our Field class is assigned to a Document.""" + self.__document__ = proxy(document) def __get__(self, obj, cls=None): diff --git a/marrow/mongo/core/field/md.py b/marrow/mongo/core/field/md.py new file mode 100644 index 00000000..0df76b03 --- /dev/null +++ b/marrow/mongo/core/field/md.py @@ -0,0 +1,40 @@ +# encoding: utf-8 + +from __future__ import unicode_literals + +from misaka import Markdown, HtmlRenderer # , SmartyPants +from misaka import HTML_ESCAPE, HTML_HARD_WRAP +from misaka import EXT_FENCED_CODE, EXT_NO_INTRA_EMPHASIS, EXT_AUTOLINK, EXT_SPACE_HEADERS, EXT_STRIKETHROUGH, EXT_SUPERSCRIPT + +from .base import String +from ....schema.compat import unicode, py3 + + +md = Markdown( + HtmlRenderer(flags=HTML_ESCAPE | HTML_HARD_WRAP), + extensions = EXT_FENCED_CODE | \ + EXT_NO_INTRA_EMPHASIS | \ + EXT_AUTOLINK | \ + EXT_SPACE_HEADERS | \ + EXT_STRIKETHROUGH | \ + EXT_SUPERSCRIPT + ) + + +class MarkdownString(unicode): + def __html__(self): + return md(self) + + +class Markdown(String): + def to_foreign(self, obj, name, value): # pylint:disable=unused-argument + if hasattr(value, '__markdown__'): + return value.__markdown__() + + if hasattr(value, 'as_markdown'): + return value.as_markdown + + return super(Markdown, self).to_foreign(obj, name, value) + + def to_native(self, obj, name, value): # pylint:disable=unused-argument + return MarkdownString(value) diff --git a/marrow/mongo/core/field/path.py b/marrow/mongo/core/field/path.py new file mode 100644 index 00000000..74a1a7b8 --- /dev/null +++ b/marrow/mongo/core/field/path.py @@ -0,0 +1,16 @@ +# encoding: utf-8 + +from __future__ import unicode_literals + +from .base import String +from ....schema.compat import unicode, py3 + +try: + from pathlib import PurePosixPath as _Path +except ImportError: + from pathlib2 import PurePosixPath as _Path + + +class Path(String): + def to_native(self, obj, name, value): # pylint:disable=unused-argument + return _Path(value) diff --git a/marrow/mongo/core/index.py b/marrow/mongo/core/index.py index 589a9cdb..e12d2695 100644 --- a/marrow/mongo/core/index.py +++ b/marrow/mongo/core/index.py @@ -62,10 +62,8 @@ def process_fields(self, fields): return result - def create_index(self, collection, **kw): - """Perform the translation needed to return the arguments for `Collection.create_index`. - - This is where final field name resolution happens, via the reference we have to the containing document class. + def create(self, collection, **kw): + """Create this index in the specified collection; keyword arguments are passed to PyMongo. http://api.mongodb.com/python/current/api/pymongo/collection.html#pymongo.collection.Collection.create_index """ @@ -89,3 +87,15 @@ def create_index(self, collection, **kw): del options[key] return collection.create_index(self.fields, **options) + + create_index = create + + def drop(self, collection): + """Drop this index from the specified collection. + + https://api.mongodb.com/python/current/api/pymongo/collection.html#pymongo.collection.Collection.drop_index + """ + + collection.drop_index(self.__name__) + + drop_index = drop diff --git a/marrow/mongo/core/trait/__init__.py b/marrow/mongo/core/trait/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/marrow/mongo/core/trait/collection.py b/marrow/mongo/core/trait/collection.py new file mode 100644 index 00000000..b1b4625b --- /dev/null +++ b/marrow/mongo/core/trait/collection.py @@ -0,0 +1,221 @@ +# encoding: utf-8 + +from __future__ import unicode_literals + +from bson.binary import STANDARD +from bson.codec_options import CodecOptions +from bson.tz_util import utc +from pymongo.collection import Collection as PyMongoCollection +from pymongo.database import Database +from pymongo.read_concern import ReadConcern +from pymongo.read_preferences import ReadPreference +from pymongo.write_concern import WriteConcern + +from ...trait import Identified + + +__all__ = ['Collection'] + + +class Collection(Identified): + """Allow this Document class to be bound to a real MongoDB database and collection. + + This extracts all "active-record"-like patterns from basic Document classes, eliminates the need to declare as a + sub-class of Identified (all top-level Collection objects are identified by default), and generally helps break + things apart into discrete groups of logical tasks. + + Currently true Active Record pattern access is not supported, nor encouraged. This provides a shortcut, as + mentioned above, and access to common collection-level activites in ways utilizing positional and parametric + helpers. Not all collection manipulating methods are proxied here; only the ones benefitting from assistance + from Marrow Mongo in terms of binding or index awareness. + + For other operations (such as `drop`, `reindex`, etc.) it is recommended to `get_collection()` and explicitly + utilize the PyMongo API. This helps reduce the liklihood of a given interface breaking with changes to PyMongo, + avoids clutter, and allows you to use some of these method names yourself. + """ + + # Metadata Defaults + + # Collection Binding + __bound__ = None # Has this class been "attached" to a live MongoDB connection? If so, this is the collection. + __collection__ = None # The name of the collection to "attach to" using a call to `bind()`. + __projection__ = None # The set of fields used during projection, to identify fields which are not loaded. + + # Data Access Options + # TODO: Attribute declaration and name allowance. + __read_preference__ = ReadPreference.PRIMARY # Default read preference to assign when binding. + __read_concern__ = ReadConcern() # Default read concern. + __write_concern__ = WriteConcern(w=1) # Default write concern. + + # Storage Options + __capped__ = False # The size of the capped collection to create in bytes. + __capped_count__ = None # The optional number of records to limit the capped collection to. + __engine__ = None # Override the default storage engine (and configuration) as a mapping of `{name: options}`. + __collation__ = None # A `pymongo.collation.Collation` object to control collation during creation. + + # Data Validation Options + __validate__ = 'off' # Control validation strictness: `off`, `strict`, or `moderate`. TODO: bool/None equiv. + __validator__ = None # The MongoDB Validation document matching these records. + + @classmethod + def __attributed__(cls): + """Executed after each new subclass is constructed.""" + + cls.__projection__ = cls._get_default_projection() + + # Data Access Binding + + @classmethod + def bind(cls, target): + """Bind a copy of the collection to the class, modified per our class' settings. + + The given target (and eventual collection returned) must be safe within the context the document sublcass + being bound is constructed within. E.g. at the module scope this binding must be thread-safe. + """ + + if cls.__bound__ is not None: + return cls + + cls.__bound__ = cls.get_collection(target) + + return cls + + # Collection Management + + @classmethod + def get_collection(cls, target=None): + """Retrieve a properly configured collection object as configured by this document class. + + If given an existing collection, will instead call `collection.with_options`. + + http://api.mongodb.com/python/current/api/pymongo/database.html#pymongo.database.Database.get_collection + """ + + if target is None: + assert cls.__bound__ is not None, "Target required when document class not bound." + return cls.__bound__ + + if isinstance(target, PyMongoCollection): + return target.with_options(**cls._collection_configuration()) + + elif isinstance(target, Database): + return target.get_collection(cls.__collection__, **cls._collection_configuration()) + + raise TypeError("Can not retrieve collection from: " + repr(target)) + + @classmethod + def create_collection(cls, target=None, drop=False, indexes=True): + """Ensure the collection identified by this document class exists, creating it if not, also creating indexes. + + **Warning:** enabling the `recreate` option **will drop the collection, erasing all data within**. + + http://api.mongodb.com/python/current/api/pymongo/database.html#pymongo.database.Database.create_collection + """ + + if target is None: + assert cls.__bound__ is not None, "Target required when document class not bound." + target = cls.__bound__ + + if isinstance(target, PyMongoCollection): + collection = target.name + target = target.database + elif isinstance(target, Database): + collection = cls.__collection__ + else: + raise TypeError("Can not retrieve database from: " + repr(target)) + + if drop: + target.drop_collection(collection) # TODO: If drop fails, try just emptying? + + collection = target.create_collection(collection, **cls._collection_configuration(True)) + + if indexes: + cls.create_indexes(collection) + + return collection + + # Index Management + + @classmethod + def create_indexes(cls, target=None, recreate=False): + """Iterate all known indexes and construct them. + + https://api.mongodb.com/python/current/api/pymongo/collection.html#pymongo.collection.Collection.create_indexes + """ + + # TODO: Nested indexes. + + results = [] + collection = cls.get_collection(target) + + if recreate: + collection.drop_indexes() + + for index in cls.__indexes__.values(): + results.append(index.create(collection)) + + return results + + # Semi-Private Data Transformation + + @classmethod + def _collection_configuration(cls, creation=False): + config = { + 'codec_options': CodecOptions( + document_class = cls.__store__, + tz_aware = True, + uuid_representation = STANDARD, + tzinfo = utc, + ), + 'read_preference': cls.__read_preference__, + 'read_concern': cls.__read_concern__, + 'write_concern': cls.__write_concern__, + } + + if not creation: + return config + + if cls.__capped__: + config['size'] = cls.__capped__ + config['capped'] = True + + if cls.__capped_count__: + config['max'] = cls.__capped_count__ + + if cls.__engine__: + config['storageEngine'] = cls.__engine__ + + if cls.__validate__ != 'off': + config['validator'] = cls.__validator__ + config['validationLevel'] = 'strict' if cls.__validate__ is True else cls.__validate__ + + if cls.__collation__ is not None: # pragma: no cover + config['collation'] = cls.__collation__ + + return config + + @classmethod + def _get_default_projection(cls): + """Construct the default projection document.""" + + projected = [] # The fields explicitly requested for inclusion. + neutral = [] # Fields returning neutral (None) status. + omitted = False # Have any fields been explicitly omitted? + + for name, field in cls.__fields__.items(): + if field.project is None: + neutral.append(name) + elif field.project: + projected.append(name) + else: + omitted = True + + if not projected and not omitted: + # No preferences specified. + return None + + elif not projected and omitted: + # No positive inclusions given, but negative ones were. + projected = neutral + + return {field: True for field in projected} diff --git a/marrow/mongo/core/trait/derived.py b/marrow/mongo/core/trait/derived.py new file mode 100644 index 00000000..e8c09677 --- /dev/null +++ b/marrow/mongo/core/trait/derived.py @@ -0,0 +1,25 @@ +# encoding: utf-8 + +from ... import Document, Index +from ...field import PluginReference + + +class Derived(Document): + """Access and store the class reference to a particular Document subclass. + + This allows for easy access to the magical `_cls` key as the `cls` attribute. + """ + + __type_store__ = '_cls' # The pseudo-field to store embedded document class references as. + + cls = PluginReference('marrow.mongo.document', '_cls', explicit=True, repr=False) + cls.__sequence__ += 1000000 + + _cls = Index('cls') + + def __init__(self, *args, **kw): + """Automatically derive and store the class path or plugin reference name.""" + + super(Derived, self).__init__(*args, **kw) + + self.cls = self.__class__ diff --git a/marrow/mongo/core/trait/expires.py b/marrow/mongo/core/trait/expires.py new file mode 100644 index 00000000..1d8dbc51 --- /dev/null +++ b/marrow/mongo/core/trait/expires.py @@ -0,0 +1,44 @@ +# encoding: utf-8 + +from ... import Document, Index, utcnow +from ...field import TTL + + +class Expires(Document): + """Record auto-expiry field with supporting TTL index and properties.""" + + # ## Expiry Field Definition + + expires = TTL(default=None, write=False) # The exact time the record should be considered "expired". + + # ## TTL Index Definition + + _expires = Index('expires', expire=0, sparse=True) + + # ## Accessor Properties + + @property + def is_expired(self): + """Determine if this document has already expired. + + We need this because MongoDB TTL indexes are culled once per minute on an as-able basis, meaning records might + be available for up to 60 seconds after their expiry time normally and that if there are many records to cull, + may be present even longer. + """ + if not self.expires: + return None # None is falsy, preserving the "no, it's not expired" interpretation, but still flagging. + + return self.expires <= utcnow() + + # ## Cooperative Behaviours + + @classmethod + def from_mongo(cls, data, expired=False, **kw): + """In the event a value that has technically already expired is loaded, swap it for None.""" + + value = super(Expires, cls).from_mongo(data, **kw) + + if not expired and value.is_expired: + return None + + return value diff --git a/marrow/mongo/core/trait/identified.py b/marrow/mongo/core/trait/identified.py new file mode 100644 index 00000000..b2aba98a --- /dev/null +++ b/marrow/mongo/core/trait/identified.py @@ -0,0 +1,38 @@ +# encoding: utf-8 + +from ... import Document +from ...field import ObjectId + + +class Identified(Document): + """A document utilizing this trait mix-in contains a MongoDB _id key. + + This provides a read-only property to retrieve the creation time as `created`. + + Identifiers are constructed on document instantiation; this means inserts are already provided an ID, bypassing + the driver's behaviour of only returning one after a successful insert. This allows for the pre-construction + of graphs of objects prior to any of them being saved, though, until all references are resolveable, the data + is effectively in a broken, inconsistent state. (Use bulk updates and plan for rollback in the event of failure!) + + No need for an explicit index on this as MongoDB will provide one automatically. + """ + + __pk__ = 'id' + + id = ObjectId('_id', assign=True, write=False, repr=False) + + def __eq__(self, other): + """Equality comparison between the IDs of the respective documents.""" + + if isinstance(other, Document): + return self.id == other.id + + return self.id == other + + def __ne__(self, other): + """Inverse equality comparison between the backing store and other value.""" + + if isinstance(other, Document): + return self.id != other.id + + return self.id != other diff --git a/marrow/mongo/core/trait/localized.py b/marrow/mongo/core/trait/localized.py new file mode 100644 index 00000000..862e510f --- /dev/null +++ b/marrow/mongo/core/trait/localized.py @@ -0,0 +1,64 @@ +# encoding: utf-8 + +from ... import Document +from ...field import String, Array, Embed, Alias +from ....package.loader import traverse +from ....schema.compat import odict + + +LANGUAGES = {'en', 'fr', 'it', 'de', 'es', 'pt', 'ru'} # EFIGSPR, ISO 639-1 +LANGUAGES |= {'da', 'nl', 'fi', 'hu', 'nb', 'pt', 'ro', 'sv', 'tr'} # Additional ISO 639-1 +LANGUAGES |= {'ara', 'prs', 'pes', 'urd'} # ISO 636-3 +LANGUAGES |= {'zhs', 'zht'} # RLP + + +class Translated(Alias): + """Reference a localized field, providing a mapping interface to the translations. + + class MyDocument(Localized, Document): + class Locale(Localized.Locale): + title = String() + + title = Translated('title') + + # Query + MyDocument.title == "Hello world!" + + inst = MyDocument([Locale('en', "Hi."), Locale('fr', "Bonjour.")]) + assert inst.title == {'en': "Hi.", 'fr': "Bonjour."} + """ + + def __init__(self, path, **kw): + super(Translated, self).__init__(path='locale.' + path, **kw) + + def __get__(self, obj, cls=None): + if obj is None: + return super(Translated, self).__get__(obj, cls) + + collection = odict() + path = self.path[7:] + + for locale in obj.locale: + collection[locale.language] = traverse(locale, path) + + return collection + + def __set__(self, obj, value): + raise TypeError("Can not assign to a translated alias.") # TODO + + +class Localized(Document): + """The annotated Document contains localized data.""" + + class Locale(Document): + __pk__ = 'language' + + language = String(choices=LANGUAGES, default='en') + + locale = Array(Embed('.Locale'), default=lambda: [], assign=True, repr=False) + + def __repr__(self): + if self.locale: + return super(Localized, self).__repr__('{' + ', '.join(i.language for i in self.locale) + '}') + + return super(Localized, self).__repr__() diff --git a/marrow/mongo/core/trait/published.py b/marrow/mongo/core/trait/published.py new file mode 100644 index 00000000..83ec8c82 --- /dev/null +++ b/marrow/mongo/core/trait/published.py @@ -0,0 +1,49 @@ +# encoding: utf-8 + +"""Data model trait mix-in for tracking record publication and retraction.""" + +from datetime import timedelta + +from ... import Document, Index, utcnow +from ...field import Date +from ...util import utcnow + + +class Published(Document): + created = Date(default=utcnow, assign=True, write=False) + modified = Date(default=None, write=False) + published = Date(default=None) + retracted = Date(default=None) + + _availability = Index('published', 'retracted') + + @classmethod + def only_published(cls, at=None): + """Produce a query fragment suitable for selecting documents public. + + Now (no arguments), at a specific time (datetime argument), or relative to now (timedelta). + """ + + if isinstance(at, timedelta): + at = utcnow() + at + else: + at = at or utcnow() + + pub, ret = cls.published, cls.retracted + + publication = (-pub) | (pub == None) | (pub <= at) + retraction = (-ret) | (ret == None) | (ret > at) + + return publication & retraction + + @property + def is_published(self): + now = utcnow() + + if self.published and self.published > now: + return False + + if self.retracted and self.retracted < now: + return False + + return True diff --git a/marrow/mongo/core/trait/queryable.py b/marrow/mongo/core/trait/queryable.py new file mode 100644 index 00000000..99c38421 --- /dev/null +++ b/marrow/mongo/core/trait/queryable.py @@ -0,0 +1,310 @@ +# encoding: utf-8 + +from __future__ import unicode_literals + +from collections import Mapping +from functools import reduce +from operator import and_ + +from pymongo.cursor import CursorType + +from ... import F, Filter, P, S +from ...trait import Collection +from ....schema.compat import odict +from ....package.loader import traverse + + +class Queryable(Collection): + """EXPERIMENTAL: Extend active collection behaviours to include querying.""" + + UNIVERSAL_OPTIONS = { + 'collation', + 'limit', + 'projection', + 'skip', + 'sort', + } + + FIND_OPTIONS = UNIVERSAL_OPTIONS | { + 'allow_partial_results', + 'await', + 'batch_size', + 'cursor_type', + 'max_time_ms', # translated -> modifiers['$maxTimeMS'] + 'modifiers', + 'no_cursor_timeout', + 'oplog_replay', + 'tail', + } + + FIND_MAPPING = { + 'allowPartialResults': 'allow_partial_results', + 'batchSize': 'batch_size', + 'cursorType': 'cursor_type', + 'maxTimeMS': 'max_time_ms', # See above. + 'maxTimeMs': 'max_time_ms', # Common typo. + 'noCursorTimeout': 'no_cursor_timeout', + 'oplogReplay': 'oplog_replay', + } + + AGGREGATE_OPTIONS = UNIVERSAL_OPTIONS | { + 'allowDiskUse', + 'batchSize', + 'maxTimeMS', + 'useCursor', + } + + AGGREGATE_MAPPING = { + 'allow_disk_use': 'allowDiskUse', + 'batch_size': 'batchSize', + 'maxTimeMs': 'maxTimeMS', # Common typo. + 'max_time_ms': 'maxTimeMS', + 'use_cursor': 'useCursor', + } + + @classmethod + def _prepare_query(cls, mapping, valid, *args, **kw): + """Process arguments to query methods. For internal use only. + + Positional arguments are treated as query components, combined using boolean AND reduction. + + Keyword arguments are processed depending on the passed in mapping and set of valid options, with non- + option arguments treated as parametric query components, also ANDed with any positionally passed query + components. + + Parametric querying with explicit `__eq` against these "reserved words" is possible to work around their + reserved-ness. + + Querying options for find and aggregate may differ in use of under_score or camelCase formatting; this + helper removes the distinction and allows either. + """ + + collection = cls.get_collection(kw.pop('source', None)) + query = Filter(document=cls, collection=collection) + options = {} + + if args: + query &= reduce(and_, args) + + # Gather any valid options. + for key in tuple(kw): + name = mapping.get(key, key) + + if name in valid: + options[name] = kw.pop(key) + + # Support parametric projection via the use of iterables of strings in the form 'field' or '-field', + # with name resolution. See the documentation for P for details. + if 'projection' in options and not isinstance(options['projection'], Mapping): + options['projection'] = P(cls, *options['projection']) + + # Support parametric sorting via the use of iterables of strings. See the documentation for S for details. + if 'sort' in options: + options['sort'] = S(cls, *options['sort']) + + if kw: # Remainder are parametric query fragments. + query &= F(cls, **kw) + + return cls, collection, query, options + + @classmethod + def _prepare_find(cls, *args, **kw): + """Execute a find and return the resulting queryset using combined plain and parametric query generation. + + Additionally, performs argument case normalization, refer to the `_prepare_query` method's docstring. + """ + + cls, collection, query, options = cls._prepare_query( + cls.FIND_MAPPING, + cls.FIND_OPTIONS, + *args, + **kw + ) + + if 'cursor_type' in options and {'tail', 'await'} & set(options): + raise TypeError("Can not combine cursor_type and tail/await arguments.") + + elif options.pop('tail', False): + options['cursor_type'] = CursorType.TAILABLE_AWAIT if options.pop('await', True) else CursorType.TAILABLE + + elif 'await' in options: + raise TypeError("Await option only applies to tailing cursors.") + + modifiers = options.get('modifiers', dict()) + + if 'max_time_ms' in options: + modifiers['$maxTimeMS'] = options.pop('max_time_ms') + + if modifiers: + options['modifiers'] = modifiers + + return cls, collection, query, options + + @classmethod + def _prepare_aggregate(cls, *args, **kw): + """Generate and execute an aggregate query pipline using combined plain and parametric query generation. + + Additionally, performs argument case normalization, refer to the `_prepare_query` method's docstring. + + This provides a find-like interface for generating aggregate pipelines with a few shortcuts that make + aggregates behave more like "find, optionally with more steps". Positional arguments that are not Filter + instances are assumed to be aggregate pipeline stages. + + https://api.mongodb.com/python/current/api/pymongo/collection.html#pymongo.collection.Collection.aggregate + """ + + stages = [] + stage_args = [] + fragments = [] + + for arg in args: # Split the positional arguments into filter fragments and projection stages. + (fragments if isinstance(arg, Filter) else stage_args).append(arg) + + cls, collection, query, options = cls._prepare_query( + cls.AGGREGATE_MAPPING, + cls.AGGREGATE_OPTIONS, + *fragments, + **kw + ) + + if query: + stages.append({'$match': query}) + + stages.extend(stage_args) + + if 'sort' in options: # Convert the find-like option to a stage with the correct semantics. + stages.append({'$sort': odict(options.pop('sort'))}) + + if 'skip' in options: # Note: Sort + limit memory optimization invalidated when skipping. + stages.append({'$skip': options.pop('skip')}) + + if 'limit' in options: + stages.append({'$limit': options.pop('limit')}) + + if 'projection' in options: + stages.append({'$project': options.pop('projection')}) + + if 'skip' in options: + stages.append({'$skip': options.pop('skip')}) + + return cls, collection, stages, options + + @classmethod + def find(cls, *args, **kw): + """Query the collection this class is bound to. + + Additional arguments are processed according to `_prepare_find` prior to passing to PyMongo, where positional + parameters are interpreted as query fragments, parametric keyword arguments combined, and other keyword + arguments passed along with minor transformation. + + https://api.mongodb.com/python/current/api/pymongo/collection.html#pymongo.collection.Collection.find + """ + + Doc, collection, query, options = cls._prepare_find(*args, **kw) + return collection.find(query, **options) + + @classmethod + def find_one(cls, *args, **kw): + """Get a single document from the collection this class is bound to. + + Additional arguments are processed according to `_prepare_find` prior to passing to PyMongo, where positional + parameters are interpreted as query fragments, parametric keyword arguments combined, and other keyword + arguments passed along with minor transformation. + + Automatically calls `to_mongo` with the retrieved data. + + https://api.mongodb.com/python/current/api/pymongo/collection.html#pymongo.collection.Collection.find_one + """ + + if len(args) == 1 and not isinstance(args[0], Filter): + args = (cls.id == args[0], ) + + Doc, collection, query, options = cls._prepare_find(*args, **kw) + result = Doc.from_mongo(collection.find_one(query, **options)) + + return result + + # https://api.mongodb.com/python/current/api/pymongo/collection.html#pymongo.collection.Collection.find_one_and_delete + # https://api.mongodb.com/python/current/api/pymongo/collection.html#pymongo.collection.Collection.find_one_and_replace + # https://api.mongodb.com/python/current/api/pymongo/collection.html#pymongo.collection.Collection.find_one_and_update + + @classmethod + def find_in_sequence(cls, field, order, *args, **kw): + """Return a QuerySet iterating the results of a query in a defined order. Technically an aggregate. + + To be successful one must be running MongoDB 3.4 or later. Document order will not be represented otherwise. + + Based on the technique described here: http://s.webcore.io/2O3i0N2E3h0r + See also: https://jira.mongodb.org/browse/SERVER-7528 + """ + + field = traverse(cls, field) + order = list(order) # We need to coalesce the value to prepare for multiple uses. + kw['sort'] = {'__order': 1} + kw.setdefault('projection', {'__order': 0}) + + cls, collection, stages, options = cls._prepare_aggregate( + field.any(order), + {'$addFields': {'__order': {'$indexOfArray': [order, '$' + ~field]}}}, + *args, + **kw + ) + + if tuple(collection.database.client.server_info()['versionArray'][:2]) < (3, 4): # noqa + raise RuntimeError("Queryable.find_in_sequence only works against MongoDB server versions 3.4 or newer.") + + return collection.aggregate(stages, **options) + + # https://api.mongodb.com/python/current/api/pymongo/collection.html#pymongo.collection.Collection.count + # https://api.mongodb.com/python/current/api/pymongo/collection.html#pymongo.collection.Collection.distinct + # https://api.mongodb.com/python/current/api/pymongo/collection.html#pymongo.collection.Collection.group + # https://api.mongodb.com/python/current/api/pymongo/collection.html#pymongo.collection.Collection.map_reduce + # https://api.mongodb.com/python/current/api/pymongo/collection.html#pymongo.collection.Collection.inline_map_reduce + # https://api.mongodb.com/python/current/api/pymongo/collection.html#pymongo.collection.Collection.parallel_scan + # https://api.mongodb.com/python/current/api/pymongo/collection.html#pymongo.collection.Collection.initialize_unordered_bulk_op + # https://api.mongodb.com/python/current/api/pymongo/collection.html#pymongo.collection.Collection.initialize_ordered_bulk_op + + def reload(self, *fields, **kw): + """Reload the entire document from the database, or refresh specific named top-level fields.""" + + Doc, collection, query, options = self._prepare_find(id=self.id, projection=fields, **kw) + result = collection.find_one(query, **options) + + if fields: # Refresh only the requested data. + for k in result: # TODO: Better merge algorithm. + if k == ~Doc.id: continue + self.__data__[k] = result[k] + else: + self.__data__ = result + + return self + + def insert_one(self, **kw): + """Insert this document, passing any additional arguments to PyMongo. + + https://api.mongodb.com/python/current/api/pymongo/collection.html#pymongo.collection.Collection.insert_one + """ + + collection = self.get_collection(kw.pop('source', None)) + return collection.insert_one(self, **kw) + + # https://api.mongodb.com/python/current/api/pymongo/collection.html#pymongo.collection.Collection.insert_many + + #def replace(self, *args, **kw): + # """Replace a single document matching the filter with this document, passing additional arguments to PyMongo. + # + # **Warning:** Be careful if the current document has only been partially projected, as the omitted fields will + # either be dropped or have their default values saved where `assign` is `True`. + # + # https://api.mongodb.com/python/current/api/pymongo/collection.html#pymongo.collection.Collection.replace_one + # """ + # pass + # https://api.mongodb.com/python/current/api/pymongo/collection.html#pymongo.collection.Collection.replace_one + + # update + # https://api.mongodb.com/python/current/api/pymongo/collection.html#pymongo.collection.Collection.update_one + # https://api.mongodb.com/python/current/api/pymongo/collection.html#pymongo.collection.Collection.update_many + + # delete + # https://api.mongodb.com/python/current/api/pymongo/collection.html#pymongo.collection.Collection.delete_one + # https://api.mongodb.com/python/current/api/pymongo/collection.html#pymongo.collection.Collection.delete_many diff --git a/marrow/mongo/param/project.py b/marrow/mongo/param/project.py index d1b5b949..7cfdff7a 100644 --- a/marrow/mongo/param/project.py +++ b/marrow/mongo/param/project.py @@ -24,7 +24,7 @@ def P(Document, *fields, **kw): projected.add(field) if not projected: # We only have exclusions from the default projection. - names = set(Document.__projection__.keys() if Document.__projection__ else Document.__fields__.keys()) + names = set(getattr(Document, '__projection__', Document.__fields__) or Document.__fields__) projected = {name for name in (names - omitted)} projected |= __always__ diff --git a/marrow/mongo/param/sort.py b/marrow/mongo/param/sort.py index cd7776bf..c0a987f4 100644 --- a/marrow/mongo/param/sort.py +++ b/marrow/mongo/param/sort.py @@ -21,7 +21,9 @@ def S(Document, *fields): continue direction = ASCENDING - field = field.replace('__', '.') + + if not field.startswith('__'): + field = field.replace('__', '.') if field[0] == '-': direction = DESCENDING @@ -29,6 +31,8 @@ def S(Document, *fields): if field[0] in ('+', '-'): field = field[1:] - result.append((~traverse(Document, field), direction)) + _field = traverse(Document, field, default=None) + + result.append(((~_field) if _field else field, direction)) return result diff --git a/marrow/mongo/param/update.py b/marrow/mongo/param/update.py index abf9bf7a..995537a7 100644 --- a/marrow/mongo/param/update.py +++ b/marrow/mongo/param/update.py @@ -4,6 +4,7 @@ from __future__ import unicode_literals +from collections import Mapping from operator import __neg__ from ...schema.compat import unicode @@ -12,6 +13,28 @@ DEFAULT_UPDATE = 'set' # If no prefix is found, this will be the default operation. + +def _push_each(value, field): + value = list(field.transformer.foreign(v, (field, field.__document__)) for v in value) + return {'$each': value} + + +def _push_slice(value): + return {'$slice': int(value)} + + +def _push_sort(value): + return {'$sort': value} + + +def _push_position(value): + return {'$position': int(value)} + + +# Identify the casting functions we pass the field definition to, as these will need to utilize it in some way. +UPDATE_MAGIC = {_push_each} + + # These allow us to easily override the interpretation of any particular operation and introduce new ones. # The keys represent prefixes, the values may be either a string (which will be prefixed with '$' automatically) or # a tuple of the same plus a callable to filter the argument's value prior to integration. A third tuple value may @@ -28,6 +51,11 @@ 'now': ('currentDate', _current_date), # A shortcut for the longer form. 'pull_all': 'pullAll', # Unserscore to camel case conversion. 'push_all': 'pushAll', # Unserscore to camel case conversion. + 'push_each': ('push', _push_each), # The longer form: value(s) to push. + 'push_pos': ('push', _push_position), # The longer form: push at a specific index. + 'push_position': ('push', _push_position), # The longer form: push at a specific index. + 'push_slice': ('push', _push_slice), # The longer form: limit number of elements. + 'push_sort': ('push', _push_sort), # The longer form: order the results after modification. 'rename': ('rename', unicode), # Typecast to unicode. 'set_on_insert': 'setOnInsert', # Underscore to camel case conversion. 'soi': 'setOnInsert', # A shortcut for the longer form. @@ -35,11 +63,36 @@ } # Update with passthrough values. -UPDATE_ALIASES.update({i: i for i in {'bit', 'inc', 'max', 'min', 'mul', 'pull', 'pullALl', 'push', 'pushAll', - 'rename', 'set', 'setOnInsert', 'unset'}}) +UPDATE_ALIASES.update({i: i for i in { + 'bit', + 'inc', + 'max', + 'min', + 'mul', + 'pull', + 'pullAll', + 'push', + 'pushAll', + 'rename', + 'set', + 'setOnInsert', + 'unset', + }}) # These should not utilize field to_foreign typecasting. -UPDATE_PASSTHROUGH = {'rename', 'unset', 'pull', 'push', 'bit', 'currentDate'} +UPDATE_PASSTHROUGH = { + 'bit', + 'currentDate', + 'pull', + 'push', + 'push_each', + 'push_pos', + 'push_position', + 'push_slice', + 'push_sort', + 'rename', + 'unset', + } def U(Document, __raw__=None, **update): @@ -60,9 +113,19 @@ def U(Document, __raw__=None, **update): operation = DEFAULT_UPDATE if isinstance(operation, tuple): - operation, cast = operation - value = cast(value) + operation, cast = ('$' + operation[0]), operation[1] + if cast in UPDATE_MAGIC: + value = cast(value, field) + else: + value = cast(value) + + if operation in ops and ~field in ops[operation] and isinstance(value, Mapping): + ops[operation][~field].update(value) + continue + + else: + operation = '$' + operation - ops &= Update({'$' + operation: {~field: value}}) + ops &= Update({operation: {~field: value}}) return ops diff --git a/marrow/mongo/query/ops.py b/marrow/mongo/query/ops.py index 28150aa5..c78dbc9b 100644 --- a/marrow/mongo/query/ops.py +++ b/marrow/mongo/query/ops.py @@ -14,7 +14,7 @@ from ..util import SENTINEL -class Ops(object): +class Ops(MutableMapping): __slots__ = ('operations', 'collection', 'document') def __init__(self, operations=None, collection=None, document=None): @@ -103,8 +103,6 @@ def setdefault(self, key, value=None): return self.operations.setdefault(key, value) -MutableMapping.register(Ops) # Metaclass conflict if we subclass. - class Filter(Ops): __slots__ = ('operations', 'collection', 'document') @@ -112,24 +110,36 @@ class Filter(Ops): # Binary Operator Protocols def __and__(self, other): + """Boolean AND joining of filter operations.""" operations = deepcopy(self.operations) - other = self.__class__( - operations = other.as_query if hasattr(other, 'as_query') else other, - collection = self.collection, - document = self.document - ) + other = other.as_query if hasattr(other, 'as_query') else other - for k, v in getattr(other, 'operations', {}).items(): + for k, v in other.items(): if k not in operations: operations[k] = v - else: - if not isinstance(operations[k], Mapping): - operations[k] = odict((('$eq', operations[k]), )) + continue + + if k == '$and': + operations.setdefault('$and', []) + operations['$and'].extend(v) + continue + + elif k == '$or': + operations.setdefault('$and', []) + operations['$and'].append(odict(((k, v), ))) - if not isinstance(v, Mapping): - v = odict((('$eq', v), )) + if '$or' in operations: + operations['$and'].append(odict((('$or', operations.pop('$or')), ))) - operations[k].update(v) + continue + + if not isinstance(operations[k], Mapping): + operations[k] = odict((('$eq', operations[k]), )) + + if not isinstance(v, Mapping): + v = odict((('$eq', v), )) + + operations[k].update(v) return self.__class__(operations=operations, collection=self.collection, document=self.document) diff --git a/marrow/mongo/query/query.py b/marrow/mongo/query/query.py index 29d308f1..4cc0219d 100644 --- a/marrow/mongo/query/query.py +++ b/marrow/mongo/query/query.py @@ -9,10 +9,11 @@ from __future__ import unicode_literals from copy import copy +from collections import Iterable from functools import reduce from operator import __and__, __or__, __xor__ -from ...schema.compat import py3, unicode +from ...schema.compat import py3, str, unicode from .ops import Filter if __debug__: @@ -76,21 +77,39 @@ def __getattr__(self, name): if self._combining: # If we are combining fields, we can not dereference further. raise AttributeError() - if not hasattr(self._field, 'kinds'): + if not hasattr(self._field, '_kind'): return getattr(self._field, name) - for kind in self._field.kinds: - if hasattr(kind, '__fields__'): - if name in kind.__fields__: - return self.__class__(self._document, kind.__fields__[name], self._name + '.') - - try: - return getattr(kind, name) - except AttributeError: - pass + kind = self._field + while getattr(kind, '_kind', None): + kind = kind._kind(self._document) + + if hasattr(kind, '__fields__') and name in kind.__fields__: + return self.__class__(self._document, kind.__fields__[name], self._name + '.') + + try: + return getattr(kind, name) + except AttributeError: + pass + + try: # Pass through to the field itself as a last resort. + return getattr(self._field, name) + except AttributeError: + pass raise AttributeError() + def __setattr__(self, name, value): + """Assign an otherwise unknown attribute on the targeted field instead.""" + + if name.startswith('_'): + return super(Q, self).__setattr__(name, value) + + if self._combining: + raise AttributeError() + + setattr(self._field, name, value) + def __getitem__(self, name): """Allows for referencing specific array elements by index. @@ -111,13 +130,13 @@ def __getitem__(self, name): if not isinstance(name, int) and not name.isdigit(): raise KeyError("Must specify an index as either a number or string representation of a number: " + name) - field = next(self._field.kinds) + field = self._field._kind(self._document) if isinstance(field, Field): # Bare simple values. field = copy(field) field.__name__ = self._name + '.' + unicode(name) - elif issubclass(field, Document): # Embedded documents. + else: # Embedded documents. field = Embed(field, name=self._name + '.' + unicode(name)) return self.__class__(self._document, field) diff --git a/marrow/mongo/util/__init__.py b/marrow/mongo/util/__init__.py index efebd270..76683706 100644 --- a/marrow/mongo/util/__init__.py +++ b/marrow/mongo/util/__init__.py @@ -2,18 +2,44 @@ from __future__ import unicode_literals +from collections import OrderedDict +from datetime import datetime +from operator import attrgetter +from bson.tz_util import utc + +from marrow.schema.meta import ElementMeta from ...package.loader import load SENTINEL = object() # Singleton value to detect unassigned values. -def adjust_attribute_sequence(amount=10000, *fields): +def adjust_attribute_sequence(*fields): """Move marrow.schema fields around to control positional instantiation order.""" + amount = None + + if fields and isinstance(fields[0], int): + amount, fields = fields[0], fields[1:] + def adjust_inner(cls): for field in fields: - cls.__dict__[field].__sequence__ += amount # Move this to the back of the bus. + if field not in cls.__dict__: + # TODO: Copy the field definition. + raise TypeError("Can only override sequence on non-inherited attributes.") + + # Adjust the sequence to re-order the field. + if amount is None: + cls.__dict__[field].__sequence__ = ElementMeta.sequence + else: + cls.__dict__[field].__sequence__ += amount # Add the given amount. + + # Update the attribute collection. + cls.__attributes__ = OrderedDict( + (k, v) for k, v in \ + sorted(cls.__attributes__.items(), + key=lambda i: i[1].__sequence__) + ) return cls @@ -32,3 +58,7 @@ def __getattr__(self, name): def __getitem__(self, name): return load(name, self._namespace) + + +def utcnow(): + return datetime.utcnow().replace(tzinfo=utc) diff --git a/marrow/mongo/util/capped.py b/marrow/mongo/util/capped.py index 1c3b3def..c3994b86 100644 --- a/marrow/mongo/util/capped.py +++ b/marrow/mongo/util/capped.py @@ -27,13 +27,12 @@ def tail(collection, filter=None, projection=None, limit=0, timeout=None, aggreg with an empty or otherwise unimportant record before attempting to use this feature. """ - if __debug__: # Completely delete this section if Python is run with optimizations enabled. - if not collection.options().get('capped', False): - raise TypeError("Can only tail capped collections.") - - # Similarly, verify that the collection isn't empty. Empty is bad. (Busy loop.) - if not collection.count(): - raise ValueError("Cowardly refusing to tail an empty collection.") + if not collection.options().get('capped', False): + raise TypeError("Can only tail capped collections.") + + # Similarly, verify that the collection isn't empty. Empty is bad. (Busy loop.) + if not collection.count(): + raise ValueError("Cowardly refusing to tail an empty collection.") cursor = collection.find(filter, projection, limit=limit, cursor_type=CursorType.TAILABLE_AWAIT) diff --git a/setup.py b/setup.py index 25770561..fe725f66 100755 --- a/setup.py +++ b/setup.py @@ -24,13 +24,15 @@ py2 = sys.version_info < (3,) py26 = sys.version_info < (2, 7) py32 = sys.version_info > (3,) and sys.version_info < (3, 3) +pypy = hasattr(sys, 'pypy_version_info') tests_require = [ 'pytest', # test collector and extensible runner 'pytest-cov', # coverage reporting 'pytest-flakes', # syntax validation - 'pytest-capturelog', # log capture + 'pytest-catchlog', # log capture 'pytest-isort', # import ordering + 'misaka', 'pygments', # Markdown field support ] @@ -82,6 +84,7 @@ 'marrow.schema>=1.2.0,<2.0.0', # Declarative schema support. 'marrow.package>=1.1.0,<2.0.0', # Plugin discovery and loading. 'pymongo>=3.2', # We require modern API. + 'pathlib2; python_version < "3.4"', # Path manipulation utility. ], extras_require = dict( @@ -89,6 +92,7 @@ development = tests_require + ['pre-commit'], # Development-time dependencies. scripting = ['javascripthon<1.0'], # Allow map/reduce functions and "stored functions" to be Python. logger = ['tzlocal'], # Timezone support to store log times in UTC like a sane person. + markdown = ['misaka', 'pygments'], # Markdown text storage. ), tests_require = tests_require, @@ -98,7 +102,8 @@ entry_points = { # ### Marrow Mongo Lookups 'marrow.mongo.document': [ # Document classes registered by name. - 'Document = marrow.mongo.core:Document', + 'Document = marrow.mongo.core.document:Document', + 'GeoJSON = marrow.mongo.geo:GeoJSON', 'GeoJSONCoord = marrow.mongo.geo:GeoJSONCoord', 'Point = marrow.mongo.geo:Point', @@ -111,6 +116,7 @@ ], 'marrow.mongo.field': [ # Field classes registered by (optionaly namespaced) name. 'Field = marrow.mongo.core.field:Field', + 'String = marrow.mongo.core.field.base:String', 'Binary = marrow.mongo.core.field.base:Binary', 'ObjectId = marrow.mongo.core.field.base:ObjectId', @@ -119,16 +125,44 @@ 'TTL = marrow.mongo.core.field.base:TTL', 'Regex = marrow.mongo.core.field.base:Regex', 'Timestamp = marrow.mongo.core.field.base:Timestamp', + 'Array = marrow.mongo.core.field.complex:Array', 'Embed = marrow.mongo.core.field.complex:Embed', 'Reference = marrow.mongo.core.field.complex:Reference', 'PluginReference = marrow.mongo.core.field.complex:PluginReference', 'Alias = marrow.mongo.core.field.complex:Alias', + 'Number = marrow.mongo.core.field.number:Number', 'Double = marrow.mongo.core.field.number:Double', 'Integer = marrow.mongo.core.field.number:Integer', 'Long = marrow.mongo.core.field.number:Long', 'Decimal = marrow.mongo.core.field.number:Decimal[decimal]', + + 'Markdown = marrow.mongo.core.field.md:Markdown[markdown]', + 'Path = marrow.mongo.core.field.path:Path', + 'Translated = marrow.mongo.core.trait.localized:Translated', + ], + 'marrow.mongo.trait': [ # Document traits for use as mix-ins. + # Active Collection Traits + 'Collection = marrow.mongo.core.trait.collection:Collection', + 'Queryable = marrow.mongo.core.trait.queryable:Queryable', + + # Behavioural Traits + 'Derived = marrow.mongo.core.trait.derived:Derived', + 'Expires = marrow.mongo.core.trait.expires:Expires', + 'Identified = marrow.mongo.core.trait.identified:Identified', + 'Localized = marrow.mongo.core.trait.localized:Localized', + 'Published = marrow.mongo.core.trait.published:Published', + # 'Stateful = marrow.mongo.core.trait.stateful:Stateful', + + # Taxonomic Traits + #'Heirarchical = marrow.mongo.core.trait.heir:Heirarchical', + #'HChildren = marrow.mongo.core.trait.heir:HChildren', + #'HParent = marrow.mongo.core.trait.heir:HParent', + #'HAncestors = marrow.mongo.core.trait.heir:HAncestors', + #'HPath = marrow.mongo.core.trait.heir:HPath', + #'HNested = marrow.mongo.core.trait.heir:HNested', + #'Taxonomy = marrow.mongo.core.trait.heir:Taxonomy', ], # ### WebCore Extensions 'web.session': [ # Session Engine diff --git a/test/core/document/test_projection.py b/test/core/document/test_projection.py index 446096ea..70e4a192 100644 --- a/test/core/document/test_projection.py +++ b/test/core/document/test_projection.py @@ -1,18 +1,22 @@ # encoding: utf-8 from marrow.mongo import Document, Field +from marrow.mongo.trait import Collection -class NoProjection(Document): +class NoProjection(Collection): + id = None default = Field() -class OnlyProjected(Document): +class OnlyProjected(Collection): + id = None default = Field() always = Field(project=True) -class RejectOnly(Document): +class RejectOnly(Collection): + id = None default = Field() never = Field(project=False) diff --git a/test/core/document/test_serialization.py b/test/core/document/test_serialization.py index a9d72245..fae91447 100644 --- a/test/core/document/test_serialization.py +++ b/test/core/document/test_serialization.py @@ -2,6 +2,7 @@ from marrow.mongo import Document from marrow.mongo.field import Number, String +from marrow.mongo.trait import Derived, Identified class Sample(Document): @@ -9,11 +10,65 @@ class Sample(Document): number = Number() +class Other(Identified, Document): + field = String() + + +class StringPk(Document): + __pk__ = 'tag' + + tag = String('_id') + + +class NumberPk(Document): + __pk__ = 'index' + + index = Number() + + +class DynamicRepr(Document): + __type_store__ = True + + first = String(repr=False) + second = String(repr=lambda doc, f: False) + + def __repr__(self, *args, **kw): + kw['key'] = 27 + return super(DynamicRepr, self).__repr__('pos', *args, **kw) + + +class TestProgrammersRepresentation(object): + def test_basic_sample(self): + record = Sample("a", 1) + assert repr(record).replace("u'", "'") == "Sample(string='a', number=1)" + + def test_identified_sample(self): + record = Other('58a8e86f0aa7399e8d735310', "Hello world!") + assert repr(record).replace("u'", "'") == "Other(58a8e86f0aa7399e8d735310, field='Hello world!')" + + def test_string_identifier(self): + record = StringPk('test') + assert repr(record) == "StringPk(test)" + + def test_other_identifier(self): + record = NumberPk(27) + assert repr(record) == "NumberPk(27)" + + def test_dynamic_repr(self): + record = DynamicRepr("27", "42") + assert repr(record).replace("u'", "'") == "test_serialization:DynamicRepr(pos, key=27)" + + class TestMongoSerialization(object): def test_unnessicary_deserialization(self): record = Sample("a", 1) assert Sample.from_mongo(record) is record + def test_none_fallback(self): + # To support iterative use of from_mongo attempting to deserialize None should return None, not explode. + record = Sample.from_mongo(None) + assert record is None + def test_rest_passthrough(self): record = Sample("a", 1) assert record.as_rest is record @@ -24,7 +79,7 @@ def test_standard_use(self): assert record.number == 27 def test_explicit_class(object): - record = Sample.from_mongo({'_cls': 'Document', 'foo': 'bar'}) + record = Derived.from_mongo({'_cls': 'Document', 'foo': 'bar'}) assert record.__class__.__name__ == 'Document' assert record['foo'] == 'bar' diff --git a/test/core/document/test_validation.py b/test/core/document/test_validation.py index ad324631..d79a27a6 100644 --- a/test/core/document/test_validation.py +++ b/test/core/document/test_validation.py @@ -2,11 +2,11 @@ from __future__ import unicode_literals -from marrow.mongo import Document from marrow.mongo.field import Array, String +from marrow.mongo.trait import Collection -class StringDocument(Document): +class StringDocument(Collection): optional = String() nonoptional = String(required=True) choose = String(choices=["Hello", "World"]) @@ -21,8 +21,3 @@ def test_string_document(): 'nonoptional': {'$type': 'string'}, 'choose': {'$or': [{'$exists': 0}, {'$type': 'string', '$in': ['Hello', 'World']}]} } - - - -class ArrayDocument(Document): - values = Array(String()) diff --git a/test/core/test_index.py b/test/core/test_index.py index d31bc612..edc6fc87 100644 --- a/test/core/test_index.py +++ b/test/core/test_index.py @@ -22,13 +22,28 @@ def test_ordering(self): assert Sample._inverse.fields == [('field_name', -1)] def test_creation(self, coll): - Sample._field.create_index(coll) + Sample._field.create(coll) indexes = coll.index_information() assert '_field' in indexes + del indexes['_field']['v'] assert indexes['_field'] == { 'background': False, 'key': [('field_name', 1)], 'ns': 'test.collection', 'sparse': False, - 'v': 1, } + + def test_removal(self, coll): + Sample._field.create(coll) + indexes = coll.index_information() + assert '_field' in indexes + del indexes['_field']['v'] + assert indexes['_field'] == { + 'background': False, + 'key': [('field_name', 1)], + 'ns': 'test.collection', + 'sparse': False, + } + Sample._field.drop(coll) + indexes = coll.index_information() + assert '_field' not in indexes diff --git a/test/core/test_util.py b/test/core/test_util.py index 5b8542bb..abdf1d7a 100644 --- a/test/core/test_util.py +++ b/test/core/test_util.py @@ -1,3 +1,38 @@ # encoding: utf-8 -from marrow.mongo.util import * +import pytest + +from marrow.mongo import Document, Field +from marrow.mongo.util import adjust_attribute_sequence, utcnow + + +def test_utcnow(): + now = utcnow() + assert now.tzinfo + + +class TestSequenceAdjustment(object): + def test_default_punt(self): + @adjust_attribute_sequence('last') + class Inner(Document): + last = Field() + first = Field() + + assert Inner('first', 'last').first == 'first' + + def test_promotion(self): + @adjust_attribute_sequence(-2, 'first') + class Inner(Document): + last = Field() + first = Field() + + assert Inner('first', 'last').first == 'first' + + def test_explosion(self): + with pytest.raises(TypeError): + class Inner(Document): + field = Field() + + @adjust_attribute_sequence('field') + class Other(Inner): + pass diff --git a/test/field/common.py b/test/field/common.py new file mode 100644 index 00000000..279f84b2 --- /dev/null +++ b/test/field/common.py @@ -0,0 +1,19 @@ +# encoding: utf-8 + +from __future__ import unicode_literals + +import pytest + +from marrow.mongo import Document + + +class FieldExam(object): + __args__ = tuple() + __kwargs__ = dict() + + @pytest.fixture() + def Sample(self, request): + class Sample(Document): + field = self.__field__(*self.__args__, **self.__kwargs__) + + return Sample diff --git a/test/field/test_array.py b/test/field/test_array.py new file mode 100644 index 00000000..199a16f0 --- /dev/null +++ b/test/field/test_array.py @@ -0,0 +1,25 @@ +# encoding: utf-8 + +from __future__ import unicode_literals + +from common import FieldExam +from marrow.mongo import Document +from marrow.mongo.field import Array + + +class TestSingularArrayField(FieldExam): + __field__ = Array + __args__ = (Document, ) + __kwargs__ = {'assign': True} + + def test_native_cast(self, Sample): + inst = Sample.from_mongo({'field': [{'foo': 27, 'bar': 42}]}) + assert isinstance(inst.field[0], Document) + assert inst.field[0]['foo'] == 27 + assert inst.field[0]['bar'] == 42 + + def test_default_value(self, Sample): + inst = Sample() + inst.field.append(Document.from_mongo({'rng': 7})) + assert isinstance(inst.field[0], Document) + assert inst.field[0]['rng'] == 7 diff --git a/test/field/test_base.py b/test/field/test_base.py deleted file mode 100644 index a14cc411..00000000 --- a/test/field/test_base.py +++ /dev/null @@ -1,150 +0,0 @@ -# encoding: utf-8 - -from __future__ import unicode_literals - -from datetime import datetime, timedelta - -import pytest -from bson import ObjectId as oid -from bson.tz_util import utc - -from marrow.mongo import Document -from marrow.mongo.field import TTL, Binary, Boolean, Date, ObjectId, Regex, String, Timestamp -from marrow.schema.compat import unicode - - -class FieldExam(object): - @pytest.fixture() - def Sample(self, request): - class Sample(Document): - field = self.__field__() - - return Sample - - -class TestStringField(FieldExam): - __field__ = String - - -class TestBinaryField(FieldExam): - __field__ = Binary - - def test_conversion(self, Sample): - inst = Sample(b'abc') - assert isinstance(inst.__data__['field'], bytes) - assert inst.field == b'abc' - - -class TestObjectIdField(FieldExam): - __field__ = ObjectId - - def test_id_default(self): - class Sample(Document): - id = ObjectId('_id') - - assert isinstance(Sample().id, oid) - - def test_cast_string(self, Sample): - inst = Sample('5832223f927cc6c1a10609f7') - - assert isinstance(inst.__data__['field'], oid) - assert unicode(inst.field) == '5832223f927cc6c1a10609f7' - - def test_cast_oid(self, Sample): - v = oid() - inst = Sample(v) - - assert inst.__data__['field'] is v - - def test_cast_datetime(self, Sample): - v = datetime.utcnow().replace(microsecond=0, tzinfo=utc) - inst = Sample(v) - - assert isinstance(inst.__data__['field'], oid) - assert inst.field.generation_time == v - - def test_cast_timedelta(self, Sample): - v = -timedelta(days=7) - r = (datetime.utcnow() + v).replace(microsecond=0, tzinfo=utc) - inst = Sample(v) - - assert isinstance(inst.__data__['field'], oid) - assert inst.field.generation_time == r - - def test_cast_document(self, Sample): - v = {'_id': oid()} - inst = Sample(v) - assert inst.field == v['_id'] - - -class TestBooleanField(FieldExam): - __field__ = Boolean - - def test_cast_boolean(self, Sample): - inst = Sample(True) - assert inst.field is True - - inst = Sample(False) - assert inst.field is False - - def test_cast_strings(self, Sample): - inst = Sample("true") - assert inst.field is True - - inst = Sample("false") - assert inst.field is False - - def test_cast_none(self, Sample): - inst = Sample(None) - assert inst.field is None - - def test_cast_other(self, Sample): - inst = Sample(1) - assert inst.field is True - - def test_cast_failure(self, Sample): - with pytest.raises(ValueError): - Sample('hehehe') - - -class TestDateField(FieldExam): - __field__ = Date - - -class TestTTLField(FieldExam): - __field__ = TTL - - def test_cast_timedelta(self, Sample): - v = timedelta(days=7) - r = (datetime.utcnow() + v).replace(microsecond=0, tzinfo=utc) - inst = Sample(v) - - assert isinstance(inst.__data__['field'], datetime) - assert inst.field.replace(microsecond=0, tzinfo=utc) == r - - def test_cast_datetime(self, Sample): - v = datetime.utcnow().replace(microsecond=0, tzinfo=utc) - inst = Sample(v) - - assert isinstance(inst.__data__['field'], datetime) - assert inst.field is v - - def test_cast_integer(self, Sample): - v = timedelta(days=7) - r = (datetime.utcnow() + v).replace(microsecond=0, tzinfo=utc) - inst = Sample(7) - - assert isinstance(inst.__data__['field'], datetime) - assert inst.field.replace(microsecond=0, tzinfo=utc) == r - - def test_cast_failure(self, Sample): - with pytest.raises(ValueError): - Sample('xyzzy') - - -class RegexField(FieldExam): - __field__ = Regex - - -class TestTimestampField(FieldExam): - __field__ = Timestamp diff --git a/test/field/test_binary.py b/test/field/test_binary.py new file mode 100644 index 00000000..74a723e0 --- /dev/null +++ b/test/field/test_binary.py @@ -0,0 +1,15 @@ +# encoding: utf-8 + +from __future__ import unicode_literals + +from common import FieldExam +from marrow.mongo.field import Binary + + +class TestBinaryField(FieldExam): + __field__ = Binary + + def test_conversion(self, Sample): + inst = Sample(b'abc') + assert isinstance(inst.__data__['field'], bytes) + assert inst.field == b'abc' diff --git a/test/field/test_boolean.py b/test/field/test_boolean.py new file mode 100644 index 00000000..65cba43a --- /dev/null +++ b/test/field/test_boolean.py @@ -0,0 +1,38 @@ +# encoding: utf-8 + +from __future__ import unicode_literals + +import pytest + +from common import FieldExam +from marrow.mongo.field import Boolean + + +class TestBooleanField(FieldExam): + __field__ = Boolean + + def test_cast_boolean(self, Sample): + inst = Sample(True) + assert inst.field is True + + inst = Sample(False) + assert inst.field is False + + def test_cast_strings(self, Sample): + inst = Sample("true") + assert inst.field is True + + inst = Sample("false") + assert inst.field is False + + def test_cast_none(self, Sample): + inst = Sample(None) + assert inst.field is None + + def test_cast_other(self, Sample): + inst = Sample(1) + assert inst.field is True + + def test_cast_failure(self, Sample): + with pytest.raises(ValueError): + Sample('hehehe') diff --git a/test/field/test_complex.py b/test/field/test_complex.py deleted file mode 100644 index d2afc6e6..00000000 --- a/test/field/test_complex.py +++ /dev/null @@ -1,331 +0,0 @@ -# encoding: utf-8 - -from __future__ import unicode_literals - -from datetime import datetime, timedelta - -import pytest -from bson import ObjectId as oid -from bson import DBRef -from bson.tz_util import utc - -from marrow.mongo import Document -from marrow.mongo.core.field.complex import _HasKinds -from marrow.mongo.field import Array, Embed, PluginReference, Reference, String -from marrow.schema.compat import odict, unicode - - -class FieldExam(object): - __args__ = () - __kw__ = {} - - @pytest.fixture() - def Sample(self, request): - class Sample(Document): - field = self.__field__(*self.__args__, **self.__kw__) - - return Sample - - -class Concrete(Document): - __collection__ = 'collection' - - foo = String() - bar = String() - - -class TestHasKinds(object): - def test_singular(self): - inst = _HasKinds(kind=1) - assert list(inst.kinds) == [1] - - def test_iterable(self): - inst = _HasKinds(kind=(1, 2)) - assert list(inst.kinds) == [1, 2] - - def test_positional(self): - inst = _HasKinds(1, 2) - assert list(inst.kinds) == [1, 2] - - def test_reference(self): - inst = _HasKinds(kind='Document') - assert list(inst.kinds) == [Document] - - -class TestSingularArrayField(FieldExam): - __field__ = Array - __args__ = (Document, ) - - def test_native_cast(self, Sample): - inst = Sample.from_mongo({'field': [{'foo': 27, 'bar': 42}]}) - assert isinstance(inst.field[0], Document) - assert inst.field[0]['foo'] == 27 - assert inst.field[0]['bar'] == 42 - - def test_foreign_cast(self, Sample): - inst = Sample(field=[{}]) - assert isinstance(inst.field[0], Document) - - -class TestMultipleArrayField(FieldExam): - __field__ = Array - __args__ = (Document, Document) - - def test_native_cast(self, Sample): - inst = Sample.from_mongo({'field': [{'foo': 27, 'bar': 42}]}) - assert isinstance(inst.field[0], Document) - assert inst.field[0]['foo'] == 27 - assert inst.field[0]['bar'] == 42 - - def test_foreign_cast_fail(self, Sample): - with pytest.raises(ValueError): - Sample(field=[{}]) - - def test_foreign_reference(self, Sample): - inst = Sample(field=[Document()]) - assert inst.field[0]['_cls'] == 'marrow.mongo.core.document:Document' - - -class TestSingularEmbedField(FieldExam): - __field__ = Embed - __args__ = (Document, ) - - def test_native_cast(self, Sample): - inst = Sample.from_mongo({'field': {'foo': 27, 'bar': 42}}) - assert isinstance(inst.field, Document) - assert inst.field['foo'] == 27 - assert inst.field['bar'] == 42 - - def test_foreign_cast(self, Sample): - inst = Sample(field={}) - assert isinstance(inst.field, Document) - - -class TestMultipleEmbedField(FieldExam): - __field__ = Embed - __args__ = (Document, Document) - - def test_native_cast(self, Sample): - inst = Sample.from_mongo({'field': {'foo': 27, 'bar': 42}}) - assert isinstance(inst.field, Document) - assert inst.field['foo'] == 27 - assert inst.field['bar'] == 42 - - def test_foreign_cast_fail(self, Sample): - with pytest.raises(ValueError): - Sample(field={}) - - def test_foreign_reference(self, Sample): - inst = Sample(field=Document()) - assert inst.field['_cls'] == 'marrow.mongo.core.document:Document' - - -class TestReferenceField(FieldExam): - __field__ = Reference - __args__ = (Document, ) - - def test_foreign(self, Sample): - assert Sample.field._field.__foreign__ == 'objectId' - - def test_foreign_cast_document_fail(self, Sample): - inst = Sample() - doc = Document() - - with pytest.raises(ValueError): - inst.field = doc - - def test_foreign_cast_document(self, Sample): - inst = Sample() - doc = Document() - doc['_id'] = 27 - inst.field = doc - assert inst['field'] == 27 - - def test_foreign_cast_string_oid(self, Sample): - inst = Sample() - inst.field = '58329b3a927cc647e94153c9' - assert inst['field'] == oid('58329b3a927cc647e94153c9') - - def test_foreign_cast_string_not_oid(self, Sample): - inst = Sample() - inst.field = '58329b3a927cz647e94153c9' - assert inst['field'] == '58329b3a927cz647e94153c9' - - -class TestMultipleReferenceField(FieldExam): - __field__ = Reference - __args__ = (Document, Document) - - def test_foreign(self, Sample): - assert Sample.field._field.__foreign__ == 'objectId' - - def test_foreign_cast_document_fail(self, Sample): - with pytest.raises(ValueError): - Sample(Document()) - - def test_foreign_cast_document(self, Sample): - inst = Sample() - doc = Document() - doc['_id'] = 27 - inst.field = doc - assert inst['field'] == 27 - - def test_foreign_cast_string_oid(self, Sample): - inst = Sample() - with pytest.raises(ValueError): - inst.field = '58329b3a927cc647e94153c9' - - def test_foreign_cast_string_not_oid(self, Sample): - inst = Sample() - with pytest.raises(ValueError): - inst.field = '58329b3a927cz647e94153c9' - - -class TestConcreteReferenceField(FieldExam): - __field__ = Reference - __args__ = (Concrete, ) - __kw__ = dict(concrete=True) - - def test_foreign(self, Sample): - assert Sample.field._field.__foreign__ == 'dbPointer' - - def test_foreign_cast_document(self, Sample): - val = Concrete() - val['_id'] = oid('58329b3a927cc647e94153c9') - inst = Sample(val) - assert isinstance(inst.field, DBRef) - - def test_foreign_cast_document_fail(self, Sample): - with pytest.raises(ValueError): - Sample(Concrete()) - - def test_foreign_cast_reference(self, Sample): - inst = Sample('58329b3a927cc647e94153c9') - assert isinstance(inst.field, DBRef) - - -class TestCachingReferenceField(FieldExam): - __field__ = Reference - __args__ = (Concrete, ) - __kw__ = dict(cache=('foo', )) - - def test_foreign(self, Sample): - assert Sample.field._field.__foreign__ == 'object' - - def test_foreign_cast_document(self, Sample): - val = Concrete() - val['_id'] = oid('58329b3a927cc647e94153c9') - val.foo = 'foo' - val.bar = 'bar' - - inst = Sample(field=val) - - assert isinstance(inst.field, odict) - assert inst.field['_id'] == val['_id'] - assert inst.field['_cls'] == 'test_complex:Concrete' - assert inst.field['foo'] == val['foo'] - - def test_foreign_cast_dict(self, Sample): - val = {'_id': oid('58329b3a927cc647e94153c9'), 'foo': 'foo', 'bar': 'bar'} - - inst = Sample(field=val) - - assert isinstance(inst.field, odict) - assert inst.field['_id'] == val['_id'] - assert '_cls' not in inst.field - assert inst.field['foo'] == val['foo'] - - def test_foreign_cast_oid(self, Sample): - val = oid('58329b3a927cc647e94153c9') - - inst = Sample(field=val) - - assert isinstance(inst.field, odict) - assert inst.field['_id'] == val - assert '_cls' not in inst.field - assert 'foo' not in inst.field - - def test_foreign_cast_string_oid(self, Sample): - val = '58329b3a927cc647e94153c9' - - inst = Sample(field=val) - - assert isinstance(inst.field, odict) - assert inst.field['_id'] == oid(val) - assert '_cls' not in inst.field - assert 'foo' not in inst.field - - def test_foreign_cast_document_fail(self, Sample): - with pytest.raises(ValueError): - Sample(Concrete()) - - def test_foreign_cast_dict_fail(self, Sample): - with pytest.raises(ValueError): - Sample({}) - - def test_foreign_cast_string_oid_fail(self, Sample): - with pytest.raises(ValueError): - Sample('58329b3a927cc647e941x3c9') - - def test_foreign_cast_other_fail(self, Sample): - with pytest.raises(ValueError): - Sample(object()) - - def test_foreign_cast_numeric_fail(self): - class Sample(Document): - field = Reference(Concrete, cache=('foo.1', )) - - with pytest.raises(ValueError): - Sample(oid('58329b3a927cc647e94153c9')) - - def test_foreign_cast_nested(self, Sample): - class Ref(Document): - field = Reference(Document, cache=('field.field', )) - - val = {'_id': oid('58329b3a927cc647e94153c9'), 'field': {'field': 27}} - inst = Ref(val) - - assert inst.field['field']['field'] == 27 - - -class TestExplicitPluginReferenceField(FieldExam): - __field__ = PluginReference - - def test_native_cast(self, Sample): - inst = Sample.from_mongo({'field': 'marrow.mongo:Document'}) - assert inst.field is Document - - def test_foreign_cast(self, Sample): - inst = Sample(Document) - assert inst['field'] == 'marrow.mongo.core.document:Document' - - -class TestNamespacedPluginReferenceField(FieldExam): - __field__ = PluginReference - __args__ = ('marrow.mongo.document', ) - - def test_native_cast(self, Sample): - inst = Sample.from_mongo({'field': 'Document'}) - assert inst.field is Document - - def test_foreign_object(self, Sample): - inst = Sample(Document) - assert inst['field'] == 'Document' - assert inst.field is Document - - def test_foreign_string(self, Sample): - inst = Sample('Document') - assert inst['field'] == 'Document' - assert inst.field is Document - - def test_foreign_fail_bad_reference(self, Sample): - with pytest.raises(ValueError): - Sample('UnknownDocumentTypeForRealsXYZZY') - - def test_foreign_fail_explicit(self, Sample): - with pytest.raises(ValueError): - Sample('marrow.mongo:Field') - - def test_foreign_fail_object(self, Sample): - with pytest.raises(ValueError): - Sample(object) diff --git a/test/field/test_date.py b/test/field/test_date.py new file mode 100644 index 00000000..07cba475 --- /dev/null +++ b/test/field/test_date.py @@ -0,0 +1,10 @@ +# encoding: utf-8 + +from __future__ import unicode_literals + +from common import FieldExam +from marrow.mongo.field import Date + + +class TestDateField(FieldExam): + __field__ = Date diff --git a/test/field/test_decimal.py b/test/field/test_decimal.py new file mode 100644 index 00000000..ce22f6d3 --- /dev/null +++ b/test/field/test_decimal.py @@ -0,0 +1,32 @@ +# encoding: utf-8 + +from __future__ import unicode_literals + +from decimal import Decimal as dec +from bson.decimal128 import Decimal128 + +from common import FieldExam +from marrow.mongo.field import Decimal + + +class TestDecimalField(FieldExam): + __field__ = Decimal + + def test_decimal(self, Sample): + v = dec('3.141592') + result = Sample(v) + assert isinstance(result['field'], Decimal128) + assert result['field'] == Decimal128('3.141592') + assert result.field == v + + def test_decimal_cast_up(self, Sample): + result = Sample.from_mongo({'field': 27.4}) + v = result.field + assert isinstance(v, dec) + assert float(v) == 27.4 + + def test_decimal_from_number(self, Sample): + result = Sample(27) + assert isinstance(result['field'], Decimal128) + assert result['field'] == Decimal128('27') + assert int(result.field) == 27 diff --git a/test/field/test_double.py b/test/field/test_double.py new file mode 100644 index 00000000..2864ac4f --- /dev/null +++ b/test/field/test_double.py @@ -0,0 +1,21 @@ +# encoding: utf-8 + +from __future__ import unicode_literals + +import pytest + +from common import FieldExam +from marrow.mongo.field import Double + + +class TestDoubleField(FieldExam): + __field__ = Double + + def test_float(self, Sample): + result = Sample(27.2).field + assert isinstance(result, float) + assert result == 27.2 + + def test_failure(self, Sample): + with pytest.raises(TypeError): + Sample(object()) diff --git a/test/field/test_embed.py b/test/field/test_embed.py new file mode 100644 index 00000000..ec2950ee --- /dev/null +++ b/test/field/test_embed.py @@ -0,0 +1,64 @@ +# encoding: utf-8 + +from __future__ import unicode_literals + +import pytest + +from common import FieldExam +from marrow.mongo import Document +from marrow.mongo.field import Embed, String +from marrow.mongo.trait import Derived + + +class Concrete(Derived, Document): + __collection__ = 'collection' + + foo = String() + bar = String() + + +class TestSingularEmbedField(FieldExam): + __field__ = Embed + __args__ = (Concrete, ) + + def test_native_ingress(self, Sample): + inst = Sample.from_mongo({'field': Concrete(foo=27, bar=42)}) + assert isinstance(inst.field, Concrete) + + def test_native_ingress_cast(self, Sample): + inst = Sample.from_mongo({'field': {'foo': 27, 'bar': 42}}) + assert isinstance(inst.field, Concrete) + assert inst.field.foo == 27 + assert inst.field.bar == 42 + + def test_native_egress_cast(self, Sample): + inst = Sample(field=Concrete()) + assert isinstance(inst.field, Concrete) + + def test_foreign_assignment_cast(self, Sample): + inst = Sample() + inst.field = {} + assert isinstance(inst['field'], Concrete) + + def test_unsupported_native_cast(self, Sample): + inst = Sample.from_mongo({'field': Document()}) + + with pytest.raises(ValueError): + inst.field + + def test_unsupported_foreign_cast(self, Sample): + inst = Sample() + + with pytest.raises(ValueError): + inst.field = Document() + + +class TestEmbedByName(FieldExam): + __field__ = Embed + __args__ = ('Document', ) + + def test_native_ingress_cast(self, Sample): + inst = Sample.from_mongo({'field': {'foo': 27, 'bar': 42}}) + assert isinstance(inst.field, Document) + assert inst.field['foo'] == 27 + assert inst.field['bar'] == 42 diff --git a/test/field/test_core.py b/test/field/test_field.py similarity index 99% rename from test/field/test_core.py rename to test/field/test_field.py index 1b4e6761..c4e18f2a 100644 --- a/test/field/test_core.py +++ b/test/field/test_field.py @@ -2,6 +2,7 @@ import pytest +from common import FieldExam from marrow.mongo import Document, field from marrow.mongo.core.field import Field, FieldTransform @@ -116,7 +117,6 @@ class Mock(Document): assert inst.__data__ == {} - class TestFieldSecurity(object): def test_writeable_predicate_simple(self): f = Field(write=None) diff --git a/test/field/test_integer.py b/test/field/test_integer.py new file mode 100644 index 00000000..12a33a78 --- /dev/null +++ b/test/field/test_integer.py @@ -0,0 +1,21 @@ +# encoding: utf-8 + +from __future__ import unicode_literals + +import pytest + +from common import FieldExam +from marrow.mongo.field import Integer + + +class TestIntegerField(FieldExam): + __field__ = Integer + + def test_integer(self, Sample): + result = Sample(27).field + assert isinstance(result, int) + assert result == 27 + + def test_failure(self, Sample): + with pytest.raises(TypeError): + Sample(object()) diff --git a/test/field/test_long.py b/test/field/test_long.py new file mode 100644 index 00000000..91279d91 --- /dev/null +++ b/test/field/test_long.py @@ -0,0 +1,14 @@ +# encoding: utf-8 + +from __future__ import unicode_literals + +from common import FieldExam +from marrow.mongo.field import Long + + +class TestLongField(FieldExam): + __field__ = Long + + def test_biginteger(self, Sample): + result = Sample(272787482374844672646272463).field + assert result == 272787482374844672646272463 diff --git a/test/field/test_md.py b/test/field/test_md.py new file mode 100644 index 00000000..4c009227 --- /dev/null +++ b/test/field/test_md.py @@ -0,0 +1,36 @@ +# encoding: utf-8 + +from __future__ import unicode_literals + +from common import FieldExam +from marrow.mongo.field import Markdown + + +class TestMarkdownField(FieldExam): + __field__ = Markdown + + def test_plain_text(self, Sample): + inst = Sample("This is a test.") + assert inst.field == 'This is a test.' + + def test_formatted_text(self, Sample): + inst = Sample("This is a **test**.") + assert inst.field == 'This is a **test**.' + assert inst.field.__html__() == '

This is a test.

\n' + + def test_cast_protocol_magic(self, Sample): + class Inner(object): + def __markdown__(self): + return "Some Markdown text." + + inst = Sample(Inner()) + assert inst.field == 'Some Markdown text.' + + def test_cast_protocol_property(self, Sample): + class Inner(object): + @property + def as_markdown(self): + return "Other Markdown text." + + inst = Sample(Inner()) + assert inst.field == 'Other Markdown text.' diff --git a/test/field/test_number.py b/test/field/test_number.py index f69b901d..584ea880 100644 --- a/test/field/test_number.py +++ b/test/field/test_number.py @@ -2,26 +2,10 @@ from __future__ import unicode_literals -from datetime import datetime, timedelta -from decimal import Decimal as dec - import pytest -from bson import ObjectId as oid -from bson.decimal128 import Decimal128 -from bson.tz_util import utc - -from marrow.mongo import Document -from marrow.mongo.field import Decimal, Double, Integer, Long, Number -from marrow.schema.compat import unicode - -class FieldExam(object): - @pytest.fixture() - def Sample(self, request): - class Sample(Document): - field = self.__field__() - - return Sample +from common import FieldExam +from marrow.mongo.field import Number class TestNumberField(FieldExam): @@ -45,60 +29,3 @@ def test_float(self, Sample): def test_failure(self, Sample): with pytest.raises(TypeError): Sample(object()) - - -class TestDoubleField(FieldExam): - __field__ = Double - - def test_float(self, Sample): - result = Sample(27.2).field - assert isinstance(result, float) - assert result == 27.2 - - def test_failure(self, Sample): - with pytest.raises(TypeError): - Sample(object()) - - -class TestIntegerField(FieldExam): - __field__ = Integer - - def test_integer(self, Sample): - result = Sample(27).field - assert isinstance(result, int) - assert result == 27 - - def test_failure(self, Sample): - with pytest.raises(TypeError): - Sample(object()) - - -class TestLongField(FieldExam): - __field__ = Long - - def test_biginteger(self, Sample): - result = Sample(272787482374844672646272463).field - assert result == 272787482374844672646272463 - - -class TestDecimalField(FieldExam): - __field__ = Decimal - - def test_decimal(self, Sample): - v = dec('3.141592') - result = Sample(v) - assert isinstance(result['field'], Decimal128) - assert result['field'] == Decimal128('3.141592') - assert result.field == v - - def test_decimal_cast_up(self, Sample): - result = Sample.from_mongo({'field': 27.4}) - v = result.field - assert isinstance(v, dec) - assert float(v) == 27.4 - - def test_decimal_from_number(self, Sample): - result = Sample(27) - assert isinstance(result['field'], Decimal128) - assert result['field'] == Decimal128('27') - assert int(result.field) == 27 diff --git a/test/field/test_objectid.py b/test/field/test_objectid.py new file mode 100644 index 00000000..1d212860 --- /dev/null +++ b/test/field/test_objectid.py @@ -0,0 +1,55 @@ +# encoding: utf-8 + +from __future__ import unicode_literals + +from datetime import timedelta + +from bson import ObjectId as oid + +from common import FieldExam +from marrow.mongo import Document +from marrow.mongo.field import ObjectId +from marrow.mongo.util import utcnow +from marrow.schema.compat import unicode + + +class TestObjectIdField(FieldExam): + __field__ = ObjectId + + def test_id_default(self): + class Sample(Document): + id = ObjectId('_id') + + assert isinstance(Sample().id, oid) + + def test_cast_string(self, Sample): + inst = Sample('5832223f927cc6c1a10609f7') + + assert isinstance(inst.__data__['field'], oid) + assert unicode(inst.field) == '5832223f927cc6c1a10609f7' + + def test_cast_oid(self, Sample): + v = oid() + inst = Sample(v) + + assert inst.__data__['field'] is v + + def test_cast_datetime(self, Sample): + v = utcnow().replace(microsecond=0) + inst = Sample(v) + + assert isinstance(inst.__data__['field'], oid) + assert inst.field.generation_time == v + + def test_cast_timedelta(self, Sample): + v = -timedelta(days=7) + r = (utcnow() + v).replace(microsecond=0) + inst = Sample(v) + + assert isinstance(inst.__data__['field'], oid) + assert inst.field.generation_time == r + + def test_cast_document(self, Sample): + v = {'_id': oid()} + inst = Sample(v) + assert inst.field == v['_id'] diff --git a/test/field/test_path.py b/test/field/test_path.py new file mode 100644 index 00000000..aa75419b --- /dev/null +++ b/test/field/test_path.py @@ -0,0 +1,23 @@ +# encoding: utf-8 + +from __future__ import unicode_literals + +from common import FieldExam +from marrow.mongo.core.field.path import _Path +from marrow.mongo.field import Path +from marrow.schema.compat import unicode + + +class TestPathField(FieldExam): + __field__ = Path + + def test_native_conversion(self, Sample): + inst = Sample.from_mongo({'field': '/foo'}) + value = inst.field + assert isinstance(value, _Path) + assert unicode(value) == '/foo' + + def test_foreign_conversion(self, Sample): + inst = Sample(_Path('/bar')) + assert isinstance(inst['field'], unicode) + assert inst['field'] == '/bar' diff --git a/test/field/test_plugin_explicit.py b/test/field/test_plugin_explicit.py new file mode 100644 index 00000000..37531cf4 --- /dev/null +++ b/test/field/test_plugin_explicit.py @@ -0,0 +1,19 @@ +# encoding: utf-8 + +from __future__ import unicode_literals + +from common import FieldExam +from marrow.mongo import Document +from marrow.mongo.field import PluginReference + + +class TestExplicitPluginReferenceField(FieldExam): + __field__ = PluginReference + + def test_native_cast(self, Sample): + inst = Sample.from_mongo({'field': 'marrow.mongo:Document'}) + assert inst.field is Document + + def test_foreign_cast(self, Sample): + inst = Sample(Document) + assert inst['field'] == 'marrow.mongo.core.document:Document' diff --git a/test/field/test_plugin_namespace.py b/test/field/test_plugin_namespace.py new file mode 100644 index 00000000..04a88495 --- /dev/null +++ b/test/field/test_plugin_namespace.py @@ -0,0 +1,40 @@ +# encoding: utf-8 + +from __future__ import unicode_literals + +import pytest + +from common import FieldExam +from marrow.mongo import Document +from marrow.mongo.field import PluginReference + + +class TestNamespacedPluginReferenceField(FieldExam): + __field__ = PluginReference + __args__ = ('marrow.mongo.document', ) + + def test_native_cast(self, Sample): + inst = Sample.from_mongo({'field': 'Document'}) + assert inst.field is Document + + def test_foreign_object(self, Sample): + inst = Sample(Document) + assert inst['field'] == 'Document' + assert inst.field is Document + + def test_foreign_string(self, Sample): + inst = Sample('Document') + assert inst['field'] == 'Document' + assert inst.field is Document + + def test_foreign_fail_bad_reference(self, Sample): + with pytest.raises(ValueError): + Sample('UnknownDocumentTypeForRealsXYZZY') + + def test_foreign_fail_explicit(self, Sample): + with pytest.raises(ValueError): + Sample('marrow.mongo:Field') + + def test_foreign_fail_object(self, Sample): + with pytest.raises(ValueError): + Sample(object) diff --git a/test/field/test_reference.py b/test/field/test_reference.py new file mode 100644 index 00000000..65e48a7a --- /dev/null +++ b/test/field/test_reference.py @@ -0,0 +1,43 @@ +# encoding: utf-8 + +from __future__ import unicode_literals + +import pytest + +from common import FieldExam +from marrow.mongo import Document +from marrow.mongo.field import Reference, String +from marrow.mongo.trait import Derived + + +class Concrete(Derived, Document): + __collection__ = 'collection' + + foo = String() + bar = String() + + +class TestReferenceField(FieldExam): + __field__ = Reference + __args__ = (Document, ) + + def test_foreign(self, Sample): + assert Sample.field._field.__foreign__ == 'objectId' + + def test_foreign_cast_document_fail(self, Sample): + inst = Sample() + doc = Document() + + with pytest.raises(ValueError): + inst.field = doc + + def test_foreign_cast_document(self, Sample): + inst = Sample() + doc = Document() + doc['_id'] = 27 + inst.field = doc + assert inst['field'] == 27 + + def test_oid_failure(self, Sample): + inst = Sample(field='z' * 24) + assert inst['field'] == 'z' * 24 diff --git a/test/field/test_reference_cached.py b/test/field/test_reference_cached.py new file mode 100644 index 00000000..6e7b931a --- /dev/null +++ b/test/field/test_reference_cached.py @@ -0,0 +1,103 @@ +# encoding: utf-8 + +from __future__ import unicode_literals + +import pytest +from bson import ObjectId as oid + +from common import FieldExam +from marrow.mongo import Document +from marrow.mongo.field import Reference, String +from marrow.mongo.trait import Derived +from marrow.schema.compat import odict + + +class Concrete(Derived, Document): + __collection__ = 'collection' + + foo = String() + bar = String() + + +class TestCachingReferenceField(FieldExam): + __field__ = Reference + __args__ = (Concrete,) + __kwargs__ = dict(cache=('foo',)) + + def test_foreign(self, Sample): + assert Sample.field._field.__foreign__ == 'object' + + def test_foreign_cast_document(self, Sample): + val = Concrete() + val['_id'] = oid('58329b3a927cc647e94153c9') + val.foo = 'foo' + val.bar = 'bar' + + inst = Sample(field=val) + + assert isinstance(inst.field, odict) + assert inst.field['_id'] == val['_id'] + assert inst.field['_cls'] == 'test_reference_cached:Concrete' + assert inst.field['foo'] == val['foo'] + + def test_foreign_cast_dict(self, Sample): + val = {'_id': oid('58329b3a927cc647e94153c9'), 'foo': 'foo', 'bar': 'bar'} + + inst = Sample(field=val) + + assert isinstance(inst.field, odict) + assert inst.field['_id'] == val['_id'] + assert '_cls' not in inst.field + assert inst.field['foo'] == val['foo'] + + def test_foreign_cast_oid(self, Sample): + val = oid('58329b3a927cc647e94153c9') + + inst = Sample(field=val) + + assert isinstance(inst.field, odict) + assert inst.field['_id'] == val + assert '_cls' not in inst.field + assert 'foo' not in inst.field + + def test_foreign_cast_string_oid(self, Sample): + val = '58329b3a927cc647e94153c9' + + inst = Sample(field=val) + + assert isinstance(inst.field, odict) + assert inst.field['_id'] == oid(val) + assert '_cls' not in inst.field + assert 'foo' not in inst.field + + def test_foreign_cast_document_fail(self, Sample): + with pytest.raises(ValueError): + Sample(Concrete()) + + def test_foreign_cast_dict_fail(self, Sample): + with pytest.raises(ValueError): + Sample({}) + + def test_foreign_cast_string_oid_fail(self, Sample): + with pytest.raises(ValueError): + Sample('58329b3a927cc647e941x3c9') + + def test_foreign_cast_other_fail(self, Sample): + with pytest.raises(ValueError): + Sample(object()) + + def test_foreign_cast_numeric_fail(self): + class Sample(Document): + field = Reference(Concrete, cache=('foo.1', )) + + with pytest.raises(ValueError): + Sample(oid('58329b3a927cc647e94153c9')) + + def test_foreign_cast_nested(self, Sample): + class Ref(Document): + field = Reference(Document, cache=('field.field', )) + + val = {'_id': oid('58329b3a927cc647e94153c9'), 'field': {'field': 27}} + inst = Ref(val) + + assert inst.field['field']['field'] == 27 diff --git a/test/field/test_reference_concrete.py b/test/field/test_reference_concrete.py new file mode 100644 index 00000000..0dae0d12 --- /dev/null +++ b/test/field/test_reference_concrete.py @@ -0,0 +1,49 @@ +# encoding: utf-8 + +from __future__ import unicode_literals + +import pytest +from bson import ObjectId as oid +from bson import DBRef + +from common import FieldExam +from marrow.mongo import Document +from marrow.mongo.field import Reference, String + + +class Concrete(Document): + __collection__ = 'collection' + + foo = String() + bar = String() + + +class TestConcreteReferenceField(FieldExam): + __field__ = Reference + __args__ = (Concrete, ) + __kwargs__ = {'concrete': True} + + def test_foreign(self, Sample): + assert Sample.field._field.__foreign__ == 'dbPointer' + + def test_foreign_cast_document(self, Sample): + val = Concrete() + val['_id'] = oid('58329b3a927cc647e94153c9') + inst = Sample(val) + assert isinstance(inst.field, DBRef) + + def test_foreign_cast_document_fail(self, Sample): + with pytest.raises(ValueError): + Sample(Concrete()) + + def test_foreign_cast_reference(self, Sample): + inst = Sample('58329b3a927cc647e94153c9') + assert isinstance(inst.field, DBRef) + +class TestConcreteAbstractReference(FieldExam): + __field__ = Reference + __kwargs__ = {'concrete': True} + + def test_unknown_abstract(self, Sample): + with pytest.raises(ValueError): + Sample('xyzzy') diff --git a/test/field/test_regex.py b/test/field/test_regex.py new file mode 100644 index 00000000..ef00fe3d --- /dev/null +++ b/test/field/test_regex.py @@ -0,0 +1,10 @@ +# encoding: utf-8 + +from __future__ import unicode_literals + +from common import FieldExam +from marrow.mongo.field import Regex + + +class RegexField(FieldExam): + __field__ = Regex diff --git a/test/field/test_string.py b/test/field/test_string.py new file mode 100644 index 00000000..b35a4430 --- /dev/null +++ b/test/field/test_string.py @@ -0,0 +1,40 @@ +# encoding: utf-8 + +from __future__ import unicode_literals + +from common import FieldExam +from marrow.mongo.field import String + + +class TestStringField(FieldExam): + __field__ = String + + def test_strip_basic(self, Sample): + Sample.field.strip = True + + inst = Sample('\tTest\n ') + assert inst.field == 'Test' + + def test_strip_specific(self, Sample): + Sample.field.strip = '* ' + + inst = Sample('** Test *') + assert inst.field == 'Test' + + def test_case_lower(self, Sample): + Sample.field.case = 'lower' + + inst = Sample('Test') + assert inst.field == 'test' + + def test_case_upper(self, Sample): + Sample.field.case = 'upper' + + inst = Sample('Test') + assert inst.field == 'TEST' + + def test_case_title(self, Sample): + Sample.field.case = 'title' + + inst = Sample('Test words') + assert inst.field == 'Test Words' diff --git a/test/field/test_timestamp.py b/test/field/test_timestamp.py new file mode 100644 index 00000000..2bbb097c --- /dev/null +++ b/test/field/test_timestamp.py @@ -0,0 +1,10 @@ +# encoding: utf-8 + +from __future__ import unicode_literals + +from common import FieldExam +from marrow.mongo.field import Timestamp + + +class TestTimestampField(FieldExam): + __field__ = Timestamp diff --git a/test/field/test_ttl.py b/test/field/test_ttl.py new file mode 100644 index 00000000..cea9dc75 --- /dev/null +++ b/test/field/test_ttl.py @@ -0,0 +1,42 @@ +# encoding: utf-8 + +from __future__ import unicode_literals + +from datetime import datetime, timedelta + +import pytest + +from common import FieldExam +from marrow.mongo.field import TTL +from marrow.mongo.util import utcnow + + +class TestTTLField(FieldExam): + __field__ = TTL + + def test_cast_timedelta(self, Sample): + v = timedelta(days=7) + r = (utcnow() + v).replace(microsecond=0) + inst = Sample(v) + + assert isinstance(inst.__data__['field'], datetime) + assert inst.field.replace(microsecond=0) == r + + def test_cast_datetime(self, Sample): + v = utcnow() + inst = Sample(v) + + assert isinstance(inst.__data__['field'], datetime) + assert inst.field is v + + def test_cast_integer(self, Sample): + v = timedelta(days=7) + r = (utcnow() + v).replace(microsecond=0) + inst = Sample(7) + + assert isinstance(inst.__data__['field'], datetime) + assert inst.field.replace(microsecond=0) == r + + def test_cast_failure(self, Sample): + with pytest.raises(ValueError): + Sample('xyzzy') diff --git a/test/param/test_project.py b/test/param/test_project.py index db0a7394..cb6406e1 100644 --- a/test/param/test_project.py +++ b/test/param/test_project.py @@ -4,14 +4,15 @@ import pytest -from marrow.mongo import Document, Field, P +from marrow.mongo import Field, P from marrow.mongo.field import Array, Number +from marrow.mongo.trait import Collection class TestParametricProjectionConstructor(object): @pytest.fixture() def D(self): - class Sample(Document): + class Sample(Collection): field = Field(project=True) number = Number('other') diff --git a/test/param/test_update.py b/test/param/test_update.py index 55d0b9eb..9f5cf269 100644 --- a/test/param/test_update.py +++ b/test/param/test_update.py @@ -47,3 +47,23 @@ def test_timestamp(self, D): q = U(D, now__field='ts') assert isinstance(q, Update) assert q.operations == {'$currentDate': {'field': {'$type': 'timestamp'}}} + + def test_push_each(self, D): + q = U(D, push_each__array=[1, 2, 3]) + + assert q.operations == {'$push': {'array': {'$each': ["1", "2", "3"]}}} + + def test_complex_push(self, D): + q = U(D, + push_each__array = [1, 2, 3], + push_sort__array = -1, + push_slice__array = 2, + push_position__array = 1, + ) + + assert q.operations == {'$push': {'array': { + '$each': ["1", "2", "3"], + '$sort': -1, + '$slice': 2, + '$position': 1 + }}} diff --git a/test/query/test_ops.py b/test/query/test_ops.py index 01d82ac6..c187e003 100644 --- a/test/query/test_ops.py +++ b/test/query/test_ops.py @@ -165,3 +165,11 @@ def test_operations_or_clean_merge(self): comb = comb | Filter({'bar': 'baz'}) assert comb.as_query == {'$or': [{'roll': 27}, {'foo': 42}, {'bar': 'baz'}]} + + def test_operations_hard_and(self): + comb = Filter({'$and': [{'a': 1}, {'b': 2}]}) & Filter({'$and': [{'c': 3}]}) + assert comb.as_query == {'$and': [{'a': 1}, {'b': 2}, {'c': 3}]} + + def test_operations_soft_and(self): + comb = Filter({'$and': [{'a': 1}, {'b': 2}]}) & Filter({'c': 3}) + assert comb.as_query == {'$and': [{'a': 1}, {'b': 2}], 'c': 3} diff --git a/test/query/test_q.py b/test/query/test_q.py index c6642ca9..b550c9dd 100644 --- a/test/query/test_q.py +++ b/test/query/test_q.py @@ -63,7 +63,6 @@ class TestQueryable(object): # TODO: Properly use pytest fixtures for this... def test_attribute_access(self): assert Sample.number.default == 27 assert Sample.array.default == 42 - assert Sample.embed.name.__name__ == 'name' with pytest.raises(AttributeError): Sample.number.asdfasdf @@ -171,6 +170,9 @@ def test_twin_merge(self, S): assert z._field == a._field + b._field + def test_combining_setattr(self, S): + with pytest.raises(AttributeError): + (S.foo & S.bar).xyzzy = 27 class TestQueryableFieldCombinations(object): diff --git a/test/core/document/test_binding.py b/test/trait/test_collection.py similarity index 61% rename from test/core/document/test_binding.py rename to test/trait/test_collection.py index 8460b98f..a4d5e682 100644 --- a/test/core/document/test_binding.py +++ b/test/trait/test_collection.py @@ -5,7 +5,8 @@ import pytest from pymongo.errors import WriteError -from marrow.mongo import Document, Field, Index +from marrow.mongo import Field, Index +from marrow.mongo.trait import Collection @pytest.fixture @@ -20,10 +21,11 @@ def coll(request, db): @pytest.fixture def Sample(request): - class Sample(Document): + class Sample(Collection): __collection__ = 'collection' __engine__ = {'mmapv1': {}} + id = None # Remove the default identifier. field = Field() other = Field() @@ -34,15 +36,29 @@ class Sample(Document): class TestDocumentBinding(object): def test_bind_fail(self, Sample): - with pytest.raises(ValueError): + with pytest.raises(TypeError): Sample.bind() def test_bind_specific_collection(self, coll, Sample): assert not Sample.__bound__ - Sample.bind(collection=coll) + Sample.bind(coll) + + assert Sample.__bound__ + + def test_bind_specific_collection_twice(self, coll, Sample): + assert not Sample.__bound__ + + Sample.bind(coll) assert Sample.__bound__ + + first = Sample.__bound__ + Sample.bind(coll) + + assert Sample.__bound__ is first + + assert Sample.get_collection() is first def test_bind_database(self, db, Sample): assert not Sample.__bound__ @@ -52,14 +68,27 @@ def test_bind_database(self, db, Sample): assert Sample.__bound__ def test_create_collection(self, db, Sample): - assert Sample.create_collection(db, indexes=False).name == 'collection' + assert Sample.create_collection(db, drop=True, indexes=False).name == 'collection' + + def test_create_bound_collection(self, db, Sample): + assert Sample.bind(db).create_collection(drop=True, indexes=False).name == 'collection' + + def test_create_collection_failure(self, Sample): + with pytest.raises(AssertionError): + Sample.create_collection() + + with pytest.raises(TypeError): + Sample.create_collection("Hoi.") def test_create_collection_collection(self, db, Sample): - assert Sample.create_collection(db.foo, True, indexes=False).name == 'foo' + assert Sample.create_collection(db.foo, True).name == 'foo' def test_get_collection_failure(self, Sample): - with pytest.raises(TypeError): + with pytest.raises(AssertionError): Sample.get_collection(None) + + with pytest.raises(TypeError): + Sample.get_collection("Hoi.") def test_validation(self, db, Sample): if tuple((int(i) for i in db.client.server_info()['version'].split('.')[:3])) < (3, 2): @@ -80,10 +109,10 @@ def test_index_construction(self, db, Sample): indexes = c.index_information() assert '_field' in indexes + del indexes['_field']['v'] assert indexes['_field'] == { 'background': False, 'key': [('field', 1)], 'ns': 'test.collection', - 'sparse': False, - 'v': 1, + 'sparse': False } diff --git a/test/trait/test_derived.py b/test/trait/test_derived.py new file mode 100644 index 00000000..8bf82b06 --- /dev/null +++ b/test/trait/test_derived.py @@ -0,0 +1,27 @@ +# encoding: utf-8 + +from __future__ import unicode_literals + +import pytest + +from marrow.mongo.trait import Derived + + +class TestDerived(object): + class Sample(Derived): + pass + + def test_reference(self): + inst = self.Sample() + assert inst['_cls'] == 'test_derived:TestDerived.Sample' + + def test_repr(self): + inst = self.Sample() + assert repr(inst).startswith('test_derived:TestDerived.Sample') + + def test_load(self): + inst = Derived.from_mongo({'_cls': 'test_derived:TestDerived.Sample'}) + assert isinstance(inst, self.Sample) + + def test_index(self): + assert '_cls' in self.Sample.__indexes__ diff --git a/test/trait/test_expires.py b/test/trait/test_expires.py new file mode 100644 index 00000000..f20e45f1 --- /dev/null +++ b/test/trait/test_expires.py @@ -0,0 +1,49 @@ +# encoding: utf-8 + +from __future__ import unicode_literals + +from datetime import timedelta + +import pytest + +from marrow.mongo.trait import Expires +from marrow.mongo.util import utcnow + + +class TestExpires(object): + class Sample(Expires): + pass + + def test_index_presence(self): + assert '_expires' in self.Sample.__indexes__ + + def test_empty_repr(self): + inst = self.Sample() + assert repr(inst) == 'Sample()' + assert inst.is_expired is None + + def test_integer_assignment(self): + now = utcnow() + inst = self.Sample(0) + assert inst.expires - now < timedelta(seconds=1) + assert inst.is_expired + + def test_timedelta_assignment(self): + now = utcnow() + inst = self.Sample(timedelta(days=1)) + assert timedelta(hours=23) < (inst.expires - now) < timedelta(hours=25) + + def test_explicit_date(self): + then = utcnow() - timedelta(days=1) + inst = self.Sample(then) + assert inst.expires == then + assert inst.is_expired + + def test_deserialize_expired(self): + now = utcnow() + inst = self.Sample.from_mongo({'expires': now}) + assert inst is None + + inst = self.Sample.from_mongo({'expires': now + timedelta(hours=1)}) + assert isinstance(inst, self.Sample) + assert not inst.is_expired diff --git a/test/trait/test_identified.py b/test/trait/test_identified.py new file mode 100644 index 00000000..27938405 --- /dev/null +++ b/test/trait/test_identified.py @@ -0,0 +1,37 @@ +# encoding: utf-8 + +from __future__ import unicode_literals + +import pytest +from bson import ObjectId + +from marrow.mongo import Field, Index +from marrow.mongo.trait import Identified + + +@pytest.fixture +def Sample(request): + class Sample(Identified): + pass + + return Sample + + +class TestIdentifiedTrait(object): + def test_document_comparison(self, Sample): + id = ObjectId() + a = Sample(id) + b = Sample(id) + assert a == b + assert not (a != b) + b.id = None + assert not (a == b) + assert a != b + + def test_identifier_comparison(self, Sample): + id = ObjectId() + a = Sample(id) + assert a == id + assert not (a != id) + assert not (a == "not the ID") + assert a != "not the ID" diff --git a/test/trait/test_localized.py b/test/trait/test_localized.py new file mode 100644 index 00000000..17ab0cf1 --- /dev/null +++ b/test/trait/test_localized.py @@ -0,0 +1,57 @@ +# encoding: utf-8 + +from __future__ import unicode_literals + +import pytest + +from marrow.mongo.field import String, Translated +from marrow.mongo.trait import Localized + + +class TestTranslated(object): + class Sample(Localized): + class Locale(Localized.Locale): + word = String() + + word = Translated('word') + + def test_construction(self): + inst = self.Sample.from_mongo({'locale': [ + {'language': 'en', 'word': 'hello'}, + {'language': 'fr', 'word': 'bonjour'} + ]}) + + assert inst.word == {'en': 'hello', 'fr': 'bonjour'} + + def test_assignment(self): + inst = self.Sample() + + with pytest.raises(TypeError): + inst.word = None + + def test_query_translated(self): + q = self.Sample.word == 'bonjour' + assert q == {'locale.word': 'bonjour'} + + +class TestLocalized(object): + class Sample(Localized): + class Locale(Localized.Locale): + word = String() + + def test_repr(self): + inst = self.Sample.from_mongo({'locale': [ + {'language': 'en', 'word': 'hello'}, + {'language': 'fr', 'word': 'bonjour'} + ]}) + + assert repr(inst) == "Sample({en, fr})" + + def test_empty_repr(self): + inst = self.Sample() + + assert repr(inst) == "Sample()" + + def test_query(self): + q = self.Sample.locale.word == 'bonjour' + assert q == {'locale.word': 'bonjour'} diff --git a/test/trait/test_published.py b/test/trait/test_published.py new file mode 100644 index 00000000..562d4985 --- /dev/null +++ b/test/trait/test_published.py @@ -0,0 +1,72 @@ +# encoding: utf-8 + +from __future__ import unicode_literals + +from datetime import timedelta + +import pytest + +from marrow.mongo.trait import Published +from marrow.mongo.util import utcnow + + +class TestPublished(object): + class Sample(Published): + pass + + def test_creation_modification_init(self): + now = utcnow() + inst = self.Sample() + + assert abs(inst.created - now) < timedelta(seconds=1) + assert inst.modified is None + + def test_bare_public(self): + inst = self.Sample() + + assert inst.is_published + + def test_future_public(self): + inst = self.Sample(published=utcnow() + timedelta(days=1)) + + assert not inst.is_published + + def test_past_retraction(self): + inst = self.Sample(retracted=utcnow() - timedelta(days=1)) + + assert not inst.is_published + + def test_within_publication_period(self): + inst = self.Sample( + published = utcnow() - timedelta(days=1), + retracted = utcnow() + timedelta(days=1) + ) + + assert inst.is_published + + def test_query_exact(self): + now = utcnow() + + query = self.Sample.only_published(now) + + assert len(query) == 1 + assert '$and' in query + assert len(query['$and']) == 2 + assert '$or' in query['$and'][0] + assert '$or' in query['$and'][1] + assert query['$and'][1]['$or'][2]['published']['$lte'] == query['$and'][0]['$or'][2]['retracted']['$gt'] == now + + def test_query_delta(self): + when = utcnow() + timedelta(days=1) + + query = self.Sample.only_published(timedelta(days=1)) + + assert len(query) == 1 + assert '$and' in query + assert len(query['$and']) == 2 + assert '$or' in query['$and'][0] + assert '$or' in query['$and'][1] + assert query['$and'][1]['$or'][2]['published']['$lte'] == query['$and'][0]['$or'][2]['retracted']['$gt'] + + v = query['$and'][1]['$or'][2]['published']['$lte'] + assert abs(when - v) < timedelta(minutes=1) diff --git a/test/trait/test_queryable.py b/test/trait/test_queryable.py new file mode 100644 index 00000000..a9c6b477 --- /dev/null +++ b/test/trait/test_queryable.py @@ -0,0 +1,94 @@ +# encoding: utf-8 + +from __future__ import unicode_literals + +import pytest +from bson import ObjectId +from pymongo.errors import WriteError + +from marrow.mongo import Field, Index, U +from marrow.mongo.field import Integer, String +from marrow.mongo.trait import Queryable + + +@pytest.fixture +def db(request, connection): + return connection.test + + +@pytest.fixture +def Sample(request, db): + class Sample(Queryable): + __collection__ = 'queryable_collection' + + string = String() + integer = Integer() + + _field = Index('integer', background=False) + + Sample.bind(db).create_collection(drop=True) + + # Don't trigger code coverage just to insert test data... + Sample.__bound__.insert_many([ + {'string': 'foo', 'integer': 7}, + {'string': 'bar', 'integer': 27}, + {'string': 'baz', 'integer': 42}, + {'_id': ObjectId('59129d460aa7397ce3f9643e'), 'string': 'pre', 'integer': None}, + ]) + + return Sample + + +class TestQueryableTrait(object): + def test_find(self, Sample): + rs = Sample.find() + assert rs.count() == 4 + assert [i['integer'] for i in rs] == [7, 27, 42, None] + + def test_find_first(self, Sample): + doc = Sample.find_one() + assert isinstance(doc, Sample) + assert doc.string == 'foo' + assert doc.integer == 7 + + def test_find_one(self, Sample): + doc = Sample.find_one(Sample.integer == 27) + assert doc.string == 'bar' + + def test_find_one_short(self, Sample): + doc = Sample.find_one('59129d460aa7397ce3f9643e') + assert doc.string == 'pre' + assert doc.integer is None + + def test_find_in_sequence(self, db, Sample): + if tuple((int(i) for i in db.client.server_info()['version'].split('.')[:3])) < (3, 4): + pytest.xfail("Test expected to fail on MongoDB versions prior to 3.4.") + + results = Sample.find_in_sequence('integer', [42, 27]) + results = list(results) + + assert [i['string'] for i in results] == ['baz', 'bar'] + + def test_reload_all(self, Sample): + doc = Sample.find_one(integer=42) + assert doc.string == 'baz' + Sample.get_collection().update(Sample.id == doc, U(Sample, integer=1337, string="hoi")) + assert doc.string == 'baz' + doc.reload() + assert doc.string == 'hoi' + assert doc.integer == 1337 + + def test_reload_specific(self, Sample): + doc = Sample.find_one(integer=42) + assert doc.string == 'baz' + Sample.get_collection().update(Sample.id == doc, U(Sample, integer=1337, string="hoi")) + assert doc.string == 'baz' + doc.reload('string') + assert doc.string == 'hoi' + assert doc.integer == 42 + + def test_insert_one(self, Sample): + doc = Sample(string='diz', integer=2029) + assert doc.id + doc.insert_one() + assert Sample.get_collection().count() == 5 diff --git a/test/util/test_capped.py b/test/util/test_capped.py index 0a663fbd..952aaafd 100644 --- a/test/util/test_capped.py +++ b/test/util/test_capped.py @@ -2,6 +2,7 @@ from __future__ import unicode_literals +import os from functools import partial from random import choice from threading import Thread @@ -10,15 +11,21 @@ import pytest from pytest import fixture -from marrow.mongo import Document +from marrow.mongo.trait import Collection from marrow.mongo.util.capped import _patch, tail +from marrow.schema.compat import pypy -class Uncapped(Document): +skip = int(os.environ.get('TEST_SKIP_CAPPED', 0)) or pypy + +pytestmark = pytest.mark.skipif(skip, reason="Slow tests skipped.") + + +class Uncapped(Collection): __collection__ = 'test_uncapped' -class Capped(Document): +class Capped(Collection): __collection__ = 'test_capped' __capped__ = 16 * 1024 * 1024 __capped_count__ = 100