Merge pull request #2721 from bagerard/clone_jschlyter_decimal128

[Clone] implement Decimal128Field

Author: bagerard

docs/changelog.rst

@@ -15,7 +15,9 @@ Development
 - Added meta ``auto_create_index_on_save`` so you can enable index creation
   on :meth:`~mongoengine.Document.save()` (as it was < 0.26.0).
 - BREAKING CHANGE: remove deprecated method ``ensure_index`` (replaced by ``create_index`` long time ago).
-
+- Addition of Decimal128Field: :class:`~mongoengine.fields.Decimal128Field` for accurate representation of Decimals (much better than the legacy field DecimalField).
+  Although it could work to switch an existing DecimalField to Decimal128Field without applying a migration script,
+  it is not recommended to do so (DecimalField uses float/str to store the value, Decimal128Field uses Decimal128).
 
 Changes in 0.25.0
 =================

mongoengine/document.py

@@ -938,7 +938,6 @@ def list_indexes(cls):
         classes = []
 
         def get_classes(cls):
-
             if cls not in classes and isinstance(cls, TopLevelDocumentMetaclass):
                 classes.append(cls)
 
@@ -965,7 +964,7 @@ def get_classes(cls):
 
         get_classes(cls)
 
-        # get the indexes spec for all of the gathered classes
+        # get the indexes spec for all the gathered classes
         def get_indexes_spec(cls):
             indexes = []
 
@@ -1000,8 +999,10 @@ def compare_indexes(cls):
         required = cls.list_indexes()
 
         existing = []
-        for info in cls._get_collection().index_information().values():
+        collection = cls._get_collection()
+        for info in collection.index_information().values():
             if "_fts" in info["key"][0]:
+                # Useful for text indexes (but not only)
                 index_type = info["key"][0][1]
                 text_index_fields = info.get("weights").keys()
                 existing.append([(key, index_type) for key in text_index_fields])

mongoengine/fields.py

@@ -12,6 +12,7 @@
 import gridfs
 import pymongo
 from bson import SON, Binary, DBRef, ObjectId
+from bson.decimal128 import Decimal128, create_decimal128_context
 from bson.int64 import Int64
 from pymongo import ReturnDocument
 
@@ -95,6 +96,7 @@
     "MultiLineStringField",
     "MultiPolygonField",
     "GeoJsonBaseField",
+    "Decimal128Field",
 )
 
 RECURSIVE_REFERENCE_CONSTANT = "self"
@@ -438,7 +440,10 @@ def prepare_query_value(self, op, value):
 
 
 class DecimalField(BaseField):
-    """Fixed-point decimal number field. Stores the value as a float by default unless `force_string` is used.
+    """Disclaimer: This field is kept for historical reason but since it converts the values to float, it
+    is not suitable for true decimal storage. Consider using :class:`~mongoengine.fields.Decimal128Field`.
+
+    Fixed-point decimal number field. Stores the value as a float by default unless `force_string` is used.
     If using floats, beware of Decimal to float conversion (potential precision loss)
     """
 
@@ -2650,3 +2655,49 @@ def to_mongo(self, document):
             )
         else:
             return super().to_mongo(document)
+
+
+class Decimal128Field(BaseField):
+    """
+    128-bit decimal-based floating-point field capable of emulating decimal
+    rounding with exact precision. This field will expose decimal.Decimal but stores the value as a
+    `bson.Decimal128` behind the scene, this field is intended for monetary data, scientific computations, etc.
+    """
+
+    DECIMAL_CONTEXT = create_decimal128_context()
+
+    def __init__(self, min_value=None, max_value=None, **kwargs):
+        self.min_value = min_value
+        self.max_value = max_value
+        super().__init__(**kwargs)
+
+    def to_mongo(self, value):
+        if value is None:
+            return None
+        if isinstance(value, Decimal128):
+            return value
+        if not isinstance(value, decimal.Decimal):
+            with decimal.localcontext(self.DECIMAL_CONTEXT) as ctx:
+                value = ctx.create_decimal(value)
+        return Decimal128(value)
+
+    def to_python(self, value):
+        if value is None:
+            return None
+        return self.to_mongo(value).to_decimal()
+
+    def validate(self, value):
+        if not isinstance(value, Decimal128):
+            try:
+                value = Decimal128(value)
+            except (TypeError, ValueError, decimal.InvalidOperation) as exc:
+                self.error("Could not convert value to Decimal128: %s" % exc)
+
+        if self.min_value is not None and value.to_decimal() < self.min_value:
+            self.error("Decimal value is too small")
+
+        if self.max_value is not None and value.to_decimal() > self.max_value:
+            self.error("Decimal value is too large")
+
+    def prepare_query_value(self, op, value):
+        return super().prepare_query_value(op, self.to_mongo(value))

tests/fields/test_decimal128_field.py

@@ -0,0 +1,149 @@
+import json
+import random
+from decimal import Decimal
+
+import pytest
+from bson.decimal128 import Decimal128
+
+from mongoengine import Decimal128Field, Document, ValidationError
+from tests.utils import MongoDBTestCase, get_as_pymongo
+
+
+class Decimal128Document(Document):
+    dec128_fld = Decimal128Field()
+    dec128_min_0 = Decimal128Field(min_value=0)
+    dec128_max_100 = Decimal128Field(max_value=100)
+
+
+def generate_test_cls() -> Document:
+    Decimal128Document.drop_collection()
+    Decimal128Document(dec128_fld=None).save()
+    Decimal128Document(dec128_fld=Decimal(1)).save()
+    return Decimal128Document
+
+
+class TestDecimal128Field(MongoDBTestCase):
+    def test_decimal128_validation_good(self):
+        doc = Decimal128Document()
+
+        doc.dec128_fld = Decimal(0)
+        doc.validate()
+
+        doc.dec128_fld = Decimal(50)
+        doc.validate()
+
+        doc.dec128_fld = Decimal(110)
+        doc.validate()
+
+        doc.dec128_fld = Decimal("110")
+        doc.validate()
+
+    def test_decimal128_validation_invalid(self):
+        """Ensure that invalid values cannot be assigned."""
+
+        doc = Decimal128Document()
+
+        doc.dec128_fld = "ten"
+
+        with pytest.raises(ValidationError):
+            doc.validate()
+
+    def test_decimal128_validation_min(self):
+        """Ensure that out of bounds values cannot be assigned."""
+
+        doc = Decimal128Document()
+
+        doc.dec128_min_0 = Decimal(50)
+        doc.validate()
+
+        doc.dec128_min_0 = Decimal(-1)
+        with pytest.raises(ValidationError):
+            doc.validate()
+
+    def test_decimal128_validation_max(self):
+        """Ensure that out of bounds values cannot be assigned."""
+
+        doc = Decimal128Document()
+
+        doc.dec128_max_100 = Decimal(50)
+        doc.validate()
+
+        doc.dec128_max_100 = Decimal(101)
+        with pytest.raises(ValidationError):
+            doc.validate()
+
+    def test_eq_operator(self):
+        cls = generate_test_cls()
+        assert cls.objects(dec128_fld=1.0).count() == 1
+        assert cls.objects(dec128_fld=2.0).count() == 0
+
+    def test_ne_operator(self):
+        cls = generate_test_cls()
+        assert cls.objects(dec128_fld__ne=None).count() == 1
+        assert cls.objects(dec128_fld__ne=1).count() == 1
+        assert cls.objects(dec128_fld__ne=1.0).count() == 1
+
+    def test_gt_operator(self):
+        cls = generate_test_cls()
+        assert cls.objects(dec128_fld__gt=0.5).count() == 1
+
+    def test_lt_operator(self):
+        cls = generate_test_cls()
+        assert cls.objects(dec128_fld__lt=1.5).count() == 1
+
+    def test_field_exposed_as_python_Decimal(self):
+        # from int
+        model = Decimal128Document(dec128_fld=100).save()
+        assert isinstance(model.dec128_fld, Decimal)
+        model = Decimal128Document.objects.get(id=model.id)
+        assert isinstance(model.dec128_fld, Decimal)
+        assert model.dec128_fld == Decimal("100")
+
+    def test_storage(self):
+        # from int
+        model = Decimal128Document(dec128_fld=100).save()
+        assert get_as_pymongo(model) == {
+            "_id": model.id,
+            "dec128_fld": Decimal128("100"),
+        }
+
+        # from str
+        model = Decimal128Document(dec128_fld="100.0").save()
+        assert get_as_pymongo(model) == {
+            "_id": model.id,
+            "dec128_fld": Decimal128("100.0"),
+        }
+
+        # from float
+        model = Decimal128Document(dec128_fld=100.0).save()
+        assert get_as_pymongo(model) == {
+            "_id": model.id,
+            "dec128_fld": Decimal128("100"),
+        }
+
+        # from Decimal
+        model = Decimal128Document(dec128_fld=Decimal(100)).save()
+        assert get_as_pymongo(model) == {
+            "_id": model.id,
+            "dec128_fld": Decimal128("100"),
+        }
+        model = Decimal128Document(dec128_fld=Decimal("100.0")).save()
+        assert get_as_pymongo(model) == {
+            "_id": model.id,
+            "dec128_fld": Decimal128("100.0"),
+        }
+
+        # from Decimal128
+        model = Decimal128Document(dec128_fld=Decimal128("100")).save()
+        assert get_as_pymongo(model) == {
+            "_id": model.id,
+            "dec128_fld": Decimal128("100"),
+        }
+
+    def test_json(self):
+        Decimal128Document.drop_collection()
+        f = str(random.random())
+        Decimal128Document(dec128_fld=f).save()
+        json_str = Decimal128Document.objects.to_json()
+        array = json.loads(json_str)
+        assert array[0]["dec128_fld"] == {"$numberDecimal": str(f)}