Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Align mongodb4.4 #28

Draft
wants to merge 35 commits into
base: master
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from 7 commits
Commits
Show all changes
35 commits
Select commit Hold shift + click to select a range
71af58a
positional mismatch on 4.4
davidlatwe Apr 27, 2021
991072a
positional non located match on 4.4
davidlatwe Apr 27, 2021
c947bba
fix deprecation warnings
davidlatwe Apr 28, 2021
084b0f9
set 4.4 as compat version
davidlatwe Apr 28, 2021
7ba6a9c
check positional key position
davidlatwe Apr 28, 2021
39c3ba1
update compat attributes
davidlatwe Apr 28, 2021
03f5061
check field not end with "."
davidlatwe Apr 28, 2021
fe7fe31
fix typo from c947bbaa
davidlatwe Apr 28, 2021
bd89b6f
Merge branch 'master' into align-mongodb4.4
davidlatwe May 29, 2021
5a7b9a6
testing mongodb 4.4
davidlatwe May 29, 2021
8e86856
Merge branch 'master' into align-mongodb4.4
davidlatwe Jun 5, 2021
7b50a9e
Merge branch 'master' into align-mongodb4.4
davidlatwe Jun 20, 2021
894e92a
flake8: bump max-complexity
davidlatwe Jun 20, 2021
725b30f
Merge branch 'master' into align-mongodb4.4
davidlatwe Jun 20, 2021
18fa399
project: fix regression
davidlatwe Jun 20, 2021
151c90e
tests: update projection test specific for 4.4
davidlatwe Jun 20, 2021
853286b
Merge branch 'master' into align-mongodb4.4
davidlatwe Jun 26, 2021
6b445e2
Merge branch 'master' into align-mongodb4.4
davidlatwe Jul 2, 2021
ce6a36c
add projection path collision check
davidlatwe Jul 7, 2021
2a99bdd
add $mod remainder check
davidlatwe Jul 7, 2021
318389b
Merge branch 'master' into align-mongodb4.4
davidlatwe Jul 7, 2021
c7a3261
flake8: bump max-complexity, again
davidlatwe Jul 7, 2021
7cdc500
flake8: fix indent (E126)
davidlatwe Jul 7, 2021
3fd4fd9
Merge branch 'master' into align-mongodb4.4
davidlatwe Jan 21, 2023
c3a2faa
Cleanup after master merged
davidlatwe Jan 21, 2023
55224a2
Add mongodb 5.0, 6.0
davidlatwe Jan 22, 2023
0735ebf
Add mongoengine to test; Drop Python 3.6
davidlatwe Jan 22, 2023
fcaa688
Update poetry.lock
davidlatwe Jan 22, 2023
0c3862d
Update poetry.lock
davidlatwe Jan 22, 2023
fe4d6b4
Merge branch 'master' into align-mongodb4.4
davidlatwe Feb 3, 2023
0ba3d5d
Fix query mod regression
davidlatwe Feb 3, 2023
aef4af7
Exclude poetry-version from matrix
davidlatwe Feb 3, 2023
e83e863
Add test cases
davidlatwe Feb 3, 2023
016b62a
Cleanup
davidlatwe Feb 3, 2023
71f8283
Fix warnings in test
davidlatwe Feb 5, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 5 additions & 4 deletions montydb/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,11 +18,12 @@
# Assembling crucial classes and functions form pymongo module,
# some of them may modified by needs.

try:
from collections.abc import MutableMapping
except ImportError:
from collections import MutableMapping

from collections import (
OrderedDict,
MutableMapping,
)
from collections import OrderedDict
from .types import (
abc,
iteritems,
Expand Down
4 changes: 2 additions & 2 deletions montydb/client.py
Original file line number Diff line number Diff line change
Expand Up @@ -128,9 +128,9 @@ def get_database(self, name):
"""
# verify database name
if platform.system() == "Windows":
is_invaild = set('/\. "$*<>:|?').intersection(set(name))
is_invaild = set(r'/\. "$*<>:|?').intersection(set(name))
else:
is_invaild = set('/\. "$').intersection(set(name))
is_invaild = set(r'/\. "$').intersection(set(name))

if is_invaild or not name:
raise errors.OperationFailure("Invaild database name.")
Expand Down
21 changes: 18 additions & 3 deletions montydb/configure.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@

URI_SCHEME_PREFIX = "montydb://"

MONGO_COMPAT_VERSIONS = ("3.6", "4.0", "4.2")
MONGO_COMPAT_VERSIONS = ("3.6", "4.0", "4.2", "4.4")


_pinned_repository = {"_": None}
Expand Down Expand Up @@ -199,7 +199,7 @@ def set_storage(repository=None,

storage = storage or DEFAULT_STORAGE

valid_versions = list(MONGO_COMPAT_VERSIONS) + ["4.4"]
valid_versions = list(MONGO_COMPAT_VERSIONS)
if mongo_version and mongo_version not in valid_versions:
raise ConfigurationError(
"Unknown mongodb version: %s, currently supported versions are: %s"
Expand Down Expand Up @@ -277,7 +277,7 @@ def _bson_init(use_bson):


def _mongo_compat(version):
from .engine import queries
from .engine import queries, project

if version.startswith("3"):
v3 = getattr(queries, "_is_comparable_ver3")
Expand All @@ -293,3 +293,18 @@ def _mongo_compat(version):
else:
setattr(queries, "_regex_options_check",
getattr(queries, "_regex_options_"))

if version == "4.4":
setattr(project, "_positional_mismatch",
davidlatwe marked this conversation as resolved.
Show resolved Hide resolved
getattr(project, "_positional_mismatch_v44"))
davidlatwe marked this conversation as resolved.
Show resolved Hide resolved
setattr(project, "_check_positional_key",
davidlatwe marked this conversation as resolved.
Show resolved Hide resolved
getattr(project, "_check_positional_key_v44"))
davidlatwe marked this conversation as resolved.
Show resolved Hide resolved
setattr(project, "_include_positional_non_located_match",
davidlatwe marked this conversation as resolved.
Show resolved Hide resolved
getattr(project, "_include_positional_non_located_match_v44"))
davidlatwe marked this conversation as resolved.
Show resolved Hide resolved
else:
setattr(project, "_positional_mismatch",
davidlatwe marked this conversation as resolved.
Show resolved Hide resolved
getattr(project, "_positional_mismatch_"))
davidlatwe marked this conversation as resolved.
Show resolved Hide resolved
setattr(project, "_check_positional_key",
davidlatwe marked this conversation as resolved.
Show resolved Hide resolved
getattr(project, "_check_positional_key_"))
davidlatwe marked this conversation as resolved.
Show resolved Hide resolved
setattr(project, "_include_positional_non_located_match",
davidlatwe marked this conversation as resolved.
Show resolved Hide resolved
getattr(project, "_include_positional_non_located_match_"))
davidlatwe marked this conversation as resolved.
Show resolved Hide resolved
65 changes: 56 additions & 9 deletions montydb/engine/project.py
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,11 @@ def _perr_doc(val):
return "{ " + ", ".join(v_lis) + " }"


_check_positional_key_ = False
_check_positional_key_v44 = True
_check_positional_key = _check_positional_key_v44


class Projector(object):
"""
"""
Expand Down Expand Up @@ -109,13 +114,9 @@ def __call__(self, fieldwalker):
fieldwalker.go(path).get()

if self.include_flag:
located_match = None
if self.matched is not None:
located_match = self.matched.located

projected = inclusion(fieldwalker,
positioned,
located_match,
self.matched,
init_doc)
else:
projected = exclusion(fieldwalker, init_doc)
Expand Down Expand Up @@ -186,6 +187,12 @@ def parser(self, spec, qfilter):
elif key == "_id" and not _is_include(val):
self.proj_with_id = False

elif _check_positional_key and key.startswith("$."):
raise OperationFailure("FieldPath field names may not start "
"with '$'.")
elif _check_positional_key and key.endswith("."):
raise OperationFailure("FieldPath must not end with a '.'.")

else:
# Normal field options, include or exclude.
flag = _is_include(val)
Expand Down Expand Up @@ -220,6 +227,15 @@ def parser(self, spec, qfilter):
"Positional projection '{}' contains the positional "
"operator more than once.".format(key))

if _check_positional_key and ".$." in key:
raise OperationFailure(
"As of 4.4, it's illegal to specify positional "
"operator in the middle of a path.Positional "
"projection may only be used at the end, for example: "
"a.b.$. If the query previously used a form like "
"a.b.$.d, remove the parts following the '$' and the "
"results will be equivalent.", code=31394)

path = key.split(".$", 1)[0]
conditions = qfilter.conditions
match_query = _is_positional_match(conditions, path)
Expand Down Expand Up @@ -304,8 +320,12 @@ def _positional(fieldwalker):
% field,
code=2)

if (int(matched_index) >= elem_count and
self.matched.full_path.startswith(node.full_path)):
if _positional_mismatch(
int(matched_index),
elem_count,
self.matched.full_path,
node.full_path
):
raise OperationFailure(
"Executor error during find command "
":: caused by :: errmsg: "
Expand All @@ -318,8 +338,20 @@ def _positional(fieldwalker):
return _positional


def inclusion(fieldwalker, positioned, located_match, init_doc):
def _positional_mismatch_(matched, elem_count, matched_path, node_path):
return matched >= elem_count and matched_path.startswith(node_path)


def _positional_mismatch_v44(matched, elem_count, matched_path, node_path):
return matched >= elem_count


_positional_mismatch = _positional_mismatch_v44


def inclusion(fieldwalker, positioned, matched, init_doc):
_doc_type = fieldwalker.doc_type
located_match = matched.located if matched else False

def _inclusion(node, init_doc=None):
doc = node.value
Expand Down Expand Up @@ -353,7 +385,10 @@ def _inclusion(node, init_doc=None):
if isinstance(child.value, _doc_type):
new_doc.append(child.value)
else:
new_doc.append(child.value)
if _include_positional_non_located_match(matched, node):
davidlatwe marked this conversation as resolved.
Show resolved Hide resolved
new_doc.append(child.value)
else:
new_doc.append(_doc_type())

return new_doc or _no_val

Expand Down Expand Up @@ -391,6 +426,18 @@ def _inclusion(node, init_doc=None):
return _inclusion(fieldwalker.tree.root, init_doc)


def _include_positional_non_located_match_(matched, node):
return True


def _include_positional_non_located_match_v44(matched, node):
return matched.full_path.startswith(node.full_path)


_include_positional_non_located_match = \
_include_positional_non_located_match_v44


def exclusion(fieldwalker, init_doc):
_doc_type = fieldwalker.doc_type

Expand Down
5 changes: 4 additions & 1 deletion montydb/engine/queries.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,10 @@
import re
from copy import deepcopy
from datetime import datetime
from collections import Mapping
try:
from collections.abc import MutableMapping
except ImportError:
from collections import MutableMapping

from ..errors import OperationFailure

Expand Down
5 changes: 4 additions & 1 deletion montydb/engine/weighted.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,9 @@

from datetime import datetime
from collections import Mapping
try:
from collections.abc import Mapping
except ImportError:
from collections import Mapping

from ..types import (
integer_types,
Expand Down
2 changes: 1 addition & 1 deletion montydb/utils/io.py
Original file line number Diff line number Diff line change
Expand Up @@ -210,7 +210,7 @@ class MongoQueryRecorder(object):

def __init__(self, mongodb, namespace=None, user=None):
self._mongodb = mongodb
self._namespace = namespace or {"$regex": mongodb.name + "\..*"}
self._namespace = namespace or {"$regex": mongodb.name + r"\..*"}
self._user = user

self._epoch = datetime(1970, 1, 1)
Expand Down
42 changes: 35 additions & 7 deletions tests/test_engine/test_projection/test_projection_positional.py
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,7 @@ def test_projection_positional_4(monty_proj, mongo_proj):
assert next(mongo_c) == next(monty_c)


def test_projection_positional_5(monty_proj, mongo_proj):
def test_projection_positional_5(monty_proj, mongo_proj, mongo_version):
docs = [
{"a": {"b": [1, 2, 3], "c": [4, 5, 6]}},
{"a": {"b": [1, 2, 3], "c": [4]}},
Expand All @@ -91,8 +91,16 @@ def test_projection_positional_5(monty_proj, mongo_proj):

assert count_documents(mongo_c, spec) == 2
assert count_documents(monty_c, spec) == count_documents(mongo_c, spec)
for i in range(2):
assert next(mongo_c) == next(monty_c)

if mongo_version[:2] >= [4, 4]:
with pytest.raises(mongo_op_fail) as mongo_err:
davidlatwe marked this conversation as resolved.
Show resolved Hide resolved
next(mongo_c)

with pytest.raises(monty_op_fail) as monty_err:
davidlatwe marked this conversation as resolved.
Show resolved Hide resolved
next(monty_c)
else:
for i in range(1):
davidlatwe marked this conversation as resolved.
Show resolved Hide resolved
assert next(mongo_c) == next(monty_c)


def test_projection_positional_6(monty_proj, mongo_proj):
Expand Down Expand Up @@ -237,7 +245,7 @@ def run(spec, proj):
run(spec, proj)


def test_projection_positional_12(monty_proj, mongo_proj):
def test_projection_positional_12(monty_proj, mongo_proj, mongo_version):
docs = [
{"a": [{"b": [{"c": 1}, {"x": 1}]}, {"b": [{"c": 1}, {"x": 1}]}]},

Expand All @@ -255,6 +263,16 @@ def run(proj):
for i in range(2):
assert next(mongo_c) == next(monty_c)

def fail(proj):
monty_c = monty_proj(docs, spec, proj)
mongo_c = mongo_proj(docs, spec, proj)

with pytest.raises(mongo_op_fail) as mongo_err:
davidlatwe marked this conversation as resolved.
Show resolved Hide resolved
next(mongo_c)

with pytest.raises(monty_op_fail) as monty_err:
davidlatwe marked this conversation as resolved.
Show resolved Hide resolved
next(monty_c)

for ie in range(2):
proj = {"a.b.5": ie}
run(proj)
Expand All @@ -272,13 +290,13 @@ def run(proj):
run(proj)

proj = {"a.b.c.": ie} # Redundant dot
run(proj)
run(proj) if mongo_version[:2] < [4, 4] else fail(proj)

proj = {"a.b.c": ie}
run(proj)


def test_projection_positional_13(monty_proj, mongo_proj):
def test_projection_positional_13(monty_proj, mongo_proj, mongo_version):
docs = [
{"a": [{"b": [1, 5]}, {"b": 2}, {"b": [3, 10, 4]}],
"c": [{"b": [1]}, {"b": 2}, {"b": [3, 5]}]},
Expand All @@ -293,11 +311,21 @@ def run(proj):
assert count_documents(monty_c, spec) == count_documents(mongo_c, spec)
assert next(mongo_c) == next(monty_c)

def fail(proj):
monty_c = monty_proj(docs, spec, proj)
mongo_c = mongo_proj(docs, spec, proj)

with pytest.raises(mongo_op_fail) as mongo_err:
davidlatwe marked this conversation as resolved.
Show resolved Hide resolved
next(mongo_c)

with pytest.raises(monty_op_fail) as monty_err:
davidlatwe marked this conversation as resolved.
Show resolved Hide resolved
next(monty_c)

proj = {"a.b.$": 1}
run(proj)

proj = {"a.$.b": 1}
run(proj)
run(proj) if mongo_version[:2] < [4, 4] else fail(proj)


def test_projection_positional_14(monty_proj, mongo_proj):
Expand Down