Sitelet https://github.com/JuDFTteam/optimade-python-tools/commit/f5e7b5773dd331c002578a62b751f3b7d8d955e7
Skip to content

Commit f5e7b57

Browse files
ml-evsml-evs
authored
Create template filtertransformer BaseTransformer (Materials-Consortia#287)
* Added BaseTransformer submodule * Created v1.0.0 grammar and a link to v0.10.1 * Tidied older transformer code: - Raise deprecation warning for DjangoTransformer - Removed debug/json transformers - Tweaked basetransformer docstrings * Updated contributing docs * Add no cover to CLI validator method
1 parent 36af320 commit f5e7b57

17 files changed

Lines changed: 390 additions & 508 deletions

File tree

‎CONTRIBUTING.md‎

Lines changed: 43 additions & 77 deletions
Original file line numberDiff line numberDiff line change
@@ -4,37 +4,40 @@ The [Materials Consortia](https://github.com/Materials-Consortia) is very open t
44

55
This may be anything from simple feedback and raising [new issues](https://github.com/Materials-Consortia/optimade-python-tools/issues/new) to creating [new PRs](https://github.com/Materials-Consortia/optimade-python-tools/compare).
66

7-
We have below recommendations for setting up an environment in which one may develop the package further.
7+
Recommendations for setting up a development environment can be found in the [Installation instructions](https://www.optimade.org/optimade-python-tools/install/#full-development-installation).
88

99
## Getting Started with Filter Parsing and Transforming
1010

1111
Example use:
1212

1313
```python
14-
from optimade.filterparser import Parser
14+
from optimade.filterparser import LarkParser
1515

16-
p = Parser(version=(0,9,7))
16+
p = LarkParser(version=(1, 0, 0))
1717
tree = p.parse("nelements<3")
1818
print(tree)
1919
```
2020

2121
```shell
22-
Tree(start, [Tree(expression, [Tree(term, [Tree(atom, [Tree(comparison, [Token(VALUE, 'nelements'), Token(OPERATOR, '<'), Token(VALUE, '3')])])])])])
22+
Tree('filter', [Tree('expression', [Tree('expression_clause', [Tree('expression_phrase', [Tree('comparison', [Tree('property_first_comparison', [Tree('property', [Token('IDENTIFIER', 'nelements')]), Tree('value_op_rhs', [Token('OPERATOR', '<'), Tree('value', [Tree('number', [Token('SIGNED_INT', '3')])])])])])])])])])
2323
```
2424

2525
```python
2626
print(tree.pretty())
2727
```
2828

2929
```shell
30-
start
30+
filter
3131
expression
32-
term
33-
atom
32+
expression_clause
33+
expression_phrase
3434
comparison
35-
nelements
36-
<
37-
3
35+
property_first_comparison
36+
property nelements
37+
value_op_rhs
38+
<
39+
value
40+
number 3
3841
```
3942

4043
```python
@@ -43,36 +46,31 @@ print(tree.pretty())
4346
```
4447

4548
```shell
46-
start
49+
filter
4750
expression
48-
term
49-
term
50-
atom
51-
comparison
52-
_mp_bandgap
53-
>
54-
5.0
55-
AND
56-
atom
51+
expression_clause
52+
expression_phrase
5753
comparison
58-
_cod_molecular_weight
59-
<
60-
350
61-
```
62-
63-
```python
64-
# Assumes graphviz installed on system (e.g. `conda install -c anaconda graphviz`) and `pip install pydot`
65-
from lark.tree import pydot__tree_to_png
66-
67-
pydot__tree_to_png(tree, "exampletree.png")
54+
property_first_comparison
55+
property _mp_bandgap
56+
value_op_rhs
57+
>
58+
value
59+
number 5.0
60+
expression_phrase
61+
comparison
62+
property_first_comparison
63+
property _cod_molecular_weight
64+
value_op_rhs
65+
<
66+
value
67+
number 350
6868
```
6969

70-
![example tree](images/exampletree.png)
71-
7270
### Flow for Parsing User-Supplied Filter and Converting to Backend Query
7371

74-
`optimade.filterparser.Parser` will take user input to generate a `lark.Tree` and feed that to a `lark.Transformer`.
75-
E.g., `optimade.filtertransformers.mongo.MongoTransformer` will turn the tree into something useful for your MondoDB backend:
72+
`optimade.filterparser.LarkParser` will take user input to generate a `lark.Tree` and feed that to a `lark.Transformer`.
73+
E.g., `optimade.filtertransformers.mongo.MongoTransformer` will turn the tree into something useful for your MongoDB backend:
7674

7775
```python
7876
# Example: Converting to MongoDB Query Syntax
@@ -85,55 +83,23 @@ query = transformer.transform(tree)
8583
print(query)
8684
```
8785

88-
```python
89-
{'$and': [{'_mp_bandgap': {'$gt': 5.0}}, {'_cod_molecular_weight': {'$lt': 350.0}}]}
86+
```json
87+
{
88+
"$and": [
89+
{"_mp_bandgap": {"$gt": 5.0}},
90+
{"_cod_molecular_weight": {"$lt": 350.0}}
91+
]
92+
}
9093
```
9194

92-
There is also a [basic JSON transformer][optimade.filtertransformers.json] you can use as a simple example for developing your own transformer.
93-
You can also use the JSON output it produces as an easy-to-parse input for a "transformer" in your programming language of choice.
94-
95-
```python
96-
class JSONTransformer(Transformer):
97-
def __init__(self, compact=False):
98-
self.compact = compact
99-
super().__init__()
100-
101-
def __default__(self, data, children):
102-
items = []
103-
for c in children:
104-
if isinstance(c, Token):
105-
token_repr = {
106-
"@module": "lark.lexer",
107-
"@class": "Token",
108-
"type_": c.type,
109-
"value": c.value,
110-
}
111-
if self.compact:
112-
del token_repr["@module"]
113-
del token_repr["@class"]
114-
items.append(token_repr)
115-
elif isinstance(c, dict):
116-
items.append(c)
117-
else:
118-
raise ValueError(f"Unknown type {type(c)} for tree child {c}")
119-
tree_repr = {
120-
"@module": "lark",
121-
"@class": "Tree",
122-
"data": data,
123-
"children": items,
124-
}
125-
if self.compact:
126-
del tree_repr["@module"]
127-
del tree_repr["@class"]
128-
return tree_repr
129-
```
13095

13196
### Developing New Filter Transformers
13297

133-
If you would like to add a new transformer, please add:
98+
If you would like to add a new transformer, please raise an issue to signal your intent (in case someone else is already working on this).
99+
Adding a transformer requires the following:
134100

135-
1. A module (.py file) in the `optimade/filtertransformers` folder.
136-
2. Any additional Python requirements must be optional and provided as a separate "`extra_requires`" entry in `setup.py`.
101+
1. A new submodule (`.py` file) in the `optimade/filtertransformers` folder containing an implementation of the transformer object, preferably one that extends `optimade.filtertransformers.base_transformer.BaseTransformer`.
102+
2. Any additional Python requirements must be optional and provided as a separate "`extra_requires`" entry in `setup.py` and in the `requirements.txt` file.
137103
3. Tests in `optimade/filtertransformers/tests` that are skipped if the required packages fail to import.
138104

139105
For examples, please check out existing filter transformers.

‎docs/api_reference/filtertransformers/debug.md‎

Lines changed: 0 additions & 3 deletions
This file was deleted.

‎docs/api_reference/filtertransformers/json.md‎

Lines changed: 0 additions & 3 deletions
This file was deleted.

‎optimade/filtertransformers/__init__.py‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
""" This module implements filter transformer classes for different backends. These
2+
classes typically parse the filter with Lark and produce an appropriate query for the
3+
given backend.
4+
5+
"""

‎optimade/filtertransformers/base_transformer.py‎

Lines changed: 179 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,179 @@
1+
import abc
2+
from lark import Transformer, v_args
3+
from typing import Dict
4+
from optimade.server.mappers import BaseResourceMapper
5+
6+
__all__ = ("BaseTransformer",)
7+
8+
9+
class BaseTransformer(abc.ABC, Transformer):
10+
"""Generic filter transformer that handles various
11+
parts of the grammar in a backend non-specific way.
12+
13+
"""
14+
15+
# map from standard comparison operators to the backend-specific version
16+
operator_map: Dict[str, str] = {
17+
"<": None,
18+
"<=": None,
19+
">": None,
20+
">=": None,
21+
"!=": None,
22+
"=": None,
23+
}
24+
25+
# map from back-end specific operators to their inverse
26+
# e.g. {"$lt": "$gt"} for MongoDB.
27+
reversed_operator_map: Dict[str, str] = {}
28+
29+
def __init__(self, mapper: BaseResourceMapper = None):
30+
"""Initialise the transformer object, optionally loading in a
31+
resource mapper for use when post-processing.
32+
33+
"""
34+
self.mapper = mapper
35+
36+
def postprocess(self, query):
37+
"""Post-process the query according to the rules defined for
38+
the backend.
39+
40+
"""
41+
return query
42+
43+
def transform(self, tree):
44+
""" Transform the query using the Lark transformer then post-process. """
45+
return self.postprocess(super().transform(tree))
46+
47+
def __default__(self, data, children, meta):
48+
raise NotImplementedError(
49+
f"Calling __default__, i.e., unknown grammar concept. data: {data}, children: {children}, meta: {meta}"
50+
)
51+
52+
def filter(self, arg):
53+
""" filter: expression* """
54+
return arg[0] if arg else None
55+
56+
@v_args(inline=True)
57+
def constant(self, value):
58+
""" constant: string | number """
59+
# Note: Return as is.
60+
return value
61+
62+
@v_args(inline=True)
63+
def value(self, value):
64+
""" value: string | number | property """
65+
# Note: Return as is.
66+
return value
67+
68+
@v_args(inline=True)
69+
def non_string_value(self, value):
70+
""" non_string_value: number | property """
71+
# Note: Return as is.
72+
return value
73+
74+
@v_args(inline=True)
75+
def not_implemented_string(self, value):
76+
"""not_implemented_string: value
77+
78+
Raises:
79+
NotImplementedError: For further information, see Materials-Consortia/OPTIMADE issue 157:
80+
https://github.com/Materials-Consortia/OPTIMADE/issues/157
81+
82+
"""
83+
raise NotImplementedError("Comparing strings is not yet implemented.")
84+
85+
def property(self, arg):
86+
""" property: IDENTIFIER ( "." IDENTIFIER )* """
87+
return ".".join(arg)
88+
89+
@v_args(inline=True)
90+
def string(self, string):
91+
""" string: ESCAPED_STRING """
92+
return string.strip('"')
93+
94+
@v_args(inline=True)
95+
def signed_int(self, number):
96+
""" signed_int : SIGNED_INT """
97+
return int(number)
98+
99+
@v_args(inline=True)
100+
def number(self, number):
101+
""" number: SIGNED_INT | SIGNED_FLOAT """
102+
if number.type == "SIGNED_INT":
103+
type_ = int
104+
elif number.type == "SIGNED_FLOAT":
105+
type_ = float
106+
return type_(number)
107+
108+
@v_args(inline=True)
109+
def comparison(self, value):
110+
""" comparison: constant_first_comparison | property_first_comparison """
111+
# Note: Return as is.
112+
return value
113+
114+
@abc.abstractmethod
115+
def value_list(self, arg):
116+
""" value_list: [ OPERATOR ] value ( "," [ OPERATOR ] value )* """
117+
118+
@abc.abstractmethod
119+
def value_zip(self, arg):
120+
""" value_zip: [ OPERATOR ] value ":" [ OPERATOR ] value (":" [ OPERATOR ] value)* """
121+
122+
@abc.abstractmethod
123+
def value_zip_list(self, arg):
124+
""" value_zip_list: value_zip ( "," value_zip )* """
125+
126+
@abc.abstractmethod
127+
def expression(self, arg):
128+
""" expression: expression_clause ( OR expression_clause ) """
129+
130+
@abc.abstractmethod
131+
def expression_clause(self, arg):
132+
""" expression_clause: expression_phrase ( AND expression_phrase )* """
133+
134+
@abc.abstractmethod
135+
def expression_phrase(self, arg):
136+
""" expression_phrase: [ NOT ] ( comparison | "(" expression ")" ) """
137+
138+
@abc.abstractmethod
139+
def property_first_comparison(self, arg):
140+
"""property_first_comparison: property ( value_op_rhs | known_op_rhs | fuzzy_string_op_rhs | set_op_rhs |
141+
set_zip_op_rhs | length_op_rhs )
142+
143+
"""
144+
145+
@abc.abstractmethod
146+
def constant_first_comparison(self, arg):
147+
""" constant_first_comparison: constant OPERATOR ( non_string_value | not_implemented_string ) """
148+
149+
@v_args(inline=True)
150+
@abc.abstractmethod
151+
def value_op_rhs(self, operator, value):
152+
""" value_op_rhs: OPERATOR value """
153+
154+
@abc.abstractmethod
155+
def known_op_rhs(self, arg):
156+
""" known_op_rhs: IS ( KNOWN | UNKNOWN ) """
157+
158+
@abc.abstractmethod
159+
def fuzzy_string_op_rhs(self, arg):
160+
""" fuzzy_string_op_rhs: CONTAINS value | STARTS [ WITH ] value | ENDS [ WITH ] value """
161+
162+
@abc.abstractmethod
163+
def set_op_rhs(self, arg):
164+
""" set_op_rhs: HAS ( [ OPERATOR ] value | ALL value_list | ANY value_list | ONLY value_list ) """
165+
166+
@abc.abstractmethod
167+
def length_op_rhs(self, arg):
168+
""" length_op_rhs: LENGTH [ OPERATOR ] value """
169+
170+
@abc.abstractmethod
171+
def set_zip_op_rhs(self, arg):
172+
"""set_zip_op_rhs: property_zip_addon HAS ( value_zip | ONLY value_zip_list | ALL value_zip_list |
173+
ANY value_zip_list )
174+
175+
"""
176+
177+
@abc.abstractmethod
178+
def property_zip_addon(self, arg):
179+
""" property_zip_addon: ":" property (":" property)* """

0 commit comments

Comments
 (0)