Skip to content

Commit a0c53db

Browse files
miguelgrinbergmiguelgrinberg
authored
support composable templates (#1943)
* support composable templates * do not error on aiohttp warning * rename new index template to composable index template * remove old comment
1 parent 0dd69f8 commit a0c53db

18 files changed

+196
-20
lines changed

‎.gitignore

+2
Original file line numberDiff line numberDiff line change
@@ -17,3 +17,5 @@ venv
1717

1818
# sample code for GitHub issues
1919
issues
20+
.direnv
21+
.envrc

‎elasticsearch_dsl/__init__.py

+10-1
Original file line numberDiff line numberDiff line change
@@ -79,7 +79,14 @@
7979
construct_field,
8080
)
8181
from .function import SF
82-
from .index import AsyncIndex, AsyncIndexTemplate, Index, IndexTemplate
82+
from .index import (
83+
AsyncComposableIndexTemplate,
84+
AsyncIndex,
85+
AsyncIndexTemplate,
86+
ComposableIndexTemplate,
87+
Index,
88+
IndexTemplate,
89+
)
8390
from .mapping import AsyncMapping, Mapping
8491
from .query import Q, Query
8592
from .response import AggResponse, Response, UpdateByQueryResponse
@@ -102,6 +109,7 @@
102109
"A",
103110
"Agg",
104111
"AggResponse",
112+
"AsyncComposableIndexTemplate",
105113
"AsyncDocument",
106114
"AsyncEmptySearch",
107115
"AsyncFacetedSearch",
@@ -117,6 +125,7 @@
117125
"Boolean",
118126
"Byte",
119127
"Completion",
128+
"ComposableIndexTemplate",
120129
"ConstantKeyword",
121130
"CustomField",
122131
"Date",

‎elasticsearch_dsl/_async/index.py

+51-3
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,47 @@ async def save(
7373
)
7474

7575

76+
class AsyncComposableIndexTemplate:
77+
def __init__(
78+
self,
79+
name: str,
80+
template: str,
81+
index: Optional["AsyncIndex"] = None,
82+
priority: Optional[int] = None,
83+
**kwargs: Any,
84+
):
85+
if index is None:
86+
self._index = AsyncIndex(template, **kwargs)
87+
else:
88+
if kwargs:
89+
raise ValueError(
90+
"You cannot specify options for Index when"
91+
" passing an Index instance."
92+
)
93+
self._index = index.clone()
94+
self._index._name = template
95+
self._template_name = name
96+
self.priority = priority
97+
98+
def __getattr__(self, attr_name: str) -> Any:
99+
return getattr(self._index, attr_name)
100+
101+
def to_dict(self) -> Dict[str, Any]:
102+
d: Dict[str, Any] = {"template": self._index.to_dict()}
103+
d["index_patterns"] = [self._index._name]
104+
if self.priority is not None:
105+
d["priority"] = self.priority
106+
return d
107+
108+
async def save(
109+
self, using: Optional[AsyncUsingType] = None
110+
) -> "ObjectApiResponse[Any]":
111+
es = get_connection(using or self._index._using)
112+
return await es.indices.put_index_template(
113+
name=self._template_name, **self.to_dict()
114+
)
115+
116+
76117
class AsyncIndex(IndexBase):
77118
_using: AsyncUsingType
78119

@@ -102,13 +143,20 @@ def as_template(
102143
pattern: Optional[str] = None,
103144
order: Optional[int] = None,
104145
) -> AsyncIndexTemplate:
105-
# TODO: should we allow pattern to be a top-level arg?
106-
# or maybe have an IndexPattern that allows for it and have
107-
# Document._index be that?
108146
return AsyncIndexTemplate(
109147
template_name, pattern or self._name, index=self, order=order
110148
)
111149

150+
def as_composable_template(
151+
self,
152+
template_name: str,
153+
pattern: Optional[str] = None,
154+
priority: Optional[int] = None,
155+
) -> AsyncComposableIndexTemplate:
156+
return AsyncComposableIndexTemplate(
157+
template_name, pattern or self._name, index=self, priority=priority
158+
)
159+
112160
async def load_mappings(self, using: Optional[AsyncUsingType] = None) -> None:
113161
await self.get_or_create_mapping().update_from_es(
114162
self._name, using=using or self._using

‎elasticsearch_dsl/_sync/index.py

+47-3
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,43 @@ def save(self, using: Optional[UsingType] = None) -> "ObjectApiResponse[Any]":
6969
return es.indices.put_template(name=self._template_name, body=self.to_dict())
7070

7171

72+
class ComposableIndexTemplate:
73+
def __init__(
74+
self,
75+
name: str,
76+
template: str,
77+
index: Optional["Index"] = None,
78+
priority: Optional[int] = None,
79+
**kwargs: Any,
80+
):
81+
if index is None:
82+
self._index = Index(template, **kwargs)
83+
else:
84+
if kwargs:
85+
raise ValueError(
86+
"You cannot specify options for Index when"
87+
" passing an Index instance."
88+
)
89+
self._index = index.clone()
90+
self._index._name = template
91+
self._template_name = name
92+
self.priority = priority
93+
94+
def __getattr__(self, attr_name: str) -> Any:
95+
return getattr(self._index, attr_name)
96+
97+
def to_dict(self) -> Dict[str, Any]:
98+
d: Dict[str, Any] = {"template": self._index.to_dict()}
99+
d["index_patterns"] = [self._index._name]
100+
if self.priority is not None:
101+
d["priority"] = self.priority
102+
return d
103+
104+
def save(self, using: Optional[UsingType] = None) -> "ObjectApiResponse[Any]":
105+
es = get_connection(using or self._index._using)
106+
return es.indices.put_index_template(name=self._template_name, **self.to_dict())
107+
108+
72109
class Index(IndexBase):
73110
_using: UsingType
74111

@@ -96,13 +133,20 @@ def as_template(
96133
pattern: Optional[str] = None,
97134
order: Optional[int] = None,
98135
) -> IndexTemplate:
99-
# TODO: should we allow pattern to be a top-level arg?
100-
# or maybe have an IndexPattern that allows for it and have
101-
# Document._index be that?
102136
return IndexTemplate(
103137
template_name, pattern or self._name, index=self, order=order
104138
)
105139

140+
def as_composable_template(
141+
self,
142+
template_name: str,
143+
pattern: Optional[str] = None,
144+
priority: Optional[int] = None,
145+
) -> ComposableIndexTemplate:
146+
return ComposableIndexTemplate(
147+
template_name, pattern or self._name, index=self, priority=priority
148+
)
149+
106150
def load_mappings(self, using: Optional[UsingType] = None) -> None:
107151
self.get_or_create_mapping().update_from_es(
108152
self._name, using=using or self._using

‎elasticsearch_dsl/index.py

+6-2
Original file line numberDiff line numberDiff line change
@@ -15,5 +15,9 @@
1515
# specific language governing permissions and limitations
1616
# under the License.
1717

18-
from ._async.index import AsyncIndex, AsyncIndexTemplate # noqa: F401
19-
from ._sync.index import Index, IndexTemplate # noqa: F401
18+
from ._async.index import ( # noqa: F401
19+
AsyncComposableIndexTemplate,
20+
AsyncIndex,
21+
AsyncIndexTemplate,
22+
)
23+
from ._sync.index import ComposableIndexTemplate, Index, IndexTemplate # noqa: F401

‎examples/alias_migration.py

+4-1
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@
4444

4545
ALIAS = "test-blog"
4646
PATTERN = ALIAS + "-*"
47+
PRIORITY = 100
4748

4849

4950
class BlogPost(Document):
@@ -81,7 +82,9 @@ def setup() -> None:
8182
deploy.
8283
"""
8384
# create an index template
84-
index_template = BlogPost._index.as_template(ALIAS, PATTERN)
85+
index_template = BlogPost._index.as_composable_template(
86+
ALIAS, PATTERN, priority=PRIORITY
87+
)
8588
# upload the template into elasticsearch
8689
# potentially overriding the one already there
8790
index_template.save()

‎examples/async/alias_migration.py

+4-1
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@
4545

4646
ALIAS = "test-blog"
4747
PATTERN = ALIAS + "-*"
48+
PRIORITY = 100
4849

4950

5051
class BlogPost(AsyncDocument):
@@ -82,7 +83,9 @@ async def setup() -> None:
8283
deploy.
8384
"""
8485
# create an index template
85-
index_template = BlogPost._index.as_template(ALIAS, PATTERN)
86+
index_template = BlogPost._index.as_composable_template(
87+
ALIAS, PATTERN, priority=PRIORITY
88+
)
8689
# upload the template into elasticsearch
8790
# potentially overriding the one already there
8891
await index_template.save()

‎examples/async/parent_child.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -226,7 +226,7 @@ async def save(self, **kwargs: Any) -> None: # type: ignore[override]
226226

227227
async def setup() -> None:
228228
"""Create an IndexTemplate and save it into elasticsearch."""
229-
index_template = Post._index.as_template("base")
229+
index_template = Post._index.as_composable_template("base", priority=100)
230230
await index_template.save()
231231

232232

‎examples/parent_child.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -225,7 +225,7 @@ def save(self, **kwargs: Any) -> None: # type: ignore[override]
225225

226226
def setup() -> None:
227227
"""Create an IndexTemplate and save it into elasticsearch."""
228-
index_template = Post._index.as_template("base")
228+
index_template = Post._index.as_composable_template("base", priority=100)
229229
index_template.save()
230230

231231

‎setup.cfg

+1
Original file line numberDiff line numberDiff line change
@@ -11,5 +11,6 @@ filterwarnings =
1111
error
1212
ignore:Legacy index templates are deprecated in favor of composable templates.:elasticsearch.exceptions.ElasticsearchWarning
1313
ignore:datetime.datetime.utcfromtimestamp\(\) is deprecated and scheduled for removal in a future version..*:DeprecationWarning
14+
default:enable_cleanup_closed ignored.*:DeprecationWarning
1415
markers =
1516
sync: mark a test as performing I/O without asyncio.

‎tests/conftest.py

+4
Original file line numberDiff line numberDiff line change
@@ -122,6 +122,7 @@ def teardown_method(self, _: Any) -> None:
122122
)
123123
self.client.indices.delete(index="*", expand_wildcards=expand_wildcards)
124124
self.client.indices.delete_template(name="*")
125+
self.client.indices.delete_index_template(name="*")
125126

126127
def es_version(self) -> Tuple[int, ...]:
127128
if not hasattr(self, "_es_version"):
@@ -172,6 +173,9 @@ def write_client(client: Elasticsearch) -> Generator[Elasticsearch, None, None]:
172173
for index_name in client.indices.get(index="test-*", expand_wildcards="all"):
173174
client.indices.delete(index=index_name)
174175
client.options(ignore_status=404).indices.delete_template(name="test-template")
176+
client.options(ignore_status=404).indices.delete_index_template(
177+
name="test-template"
178+
)
175179

176180

177181
@pytest_asyncio.fixture

‎tests/test_integration/_async/test_index.py

+26-1
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
from elasticsearch import AsyncElasticsearch
2020

2121
from elasticsearch_dsl import (
22+
AsyncComposableIndexTemplate,
2223
AsyncDocument,
2324
AsyncIndex,
2425
AsyncIndexTemplate,
@@ -35,7 +36,31 @@ class Post(AsyncDocument):
3536

3637
@pytest.mark.asyncio
3738
async def test_index_template_works(async_write_client: AsyncElasticsearch) -> None:
38-
it = AsyncIndexTemplate("test-template", "test-*")
39+
it = AsyncIndexTemplate("test-template", "test-legacy-*")
40+
it.document(Post)
41+
it.settings(number_of_replicas=0, number_of_shards=1)
42+
await it.save()
43+
44+
i = AsyncIndex("test-legacy-blog")
45+
await i.create()
46+
47+
assert {
48+
"test-legacy-blog": {
49+
"mappings": {
50+
"properties": {
51+
"title": {"type": "text", "analyzer": "my_analyzer"},
52+
"published_from": {"type": "date"},
53+
}
54+
}
55+
}
56+
} == await async_write_client.indices.get_mapping(index="test-legacy-blog")
57+
58+
59+
@pytest.mark.asyncio
60+
async def test_composable_index_template_works(
61+
async_write_client: AsyncElasticsearch,
62+
) -> None:
63+
it = AsyncComposableIndexTemplate("test-template", "test-*")
3964
it.document(Post)
4065
it.settings(number_of_replicas=0, number_of_shards=1)
4166
await it.save()

‎tests/test_integration/_sync/test_index.py

+34-2
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,15 @@
1818
import pytest
1919
from elasticsearch import Elasticsearch
2020

21-
from elasticsearch_dsl import Date, Document, Index, IndexTemplate, Text, analysis
21+
from elasticsearch_dsl import (
22+
ComposableIndexTemplate,
23+
Date,
24+
Document,
25+
Index,
26+
IndexTemplate,
27+
Text,
28+
analysis,
29+
)
2230

2331

2432
class Post(Document):
@@ -28,7 +36,31 @@ class Post(Document):
2836

2937
@pytest.mark.sync
3038
def test_index_template_works(write_client: Elasticsearch) -> None:
31-
it = IndexTemplate("test-template", "test-*")
39+
it = IndexTemplate("test-template", "test-legacy-*")
40+
it.document(Post)
41+
it.settings(number_of_replicas=0, number_of_shards=1)
42+
it.save()
43+
44+
i = Index("test-legacy-blog")
45+
i.create()
46+
47+
assert {
48+
"test-legacy-blog": {
49+
"mappings": {
50+
"properties": {
51+
"title": {"type": "text", "analyzer": "my_analyzer"},
52+
"published_from": {"type": "date"},
53+
}
54+
}
55+
}
56+
} == write_client.indices.get_mapping(index="test-legacy-blog")
57+
58+
59+
@pytest.mark.sync
60+
def test_composable_index_template_works(
61+
write_client: Elasticsearch,
62+
) -> None:
63+
it = ComposableIndexTemplate("test-template", "test-*")
3264
it.document(Post)
3365
it.settings(number_of_replicas=0, number_of_shards=1)
3466
it.save()

‎tests/test_integration/test_examples/_async/test_alias_migration.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ async def test_alias_migration(async_write_client: AsyncElasticsearch) -> None:
2828
await alias_migration.setup()
2929

3030
# verify that template, index, and alias has been set up
31-
assert await async_write_client.indices.exists_template(name=ALIAS)
31+
assert await async_write_client.indices.exists_index_template(name=ALIAS)
3232
assert await async_write_client.indices.exists(index=PATTERN)
3333
assert await async_write_client.indices.exists_alias(name=ALIAS)
3434

‎tests/test_integration/test_examples/_async/test_parent_child.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,7 @@
4545
@pytest_asyncio.fixture
4646
async def question(async_write_client: AsyncElasticsearch) -> Question:
4747
await setup()
48-
assert await async_write_client.indices.exists_template(name="base")
48+
assert await async_write_client.indices.exists_index_template(name="base")
4949

5050
# create a question object
5151
q = Question(

‎tests/test_integration/test_examples/_sync/test_alias_migration.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ def test_alias_migration(write_client: Elasticsearch) -> None:
2828
alias_migration.setup()
2929

3030
# verify that template, index, and alias has been set up
31-
assert write_client.indices.exists_template(name=ALIAS)
31+
assert write_client.indices.exists_index_template(name=ALIAS)
3232
assert write_client.indices.exists(index=PATTERN)
3333
assert write_client.indices.exists_alias(name=ALIAS)
3434

‎tests/test_integration/test_examples/_sync/test_parent_child.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@
4444
@pytest.fixture
4545
def question(write_client: Elasticsearch) -> Question:
4646
setup()
47-
assert write_client.indices.exists_template(name="base")
47+
assert write_client.indices.exists_index_template(name="base")
4848

4949
# create a question object
5050
q = Question(

0 commit comments

Comments
 (0)