Skip to content

Commit 6091478

Browse files
hakbaileyhakbailey
committed
Add bulk-delete command
Why these changes are being introduced: In order to handle deleted records from TIMDEX sources, we need to be able to bulk delete them from a file. How this addresses that need: * Adds bulk_delete function to the opensearch module. * Adds bulk-delete cli command. * Updates helpers module to handle delete action when generating bulk actions. * Adds helper function to parse deleted records files, which are structured differently than ingest records files. * Refactors bulk cli options validation out of the cli module and into the helpers module so it can be used by both bulk cli commands. * Adds/updates fixtures and tests to reflect changes. Relevant ticket(s): * https://mitlibraries.atlassian.net/browse/TIMX-143
1 parent 2308c63 commit 6091478

15 files changed

+682
-69
lines changed

‎tests/fixtures/cassettes/cli/bulk_delete_with_index_name_success.yaml

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
interactions:
2+
- request:
3+
body: null
4+
headers:
5+
content-type:
6+
- application/json
7+
user-agent:
8+
- opensearch-py/2.0.1 (Python 3.10.8)
9+
method: HEAD
10+
uri: http://localhost:9200/alma-2022-09-01t00-00-00
11+
response:
12+
body:
13+
string: ''
14+
headers:
15+
content-length:
16+
- '5009'
17+
content-type:
18+
- application/json; charset=UTF-8
19+
status:
20+
code: 200
21+
message: OK
22+
- request:
23+
body: '{"delete":{"_id":"mit:alma:990026671500206761","_index":"alma-2022-09-01t00-00-00"}}
24+
25+
{"delete":{"_id":"mit:alma:990027672770206761","_index":"alma-2022-09-01t00-00-00"}}
26+
27+
{"delete":{"_id":"mit:alma:9933052979806761","_index":"alma-2022-09-01t00-00-00"}}
28+
29+
'
30+
headers:
31+
Content-Length:
32+
- '253'
33+
content-type:
34+
- application/json
35+
user-agent:
36+
- opensearch-py/2.0.1 (Python 3.10.8)
37+
method: POST
38+
uri: http://localhost:9200/_bulk
39+
response:
40+
body:
41+
string: '{"took":29,"errors":false,"items":[{"delete":{"_index":"alma-2022-09-01t00-00-00","_type":"_doc","_id":"mit:alma:990026671500206761","_version":2,"result":"deleted","_shards":{"total":2,"successful":1,"failed":0},"_seq_no":6,"_primary_term":1,"status":200}},{"delete":{"_index":"alma-2022-09-01t00-00-00","_type":"_doc","_id":"mit:alma:990027672770206761","_version":2,"result":"deleted","_shards":{"total":2,"successful":1,"failed":0},"_seq_no":7,"_primary_term":1,"status":200}},{"delete":{"_index":"alma-2022-09-01t00-00-00","_type":"_doc","_id":"mit:alma:9933052979806761","_version":2,"result":"deleted","_shards":{"total":2,"successful":1,"failed":0},"_seq_no":8,"_primary_term":1,"status":200}}]}'
42+
headers:
43+
content-length:
44+
- '703'
45+
content-type:
46+
- application/json; charset=UTF-8
47+
status:
48+
code: 200
49+
message: OK
50+
- request:
51+
body: null
52+
headers:
53+
Content-Length:
54+
- '0'
55+
content-type:
56+
- application/json
57+
user-agent:
58+
- opensearch-py/2.0.1 (Python 3.10.8)
59+
method: POST
60+
uri: http://localhost:9200/alma-2022-09-01t00-00-00/_refresh
61+
response:
62+
body:
63+
string: '{"_shards":{"total":2,"successful":1,"failed":0}}'
64+
headers:
65+
content-length:
66+
- '49'
67+
content-type:
68+
- application/json; charset=UTF-8
69+
status:
70+
code: 200
71+
message: OK
72+
version: 1

‎tests/fixtures/cassettes/cli/bulk_delete_with_source_success.yaml

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
interactions:
2+
- request:
3+
body: null
4+
headers:
5+
content-type:
6+
- application/json
7+
user-agent:
8+
- opensearch-py/2.0.1 (Python 3.10.8)
9+
method: GET
10+
uri: http://localhost:9200/_cat/aliases?format=json
11+
response:
12+
body:
13+
string: '[{"alias":"all-current","index":"alma-2022-09-01t00-00-00","filter":"-","routing.index":"-","routing.search":"-","is_write_index":"-"}]'
14+
headers:
15+
content-length:
16+
- '135'
17+
content-type:
18+
- application/json; charset=UTF-8
19+
status:
20+
code: 200
21+
message: OK
22+
- request:
23+
body: '{"delete":{"_id":"mit:alma:990026671500206761","_index":"alma-2022-09-01t00-00-00"}}
24+
25+
{"delete":{"_id":"mit:alma:990027672770206761","_index":"alma-2022-09-01t00-00-00"}}
26+
27+
{"delete":{"_id":"mit:alma:9933052979806761","_index":"alma-2022-09-01t00-00-00"}}
28+
29+
'
30+
headers:
31+
Content-Length:
32+
- '253'
33+
content-type:
34+
- application/json
35+
user-agent:
36+
- opensearch-py/2.0.1 (Python 3.10.8)
37+
method: POST
38+
uri: http://localhost:9200/_bulk
39+
response:
40+
body:
41+
string: '{"took":92,"errors":false,"items":[{"delete":{"_index":"alma-2022-09-01t00-00-00","_type":"_doc","_id":"mit:alma:990026671500206761","_version":2,"result":"deleted","_shards":{"total":2,"successful":1,"failed":0},"_seq_no":15,"_primary_term":1,"status":200}},{"delete":{"_index":"alma-2022-09-01t00-00-00","_type":"_doc","_id":"mit:alma:990027672770206761","_version":2,"result":"deleted","_shards":{"total":2,"successful":1,"failed":0},"_seq_no":16,"_primary_term":1,"status":200}},{"delete":{"_index":"alma-2022-09-01t00-00-00","_type":"_doc","_id":"mit:alma:9933052979806761","_version":2,"result":"deleted","_shards":{"total":2,"successful":1,"failed":0},"_seq_no":17,"_primary_term":1,"status":200}}]}'
42+
headers:
43+
content-length:
44+
- '706'
45+
content-type:
46+
- application/json; charset=UTF-8
47+
status:
48+
code: 200
49+
message: OK
50+
- request:
51+
body: null
52+
headers:
53+
Content-Length:
54+
- '0'
55+
content-type:
56+
- application/json
57+
user-agent:
58+
- opensearch-py/2.0.1 (Python 3.10.8)
59+
method: POST
60+
uri: http://localhost:9200/alma-2022-09-01t00-00-00/_refresh
61+
response:
62+
body:
63+
string: '{"_shards":{"total":2,"successful":1,"failed":0}}'
64+
headers:
65+
content-length:
66+
- '49'
67+
content-type:
68+
- application/json; charset=UTF-8
69+
status:
70+
code: 200
71+
message: OK
72+
version: 1

‎tests/fixtures/cassettes/cli/bulk_index_no_primary_index_for_source.yaml renamed to ‎tests/fixtures/cassettes/helpers/bulk_cli_no_primary_index_for_source.yaml

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,15 +5,15 @@ interactions:
55
content-type:
66
- application/json
77
user-agent:
8-
- opensearch-py/2.0.0 (Python 3.10.6)
8+
- opensearch-py/2.0.1 (Python 3.10.8)
99
method: GET
1010
uri: http://localhost:9200/_cat/aliases?format=json
1111
response:
1212
body:
13-
string: '[]'
13+
string: '[{"alias":"all-current","index":"alma-2022-09-01t00-00-00","filter":"-","routing.index":"-","routing.search":"-","is_write_index":"-"}]'
1414
headers:
1515
content-length:
16-
- '2'
16+
- '135'
1717
content-type:
1818
- application/json; charset=UTF-8
1919
status:

‎tests/fixtures/cassettes/cli/bulk_index_nonexistent_index.yaml renamed to ‎tests/fixtures/cassettes/helpers/bulk_cli_nonexistent_index.yaml

File renamed without changes.

‎tests/fixtures/cassettes/opensearch/bulk_delete_record_not_found.yaml

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
interactions:
2+
- request:
3+
body: '{"delete":{"_id":"no-record-here","_index":"alma-2022-09-01t00-00-00"}}
4+
5+
'
6+
headers:
7+
Content-Length:
8+
- '72'
9+
content-type:
10+
- application/json
11+
user-agent:
12+
- opensearch-py/2.0.1 (Python 3.10.8)
13+
method: POST
14+
uri: http://localhost:9200/_bulk
15+
response:
16+
body:
17+
string: '{"took":5,"errors":false,"items":[{"delete":{"_index":"alma-2022-09-01t00-00-00","_type":"_doc","_id":"no-record-here","_version":1,"result":"not_found","_shards":{"total":2,"successful":1,"failed":0},"_seq_no":27,"_primary_term":1,"status":404}}]}'
18+
headers:
19+
content-length:
20+
- '248'
21+
content-type:
22+
- application/json; charset=UTF-8
23+
status:
24+
code: 200
25+
message: OK
26+
- request:
27+
body: null
28+
headers:
29+
Content-Length:
30+
- '0'
31+
content-type:
32+
- application/json
33+
user-agent:
34+
- opensearch-py/2.0.1 (Python 3.10.8)
35+
method: POST
36+
uri: http://localhost:9200/alma-2022-09-01t00-00-00/_refresh
37+
response:
38+
body:
39+
string: '{"_shards":{"total":2,"successful":1,"failed":0}}'
40+
headers:
41+
content-length:
42+
- '49'
43+
content-type:
44+
- application/json; charset=UTF-8
45+
status:
46+
code: 200
47+
message: OK
48+
version: 1

‎tests/fixtures/cassettes/opensearch/bulk_delete_record_weird_response.yaml

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
interactions:
2+
- request:
3+
body:
4+
'{"delete":{"_id":"no-record-here","_index":"alma-2022-09-01t00-00-00"}}
5+
6+
'
7+
headers:
8+
Content-Length:
9+
- "72"
10+
content-type:
11+
- application/json
12+
user-agent:
13+
- opensearch-py/2.0.1 (Python 3.10.8)
14+
method: POST
15+
uri: http://localhost:9200/_bulk
16+
response:
17+
body:
18+
string: '{"took":5,"errors":true,"items":[{"delete":{"_index":"alma-2022-09-01t00-00-00","_type":"_doc","_id":"no-record-here","_version":1,"result":"surprise!","_shards":{"total":2,"successful":1,"failed":0},"_seq_no":27,"_primary_term":1,"status":404}}]}'
19+
headers:
20+
content-length:
21+
- "249"
22+
content-type:
23+
- application/json; charset=UTF-8
24+
status:
25+
code: 200
26+
message: OK
27+
- request:
28+
body: null
29+
headers:
30+
Content-Length:
31+
- "0"
32+
content-type:
33+
- application/json
34+
user-agent:
35+
- opensearch-py/2.0.1 (Python 3.10.8)
36+
method: POST
37+
uri: http://localhost:9200/alma-2022-09-01t00-00-00/_refresh
38+
response:
39+
body:
40+
string: '{"_shards":{"total":2,"successful":1,"failed":0}}'
41+
headers:
42+
content-length:
43+
- "49"
44+
content-type:
45+
- application/json; charset=UTF-8
46+
status:
47+
code: 200
48+
message: OK
49+
version: 1

0 commit comments

Comments
 (0)