Skip to content

Commit ceddfb1

Browse files
guilloadguilload
committed
Fix metastore backward compat tests and a bunch of other shit
1 parent d05dd38 commit ceddfb1

File tree

22 files changed

+91
-91
lines changed

22 files changed

+91
-91
lines changed

‎quickwit/quickwit-cli/src/source.rs

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -823,7 +823,7 @@ mod tests {
823823
.collect();
824824
let sources = vec![SourceConfig {
825825
source_id: "foo-source".to_string(),
826-
num_pipelines: NonZeroUsize::new(1).unwrap(),
826+
num_pipelines: NonZeroUsize::MIN,
827827
enabled: true,
828828
source_params: SourceParams::file_from_str("path/to/file").unwrap(),
829829
transform_config: None,
@@ -884,15 +884,15 @@ mod tests {
884884
let sources = [
885885
SourceConfig {
886886
source_id: "foo-source".to_string(),
887-
num_pipelines: NonZeroUsize::new(1).unwrap(),
887+
num_pipelines: NonZeroUsize::MIN,
888888
enabled: true,
889889
source_params: SourceParams::stdin(),
890890
transform_config: None,
891891
input_format: SourceInputFormat::Json,
892892
},
893893
SourceConfig {
894894
source_id: "bar-source".to_string(),
895-
num_pipelines: NonZeroUsize::new(1).unwrap(),
895+
num_pipelines: NonZeroUsize::MIN,
896896
enabled: true,
897897
source_params: SourceParams::stdin(),
898898
transform_config: None,

‎quickwit/quickwit-cli/src/tool.rs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -415,7 +415,7 @@ pub async fn local_ingest_docs_cli(args: LocalIngestDocsArgs) -> anyhow::Result<
415415
.map(|vrl_script| TransformConfig::new(vrl_script, None));
416416
let source_config = SourceConfig {
417417
source_id: CLI_SOURCE_ID.to_string(),
418-
num_pipelines: NonZeroUsize::new(1).expect("1 is always non-zero."),
418+
num_pipelines: NonZeroUsize::MIN,
419419
enabled: true,
420420
source_params,
421421
transform_config,
@@ -605,7 +605,7 @@ pub async fn merge_cli(args: MergeArgs) -> anyhow::Result<()> {
605605
index_id: args.index_id,
606606
source_config: SourceConfig {
607607
source_id: args.source_id,
608-
num_pipelines: NonZeroUsize::new(1).unwrap(),
608+
num_pipelines: NonZeroUsize::MIN,
609609
enabled: true,
610610
source_params: SourceParams::Vec(VecSourceParams::default()),
611611
transform_config: None,

‎quickwit/quickwit-config/resources/tests/index_config/hdfs-logs.toml

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -25,9 +25,6 @@ schedule = "daily"
2525
commit_timeout_secs = 61
2626
split_num_docs_target = 10_000_001
2727

28-
[ingest_settings]
29-
min_shards = 12
30-
3128
[indexing_settings.merge_policy]
3229
type = "stable_log"
3330
merge_factor = 9
@@ -37,5 +34,8 @@ maturation_period = "48 hours"
3734
[indexing_settings.resources]
3835
heap_size = "3G"
3936

37+
[ingest_settings]
38+
min_shards = 12
39+
4040
[search_settings]
4141
default_search_fields = [ "severity_text", "body" ]

‎quickwit/quickwit-config/src/index_config/mod.rs

Lines changed: 15 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -167,14 +167,14 @@ impl Default for IndexingSettings {
167167
#[serde(deny_unknown_fields)]
168168
pub struct IngestSettings {
169169
/// Configures the minimum number of shards to use for ingestion.
170-
#[schema(default = 1)]
170+
#[schema(default = 1, value_type = usize)]
171171
#[serde(default = "IngestSettings::default_min_shards")]
172172
pub min_shards: NonZeroUsize,
173173
}
174174

175175
impl IngestSettings {
176176
pub fn default_min_shards() -> NonZeroUsize {
177-
NonZeroUsize::new(1).unwrap()
177+
NonZeroUsize::MIN
178178
}
179179
}
180180

@@ -462,10 +462,6 @@ impl crate::TestableForRegression for IndexConfig {
462462
store_source: true,
463463
tokenizers: vec![tokenizer],
464464
};
465-
let retention_policy = Some(RetentionPolicy {
466-
retention_period: "90 days".to_string(),
467-
evaluation_schedule: "daily".to_string(),
468-
});
469465
let stable_log_config = StableLogMergePolicyConfig {
470466
merge_factor: 9,
471467
max_merge_factor: 11,
@@ -489,14 +485,18 @@ impl crate::TestableForRegression for IndexConfig {
489485
let search_settings = SearchSettings {
490486
default_search_fields: vec!["message".to_string()],
491487
};
488+
let retention_policy_opt = Some(RetentionPolicy {
489+
retention_period: "90 days".to_string(),
490+
evaluation_schedule: "daily".to_string(),
491+
});
492492
IndexConfig {
493493
index_id: "my-index".to_string(),
494494
index_uri: Uri::for_test("s3://quickwit-indexes/my-index"),
495495
doc_mapping,
496496
indexing_settings,
497497
ingest_settings,
498498
search_settings,
499-
retention_policy_opt: retention_policy,
499+
retention_policy_opt,
500500
}
501501
}
502502

@@ -674,14 +674,14 @@ mod tests {
674674
assert_eq!(index_config.doc_mapping.field_mappings.len(), 1);
675675
assert_eq!(index_config.doc_mapping.field_mappings[0].name, "body");
676676
assert!(!index_config.doc_mapping.store_source);
677-
assert_eq!(index_config.ingest_settings, IngestSettings::default());
678677
assert_eq!(index_config.indexing_settings, IndexingSettings::default());
679-
assert_eq!(
680-
index_config.search_settings,
681-
SearchSettings {
682-
default_search_fields: vec!["body".to_string()],
683-
}
684-
);
678+
assert_eq!(index_config.ingest_settings, IngestSettings::default());
679+
680+
let expected_search_settings = SearchSettings {
681+
default_search_fields: vec!["body".to_string()],
682+
};
683+
assert_eq!(index_config.search_settings, expected_search_settings);
684+
assert!(index_config.retention_policy_opt.is_none());
685685
}
686686
{
687687
let index_config_filepath = get_index_config_filepath("partial-hdfs-logs.yaml");
@@ -943,7 +943,7 @@ mod tests {
943943
#[test]
944944
fn test_ingest_settings_serde() {
945945
let ingest_settings = IngestSettings {
946-
min_shards: NonZeroUsize::new(1).unwrap(),
946+
min_shards: NonZeroUsize::MIN,
947947
};
948948
let ingest_settings_yaml = serde_yaml::to_string(&ingest_settings).unwrap();
949949
let ingest_settings_roundtrip: IngestSettings =

‎quickwit/quickwit-config/src/lib.rs

Lines changed: 18 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -98,35 +98,36 @@ pub fn disable_ingest_v1() -> bool {
9898

9999
#[derive(utoipa::OpenApi)]
100100
#[openapi(components(schemas(
101-
IndexingResources,
102-
IndexingSettings,
103-
SearchSettings,
104-
RetentionPolicy,
105-
MergePolicyConfig,
101+
ConstWriteAmplificationMergePolicyConfig,
106102
DocMapping,
107-
VersionedSourceConfig,
108-
SourceConfigV0_7,
109-
SourceConfigV0_8,
110-
VersionedIndexConfig,
111-
IndexConfigV0_8,
112-
VersionedIndexTemplate,
113-
IndexTemplateV0_8,
114-
SourceInputFormat,
115-
SourceParams,
116103
FileSourceMessageType,
117104
FileSourceNotification,
118105
FileSourceParamsForSerde,
119106
FileSourceSqs,
120-
PubSubSourceParams,
107+
IndexConfigV0_8,
108+
IndexingResources,
109+
IndexingSettings,
110+
IndexTemplateV0_8,
111+
IngestSettings,
121112
KafkaSourceParams,
122113
KinesisSourceParams,
123-
PulsarSourceParams,
114+
MergePolicyConfig,
115+
PubSubSourceParams,
124116
PulsarSourceAuth,
117+
PulsarSourceParams,
125118
RegionOrEndpoint,
126-
ConstWriteAmplificationMergePolicyConfig,
119+
RetentionPolicy,
120+
SearchSettings,
121+
SourceConfigV0_7,
122+
SourceConfigV0_8,
123+
SourceInputFormat,
124+
SourceParams,
127125
StableLogMergePolicyConfig,
128126
TransformConfig,
129127
VecSourceParams,
128+
VersionedIndexConfig,
129+
VersionedIndexTemplate,
130+
VersionedSourceConfig,
130131
VoidSourceParams,
131132
)))]
132133
/// Schema used for the OpenAPI generation which are apart of this crate.

‎quickwit/quickwit-config/src/node_config/mod.rs

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -194,8 +194,7 @@ impl IndexerConfig {
194194
}
195195

196196
pub fn default_merge_concurrency() -> NonZeroUsize {
197-
NonZeroUsize::new(quickwit_common::num_cpus() * 2 / 3)
198-
.unwrap_or(NonZeroUsize::new(1).unwrap())
197+
NonZeroUsize::new(quickwit_common::num_cpus() * 2 / 3).unwrap_or(NonZeroUsize::MIN)
199198
}
200199

201200
fn default_cpu_capacity() -> CpuCapacity {

‎quickwit/quickwit-control-plane/src/ingest/ingest_controller.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2642,7 +2642,7 @@ mod tests {
26422642
num_open_shards: 2,
26432643
..Default::default()
26442644
};
2645-
let min_shards = NonZeroUsize::new(1).unwrap();
2645+
let min_shards = NonZeroUsize::MIN;
26462646
let mut model = ControlPlaneModel::default();
26472647
let progress = Progress::default();
26482648

‎quickwit/quickwit-control-plane/src/ingest/scaling_arbiter.rs

Lines changed: 16 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -131,7 +131,7 @@ mod tests {
131131
avg_short_term_ingestion_rate: 0.0,
132132
avg_long_term_ingestion_rate: 0.0,
133133
},
134-
NonZeroUsize::new(1).unwrap()
134+
NonZeroUsize::MIN
135135
),
136136
None,
137137
);
@@ -142,7 +142,7 @@ mod tests {
142142
avg_short_term_ingestion_rate: 5.0,
143143
avg_long_term_ingestion_rate: 6.0,
144144
},
145-
NonZeroUsize::new(1).unwrap()
145+
NonZeroUsize::MIN
146146
),
147147
None
148148
);
@@ -153,7 +153,7 @@ mod tests {
153153
avg_short_term_ingestion_rate: 8.1,
154154
avg_long_term_ingestion_rate: 8.1,
155155
},
156-
NonZeroUsize::new(1).unwrap()
156+
NonZeroUsize::MIN
157157
),
158158
Some(ScalingMode::Up(1))
159159
);
@@ -164,7 +164,7 @@ mod tests {
164164
avg_short_term_ingestion_rate: 8.1,
165165
avg_long_term_ingestion_rate: 8.1,
166166
},
167-
NonZeroUsize::new(1).unwrap()
167+
NonZeroUsize::MIN
168168
),
169169
Some(ScalingMode::Up(1))
170170
);
@@ -175,7 +175,7 @@ mod tests {
175175
avg_short_term_ingestion_rate: 3.0,
176176
avg_long_term_ingestion_rate: 1.5,
177177
},
178-
NonZeroUsize::new(1).unwrap()
178+
NonZeroUsize::MIN
179179
),
180180
Some(ScalingMode::Down)
181181
);
@@ -186,7 +186,7 @@ mod tests {
186186
avg_short_term_ingestion_rate: 3.0,
187187
avg_long_term_ingestion_rate: 1.5,
188188
},
189-
NonZeroUsize::new(1).unwrap()
189+
NonZeroUsize::MIN
190190
),
191191
None,
192192
);
@@ -197,7 +197,7 @@ mod tests {
197197
avg_short_term_ingestion_rate: 8.0,
198198
avg_long_term_ingestion_rate: 3.0,
199199
},
200-
NonZeroUsize::new(1).unwrap()
200+
NonZeroUsize::MIN
201201
),
202202
None,
203203
);
@@ -215,7 +215,7 @@ mod tests {
215215
avg_short_term_ingestion_rate: 0.0,
216216
avg_long_term_ingestion_rate: 0.0,
217217
},
218-
NonZeroUsize::new(1).unwrap()
218+
NonZeroUsize::MIN
219219
),
220220
None,
221221
);
@@ -226,7 +226,7 @@ mod tests {
226226
avg_short_term_ingestion_rate: 5.0,
227227
avg_long_term_ingestion_rate: 6.0,
228228
},
229-
NonZeroUsize::new(1).unwrap()
229+
NonZeroUsize::MIN
230230
),
231231
None
232232
);
@@ -237,7 +237,7 @@ mod tests {
237237
avg_short_term_ingestion_rate: 8.1,
238238
avg_long_term_ingestion_rate: 8.1,
239239
},
240-
NonZeroUsize::new(1).unwrap()
240+
NonZeroUsize::MIN
241241
),
242242
Some(ScalingMode::Up(1))
243243
);
@@ -248,7 +248,7 @@ mod tests {
248248
avg_short_term_ingestion_rate: 8.1,
249249
avg_long_term_ingestion_rate: 8.1,
250250
},
251-
NonZeroUsize::new(1).unwrap()
251+
NonZeroUsize::MIN
252252
),
253253
Some(ScalingMode::Up(2))
254254
);
@@ -259,7 +259,7 @@ mod tests {
259259
avg_short_term_ingestion_rate: 3.0,
260260
avg_long_term_ingestion_rate: 1.5,
261261
},
262-
NonZeroUsize::new(1).unwrap()
262+
NonZeroUsize::MIN
263263
),
264264
Some(ScalingMode::Down)
265265
);
@@ -270,7 +270,7 @@ mod tests {
270270
avg_short_term_ingestion_rate: 3.0,
271271
avg_long_term_ingestion_rate: 1.5,
272272
},
273-
NonZeroUsize::new(1).unwrap()
273+
NonZeroUsize::MIN
274274
),
275275
None,
276276
);
@@ -281,7 +281,7 @@ mod tests {
281281
avg_short_term_ingestion_rate: 8.0,
282282
avg_long_term_ingestion_rate: 3.1,
283283
},
284-
NonZeroUsize::new(1).unwrap()
284+
NonZeroUsize::MIN
285285
),
286286
None,
287287
);
@@ -293,7 +293,7 @@ mod tests {
293293
avg_short_term_ingestion_rate: 8.1,
294294
avg_long_term_ingestion_rate: 5.,
295295
},
296-
NonZeroUsize::new(1).unwrap()
296+
NonZeroUsize::MIN
297297
),
298298
Some(ScalingMode::Up(1)),
299299
);
@@ -386,7 +386,7 @@ mod tests {
386386
avg_short_term_ingestion_rate: 0.0,
387387
avg_long_term_ingestion_rate: 0.0,
388388
};
389-
let min_shards = NonZeroUsize::new(1).unwrap();
389+
let min_shards = NonZeroUsize::MIN;
390390
let scaling_mode = scaling_arbiter.should_scale(shard_stats, min_shards);
391391
assert!(scaling_mode.is_none());
392392

‎quickwit/quickwit-indexing/src/source/file_source.rs

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -232,7 +232,7 @@ mod tests {
232232
};
233233
let source_config = SourceConfig {
234234
source_id: "test-file-source".to_string(),
235-
num_pipelines: NonZeroUsize::new(1).unwrap(),
235+
num_pipelines: NonZeroUsize::MIN,
236236
enabled: true,
237237
source_params: SourceParams::File(params.clone()),
238238
transform_config: None,
@@ -284,7 +284,7 @@ mod tests {
284284
let params = FileSourceParams::Filepath(uri.clone());
285285
let source_config = SourceConfig {
286286
source_id: "test-file-source".to_string(),
287-
num_pipelines: NonZeroUsize::new(1).unwrap(),
287+
num_pipelines: NonZeroUsize::MIN,
288288
enabled: true,
289289
source_params: SourceParams::File(params.clone()),
290290
transform_config: None,
@@ -348,7 +348,7 @@ mod tests {
348348
let params = FileSourceParams::Filepath(uri.clone());
349349
let source_config = SourceConfig {
350350
source_id: "test-file-source".to_string(),
351-
num_pipelines: NonZeroUsize::new(1).unwrap(),
351+
num_pipelines: NonZeroUsize::MIN,
352352
enabled: true,
353353
source_params: SourceParams::File(params.clone()),
354354
transform_config: None,

‎quickwit/quickwit-indexing/src/source/gcp_pubsub_source.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -306,7 +306,7 @@ mod gcp_pubsub_emulator_tests {
306306
let source_id = append_random_suffix("test-gcp-pubsub-source--source");
307307
SourceConfig {
308308
source_id,
309-
num_pipelines: NonZeroUsize::new(1).unwrap(),
309+
num_pipelines: NonZeroUsize::MIN,
310310
enabled: true,
311311
source_params: SourceParams::PubSub(PubSubSourceParams {
312312
project_id: Some(GCP_TEST_PROJECT.to_string()),

0 commit comments

Comments
 (0)