Skip to content

Commit 28ca9ba

Browse files
authored
Fix a bug: incorrect outputFields of query/search with milvus v2.5.8 (#1355)
Signed-off-by: yhmo <[email protected]>
1 parent 0c2bc34 commit 28ca9ba

File tree

8 files changed

+97
-73
lines changed

8 files changed

+97
-73
lines changed

examples/src/main/java/io/milvus/v1/BulkWriterExample.java

+3-32
Original file line numberDiff line numberDiff line change
@@ -79,6 +79,7 @@
7979
import java.util.ArrayList;
8080
import java.util.Iterator;
8181
import java.util.List;
82+
import java.util.Map;
8283
import java.util.concurrent.TimeUnit;
8384

8485

@@ -647,38 +648,8 @@ private void retrieveImportData() {
647648
List<QueryResultsWrapper.RowRecord> rowRecords = query(expr, Lists.newArrayList("*"));
648649
System.out.println("Query results:");
649650
for (QueryResultsWrapper.RowRecord record : rowRecords) {
650-
JsonObject rowObject = new JsonObject();
651-
// scalar field
652-
rowObject.addProperty("id", (Long)record.get("id"));
653-
rowObject.addProperty("bool", (Boolean) record.get("bool"));
654-
rowObject.addProperty("int8", (Integer) record.get("int8"));
655-
rowObject.addProperty("int16", (Integer) record.get("int16"));
656-
rowObject.addProperty("int32", (Integer) record.get("int32"));
657-
rowObject.addProperty("float", (Float) record.get("float"));
658-
rowObject.addProperty("double", (Double) record.get("double"));
659-
rowObject.addProperty("varchar", (String) record.get("varchar"));
660-
rowObject.add("json", (JsonElement) record.get("json"));
661-
662-
// vector field
663-
rowObject.add("float_vector", GSON_INSTANCE.toJsonTree(record.get("float_vector")));
664-
rowObject.add("binary_vector", GSON_INSTANCE.toJsonTree(((ByteBuffer)record.get("binary_vector")).array()));
665-
rowObject.add("float16_vector", GSON_INSTANCE.toJsonTree(((ByteBuffer)record.get("float16_vector")).array()));
666-
rowObject.add("sparse_vector", GSON_INSTANCE.toJsonTree(record.get("sparse_vector")));
667-
668-
// array field
669-
rowObject.add("array_bool", GSON_INSTANCE.toJsonTree(record.get("array_bool")));
670-
rowObject.add("array_int8", GSON_INSTANCE.toJsonTree(record.get("array_int8")));
671-
rowObject.add("array_int16", GSON_INSTANCE.toJsonTree(record.get("array_int16")));
672-
rowObject.add("array_int32", GSON_INSTANCE.toJsonTree(record.get("array_int32")));
673-
rowObject.add("array_int64", GSON_INSTANCE.toJsonTree(record.get("array_int64")));
674-
rowObject.add("array_varchar", GSON_INSTANCE.toJsonTree(record.get("array_varchar")));
675-
rowObject.add("array_float", GSON_INSTANCE.toJsonTree(record.get("array_float")));
676-
rowObject.add("array_double", GSON_INSTANCE.toJsonTree(record.get("array_double")));
677-
678-
// dynamic field
679-
rowObject.addProperty("dynamic", (String) record.get("dynamic"));
680-
681-
System.out.println(rowObject);
651+
Map<String, Object> fieldValues = record.getFieldValues();
652+
System.out.println(fieldValues);
682653
}
683654
}
684655

examples/src/main/java/io/milvus/v1/JsonFieldExample.java

+11-4
Original file line numberDiff line numberDiff line change
@@ -48,8 +48,7 @@ private static void queryWithExpr(MilvusClient client, String expr) {
4848
R<QueryResults> queryRet = client.query(QueryParam.newBuilder()
4949
.withCollectionName(COLLECTION_NAME)
5050
.withExpr(expr)
51-
.addOutField(ID_FIELD)
52-
.addOutField(JSON_FIELD)
51+
.withOutFields(Arrays.asList(ID_FIELD, JSON_FIELD, "dynamic1", "dynamic2"))
5352
.build());
5453
QueryResultsWrapper queryWrapper = new QueryResultsWrapper(queryRet.getData());
5554
System.out.println("\nQuery with expression: " + expr);
@@ -87,7 +86,7 @@ public static void main(String[] args) {
8786
);
8887

8988
CollectionSchemaParam collectionSchemaParam = CollectionSchemaParam.newBuilder()
90-
.withEnableDynamicField(false)
89+
.withEnableDynamicField(true)
9190
.withFieldTypes(fieldsSchema)
9291
.build();
9392

@@ -140,7 +139,14 @@ public static void main(String[] args) {
140139
}
141140
metadata.add("flags", gson.toJsonTree(Arrays.asList(i, i + 1, i + 2)));
142141
row.add(JSON_FIELD, metadata);
143-
System.out.println(metadata);
142+
// System.out.println(metadata);
143+
144+
// dynamic fields
145+
if (i%2 == 0) {
146+
row.addProperty("dynamic1", (double)i/3);
147+
} else {
148+
row.addProperty("dynamic2", "ok");
149+
}
144150

145151
client.insert(InsertParam.newBuilder()
146152
.withCollectionName(COLLECTION_NAME)
@@ -166,5 +172,6 @@ public static void main(String[] args) {
166172
queryWithExpr(client, "JSON_CONTAINS(metadata[\"flags\"], 9)");
167173
queryWithExpr(client, "JSON_CONTAINS_ANY(metadata[\"flags\"], [8, 9, 10])");
168174
queryWithExpr(client, "JSON_CONTAINS_ALL(metadata[\"flags\"], [8, 9, 10])");
175+
queryWithExpr(client, "dynamic1 < 2.0");
169176
}
170177
}

examples/src/main/java/io/milvus/v2/BulkWriterExample.java

+2
Original file line numberDiff line numberDiff line change
@@ -786,6 +786,8 @@ private static void verifyImportData(CreateCollectionReq.CollectionSchema collec
786786
comparePrint(collectionSchema, originalEntity, fetchedEntity, "binary_vector");
787787
comparePrint(collectionSchema, originalEntity, fetchedEntity, "float16_vector");
788788
comparePrint(collectionSchema, originalEntity, fetchedEntity, "sparse_vector");
789+
790+
System.out.println(fetchedEntity);
789791
}
790792
System.out.println("Result is correct!");
791793
}

examples/src/main/java/io/milvus/v2/JsonFieldExample.java

+11-2
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@ private static void queryWithExpr(MilvusClientV2 client, String expr) {
4747
QueryResp queryRet = client.query(QueryReq.builder()
4848
.collectionName(COLLECTION_NAME)
4949
.filter(expr)
50-
.outputFields(Arrays.asList(ID_FIELD, JSON_FIELD))
50+
.outputFields(Arrays.asList(ID_FIELD, JSON_FIELD, "dynamic1", "dynamic2"))
5151
.build());
5252
System.out.println("\nQuery with expression: " + expr);
5353
List<QueryResp.QueryResult> records = queryRet.getQueryResults();
@@ -70,6 +70,7 @@ public static void main(String[] args) {
7070

7171
// Create collection
7272
CreateCollectionReq.CollectionSchema collectionSchema = CreateCollectionReq.CollectionSchema.builder()
73+
.enableDynamicField(true)
7374
.build();
7475
collectionSchema.addField(AddFieldReq.builder()
7576
.fieldName(ID_FIELD)
@@ -119,7 +120,14 @@ public static void main(String[] args) {
119120
}
120121
metadata.add("flags", gson.toJsonTree(Arrays.asList(i, i + 1, i + 2)));
121122
row.add(JSON_FIELD, metadata);
122-
System.out.println(metadata);
123+
// System.out.println(metadata);
124+
125+
// dynamic fields
126+
if (i%2 == 0) {
127+
row.addProperty("dynamic1", (double)i/3);
128+
} else {
129+
row.addProperty("dynamic2", "ok");
130+
}
123131

124132
client.insert(InsertReq.builder()
125133
.collectionName(COLLECTION_NAME)
@@ -143,6 +151,7 @@ public static void main(String[] args) {
143151
queryWithExpr(client, "JSON_CONTAINS(metadata[\"flags\"], 9)");
144152
queryWithExpr(client, "JSON_CONTAINS_ANY(metadata[\"flags\"], [8, 9, 10])");
145153
queryWithExpr(client, "JSON_CONTAINS_ALL(metadata[\"flags\"], [8, 9, 10])");
154+
queryWithExpr(client, "dynamic1 < 2.0");
146155

147156
client.close();
148157
}

examples/src/main/java/io/milvus/v2/TextMatchExample.java

+4
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
import io.milvus.v2.service.collection.request.AddFieldReq;
1212
import io.milvus.v2.service.collection.request.CreateCollectionReq;
1313
import io.milvus.v2.service.collection.request.DropCollectionReq;
14+
import io.milvus.v2.service.utility.request.FlushReq;
1415
import io.milvus.v2.service.vector.request.InsertReq;
1516
import io.milvus.v2.service.vector.request.QueryReq;
1617
import io.milvus.v2.service.vector.request.SearchReq;
@@ -145,6 +146,9 @@ public static void main(String[] args) {
145146
.build());
146147
System.out.printf("%d rows in collection\n", (long)countR.getQueryResults().get(0).getEntity().get("count(*)"));
147148

149+
// TEXT_MATCH requires the data is persisted
150+
client.flush(FlushReq.builder().collectionNames(Collections.singletonList(COLLECTION_NAME)).build());
151+
148152
// Query by keyword filtering expression
149153
queryWithFilter(client, "TEXT_MATCH(text, \"distance\")");
150154
queryWithFilter(client, "TEXT_MATCH(text, \"Milvus\") or TEXT_MATCH(text, \"distance\")");

sdk-core/src/main/java/io/milvus/response/basic/RowRecordWrapper.java

+63-32
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,8 @@
3232
public abstract class RowRecordWrapper {
3333
// a cache for output fields
3434
private ConcurrentHashMap<String, FieldDataWrapper> outputFieldsData = new ConcurrentHashMap<>();
35+
// a cache for output dynamic field names
36+
private List<String> dynamicFieldNames = null;
3537

3638
public abstract List<QueryResultsWrapper.RowRecord> getRowRecords();
3739

@@ -69,46 +71,75 @@ public FieldDataWrapper getDynamicWrapper() throws ParamException {
6971
* @return <code>RowRecord</code> a row record of the result
7072
*/
7173
protected QueryResultsWrapper.RowRecord buildRowRecord(QueryResultsWrapper.RowRecord record, long index) {
72-
for (String outputKey : getOutputFields()) {
73-
boolean isField = false;
74-
for (FieldData field : getFieldDataList()) {
75-
if (outputKey.equals(field.getFieldName())) {
76-
FieldDataWrapper wrapper = getFieldWrapperInternal(field);
77-
if (index < 0 || index >= wrapper.getRowCount()) {
78-
throw new ParamException("Index out of range");
79-
}
80-
Object value = wrapper.valueByIdx((int)index);
81-
if (wrapper.isJsonField()) {
82-
JsonElement jsonField = FieldDataWrapper.ParseJSONObject(value);
83-
if (wrapper.isDynamicField() && jsonField instanceof JsonObject) {
84-
JsonObject jsonObj = (JsonObject) jsonField;
85-
for (String key: jsonObj.keySet()) {
86-
record.put(key, FieldDataWrapper.ValueOfJSONElement(jsonObj.get(key)));
87-
}
88-
} else {
89-
record.put(field.getFieldName(), jsonField);
90-
}
91-
} else {
92-
record.put(field.getFieldName(), value);
93-
}
94-
isField = true;
95-
break;
96-
}
74+
List<String> dynamicFields = getDynamicFieldNames();
75+
List<FieldData> fieldsData = getFieldDataList();
76+
for (FieldData field : fieldsData) {
77+
FieldDataWrapper wrapper = getFieldWrapperInternal(field);
78+
if (index < 0 || index >= wrapper.getRowCount()) {
79+
throw new ParamException("Index out of range");
9780
}
81+
Object value = wrapper.valueByIdx((int)index);
82+
if (wrapper.isJsonField()) {
83+
JsonElement jsonValue = FieldDataWrapper.ParseJSONObject(value);
84+
if (!field.getIsDynamic()) {
85+
record.put(field.getFieldName(), jsonValue);
86+
continue;
87+
}
9888

99-
// if the output field is not a field name, fetch it from dynamic field
100-
if (!isField) {
101-
FieldDataWrapper dynamicField = getDynamicWrapper();
102-
Object obj = dynamicField.get((int)index, outputKey);
103-
if (obj != null) {
104-
record.put(outputKey, obj);
89+
// dynamic field, the value must be a dict
90+
if (!(jsonValue instanceof JsonObject)) {
91+
throw new ParamException("The content of dynamic field is not a JSON dict");
10592
}
93+
94+
JsonObject jsonDict = (JsonObject)jsonValue;
95+
// the outputFields of QueryRequest/SearchRequest contains a "$meta"
96+
// put all key/value pairs of "$meta" into record
97+
// else pick some key/value pairs according to the dynamicFields
98+
for (String key: jsonDict.keySet()) {
99+
if (dynamicFields.isEmpty() || dynamicFields.contains(key)) {
100+
record.put(key, FieldDataWrapper.ValueOfJSONElement(jsonDict.get(key)));
101+
}
102+
}
103+
} else {
104+
record.put(field.getFieldName(), value);
106105
}
107106
}
107+
108108
return record;
109109
}
110110

111+
private List<String> getDynamicFieldNames() {
112+
if (dynamicFieldNames != null) {
113+
return dynamicFieldNames;
114+
}
115+
116+
dynamicFieldNames = new ArrayList<>();
117+
// find out dynamic field names
118+
List<FieldData> fieldsData = getFieldDataList();
119+
String dynamicFieldName = null;
120+
List<String> fieldNames = new ArrayList<>();
121+
for (FieldData field : fieldsData) {
122+
if (!fieldNames.contains(field.getFieldName())) {
123+
fieldNames.add(field.getFieldName());
124+
}
125+
if (field.getIsDynamic()) {
126+
dynamicFieldName = field.getFieldName();
127+
}
128+
}
129+
130+
List<String> outputNames = getOutputFields();
131+
for (String name : outputNames) {
132+
if (name.equals(dynamicFieldName)) {
133+
dynamicFieldNames.clear();
134+
break;
135+
}
136+
if (!fieldNames.contains(name)) {
137+
dynamicFieldNames.add(name);
138+
}
139+
}
140+
return dynamicFieldNames;
141+
}
142+
111143
protected abstract List<FieldData> getFieldDataList();
112144
protected abstract List<String> getOutputFields();
113-
114145
}

sdk-core/src/test/java/io/milvus/client/MilvusClientDockerTest.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -78,7 +78,7 @@ class MilvusClientDockerTest {
7878
private static final TestUtils utils = new TestUtils(DIMENSION);
7979

8080
@Container
81-
private static final MilvusContainer milvus = new MilvusContainer("milvusdb/milvus:v2.5.7");
81+
private static final MilvusContainer milvus = new MilvusContainer("milvusdb/milvus:v2.5.8");
8282

8383
@BeforeAll
8484
public static void setUp() {

sdk-core/src/test/java/io/milvus/v2/client/MilvusClientV2DockerTest.java

+2-2
Original file line numberDiff line numberDiff line change
@@ -80,7 +80,7 @@ class MilvusClientV2DockerTest {
8080
private static final TestUtils utils = new TestUtils(DIMENSION);
8181

8282
@Container
83-
private static final MilvusContainer milvus = new MilvusContainer("milvusdb/milvus:v2.5.7");
83+
private static final MilvusContainer milvus = new MilvusContainer("milvusdb/milvus:v2.5.8");
8484

8585
@BeforeAll
8686
public static void setUp() {
@@ -1932,7 +1932,7 @@ void testNullableAndDefaultValue() {
19321932
QueryResp queryResp = client.query(QueryReq.builder()
19331933
.collectionName(randomCollectionName)
19341934
.filter("id >= 0")
1935-
.outputFields(Lists.newArrayList("*"))
1935+
.outputFields(Arrays.asList("desc", "flag"))
19361936
.consistencyLevel(ConsistencyLevel.STRONG)
19371937
.build());
19381938
List<QueryResp.QueryResult> queryResults = queryResp.getQueryResults();

0 commit comments

Comments
 (0)