Skip to content

Commit eac93be

Browse files
asya999asya999
committed
queries to check consistency, atomicity, etc.
1 parent 5d354b2 commit eac93be

File tree

1 file changed

+69
-49
lines changed

1 file changed

+69
-49
lines changed

‎verify.js

Lines changed: 69 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,27 @@
1-
function checkConsistency(rule, collection, agg) {
2-
var r = db.getCollection(collection).aggregate(agg).toArray();
1+
function checkConsistency(rule, collection, agg, sample=false) {
2+
agg.push({$count:"c"});
3+
if (sample) {
4+
var c=db.WAREHOUSE.count();
5+
var start=Math.round(Math.random()*c);
6+
var firstStage={$match:{}};
7+
var wid="W_ID";
8+
var did="D_ID";
9+
if (collection=="NEW_ORDER") {
10+
wid="NO_"+wid;
11+
did="NO_"+did;
12+
} else if (collection=="WAREHOUSE") {
13+
did=collection[0]+"_"+did;
14+
} else if (collection=="DISTRICT") {
15+
wid=collection[0]+"_"+wid;
16+
} else {
17+
wid=collection[0]+"_"+wid;
18+
did=collection[0]+"_"+did;
19+
}
20+
firstStage["$match"][wid]=start;
21+
/* firstStage["$match"][did]=(start%10)+1; */
22+
agg.unshift(firstStage);
23+
}
24+
var r = db.getCollection(collection).aggregate(agg, {maxTimeMS:60000}).toArray();
325
if (r.length != 0) {
426
print("Consistency rule " + rule + " failed with count " + r[0]["c"]);
527
}
@@ -8,117 +30,115 @@ function checkConsistency(rule, collection, agg) {
830
c=[];
931
c[1]={r:"1", c:"WAREHOUSE", pipeline: [
1032
{$lookup:{
11-
from:"DISTRICT",
12-
as:"d",
13-
localField:"W_ID",
33+
from:"DISTRICT",
34+
as:"d",
35+
localField:"W_ID",
1436
foreignField:"D_W_ID"}},
1537
{$match:{$expr:{$ne:[0,{$toLong:{$subtract:["$W_YTD",{$sum:"$d.D_YTD"}]}}]}}},
16-
{$count:"c"}
1738
]};
1839
// consistency check 2
1940
c[2]={r:"2", c:"DISTRICT", pipeline: [
2041
{$project:{w:"$D_W_ID", d:"$D_ID",next:"$D_NEXT_O_ID"}},
2142
{$lookup:{from:"ORDERS",as:"maxOID",let:{w:"$w",d:"$d"},pipeline:[
2243
{$match:{$expr:{$and:[{$eq:["$$w","$O_W_ID"]},{$eq:["$$d","$O_D_ID"]}]}}},
23-
{$group:{_id:0,maxO:{$max:"$O_ID"}}}]}},
24-
{$unwind:"$maxOID"},
44+
{$sort:{"O_ID":-1}},
45+
{$limit:1},
46+
{$group:{_id:0,maxO:{$first:"$O_ID"}}}]}},
47+
{$unwind:"$maxOID"},
2548
{$lookup:{from:"NEW_ORDER",as:"maxNOID",let:{w:"$w",d:"$d"},pipeline:[
2649
{$match:{$expr:{$and:[{$eq:["$$w","$NO_W_ID"]},{$eq:["$$d","$NO_D_ID"]}]}}},
50+
{$sort:{"NO_O_ID":-1}},
51+
{$limit:1},
2752
{$group:{_id:0,maxO:{$max:"$NO_O_ID"}}}]}},
28-
{$unwind:"$maxNOID"},
53+
{$unwind:"$maxNOID"},
2954
{$match:{$or:[{$expr:{$ne:["$maxOID.maxO","$maxNOID.maxO"]}}, {$expr:{$ne:[ "$maxOID.maxO",{$subtract:["$next",1]}]}} ]}},
30-
{$count:"c"}
3155
]};
3256
// consistency check 3
33-
c[3]={r:"3", c:"NEW_ORDER", pipeline:[
57+
c[3]={r:"3", c:"NEW_ORDER", pipeline:[
3458
{$group:{_id:{w:"$NO_W_ID",d:"$NO_D_ID"},min:{$min:"$NO_O_ID"},max:{$max:"$NO_O_ID"},count:{$sum:1}}},
3559
{$project:{count:1, diff:{$add:[1,{$subtract:["$max","$min"]}]}}},
3660
{$match:{$expr:{$ne:["$count","$diff"]}}},
37-
{$count:"c"}
3861
]};
3962
// consistency check 4
40-
c[4]={r:"4", c:"ORDERS", pipeline:[
63+
c[4]={r:"4", c:"ORDERS", pipeline:[
4164
{$sort:{O_W_ID:1, O_D_ID:1}},
65+
{$limit:10000},
4266
{$group:{_id:{w:"$O_W_ID",d:"$O_D_ID"},O_CLs:{$sum:"$O_OL_CNT"}, OL_CLs:{$sum:{$size:"$ORDER_LINE"}}}},
4367
{$match:{$expr:{$ne:[ "$O_CLs","$OL_CLs"]}}},
44-
{$count:"c"}
4568
]};
4669
// consistency check 5
47-
c[5]={r:"5", c:"ORDERS", pipeline:[
70+
c[5]={r:"5", c:"ORDERS", pipeline:[
4871
{$match:{O_CARRIER_ID:0}},
72+
{$sort:{O_ID:-1}},
73+
{$limit:10000},
4974
{$lookup:{from:"NEW_ORDER", as:"NO_count",let:{w:"$O_W_ID",d:"$O_D_ID",o:"$O_ID"}, pipeline:[
5075
{$match:{$expr:{$and:[{$eq:["$$w","$NO_W_ID"]},{$eq:["$$d","$NO_D_ID"]},{$eq:["$$o","$NO_O_ID"]} ]}}},
5176
{$count:"c"}]}},
5277
{$addFields:{count:{$ifNull:[{$arrayElemAt:["$NO_count.c",0]},0]}}},
5378
{$match:{"count":{$ne:1}}},
54-
{$count:"c"}
5579
]};
5680
// consistency check 6
57-
c[6]={r:"6", c:"ORDERS", pipeline:[
81+
c[6]={r:"6", c:"ORDERS", pipeline:[
82+
{$limit:10000},
5883
{$match:{$expr:{$ne:[ "$O_OL_CNT",{$size:"$ORDER_LINE"}]}}},
59-
{$count:"c"}
6084
]};
6185
// consistency check 7
62-
c[7]={r:"7", c:"ORDERS", pipeline:[
86+
c[7]={r:"7", c:"ORDERS", pipeline:[
87+
{$limit:10000},
6388
{$match:{O_CARRIER_ID:{$ne:0}, "ORDER_LINE.OL_DELIVERY_D":null}},
64-
{$count:"c"}
6589
]};
66-
// consistency
67-
c[8]={r:"8", c:"HISTORY", pipeline:[
68-
{$group:{_id:"$H_W_ID",sum:{$sum:{$toDecimal:"$H_AMOUNT"}}}},
90+
// consistency
91+
c[8]={r:"8", c:"HISTORY", pipeline:[
92+
{$group:{_id:"$H_W_ID",sum:{$sum:{$toDecimal:"$H_AMOUNT"}}}},
6993
{$lookup:{from:"WAREHOUSE", localField:"_id", foreignField:"W_ID",as:"w"}},
70-
{$match:{$expr:{$ne:[0, {$toLong:{$subtract:["$sum", {$arrayElemAt:["$w.W_YTD",0]}]}}]}}},
71-
{$count:"c"}
94+
{$match:{$expr:{$ne:[0, {$toLong:{$subtract:["$sum", {$arrayElemAt:["$w.W_YTD",0]}]}}]}}},
7295
]};
7396
// consistency check 9
74-
c[9]={r:"9", c:"HISTORY", pipeline:[
75-
{$group:{_id:{w:"$H_W_ID", d:"$H_D_ID"}, sum:{$sum:{$toDecimal:"$H_AMOUNT"}}}},
76-
{$lookup:{from:"DISTRICT", as:"d", let:{ w: "$_id.w", d:"$_id.d"}, pipeline:[
77-
{$match: {$expr: {$and: [ {$eq: ["$$w","$D_W_ID"]},{$eq:["$$d","$D_ID" ]}]}}},
78-
{$group:{_id:0, sum:{$sum:{$toDecimal:"$D_YTD"}}}}]}},
97+
c[9]={r:"9", c:"HISTORY", pipeline:[
98+
{$group:{_id:{w:"$H_W_ID", d:"$H_D_ID"}, sum:{$sum:{$toDecimal:"$H_AMOUNT"}}}},
99+
{$lookup:{from:"DISTRICT", as:"d", let:{ w: "$_id.w", d:"$_id.d"}, pipeline:[
100+
{$match: {$expr: {$and: [ {$eq: ["$$w","$D_W_ID"]},{$eq:["$$d","$D_ID" ]}]}}},
101+
{$group:{_id:0, sum:{$sum:{$toDecimal:"$D_YTD"}}}}]}},
79102
{$match:{$expr:{$ne:[{$toLong:"$sum"},{$toLong:{$arrayElemAt:["$d.sum",0]}}]}}},
80-
{$count:"c"}
81103
]};
82104
// *** consistency check 10 don't run unless there is an index
83105
/* adding one warehouse filter to limit checking, needed to add index to HISTORY.H_W_ID,etc to make reasonably fast even on one */
84-
c[10]={r:"10", c:"CUSTOMER", pipeline:[
85-
{$match:{C_W_ID:1}},
86-
{$lookup:{from:"ORDERS", as:"o", let:{ w: "$C_W_ID", d:"$C_D_ID", c:"$C_ID"}, pipeline:[
87-
{$match: {O_CARRIER_ID:{$ne:0}, $expr: {$and: [ {$eq: ["$$w","$O_W_ID"]},{$eq:["$$d","$O_D_ID"]}, {$eq:["$$c","$O_C_ID"]}]}}},
106+
c[10]={r:"10", c:"CUSTOMER", pipeline:[
107+
{$match:{C_W_ID:1,C_D_ID:1,C_ID:{$lt:100,$gt:80}}},
108+
{$lookup:{from:"ORDERS", as:"o", let:{ w: "$C_W_ID", d:"$C_D_ID", c:"$C_ID"}, pipeline:[
109+
{$match: {O_CARRIER_ID:{$ne:0}, $expr: {$and: [ {$eq: ["$$w","$O_W_ID"]},{$eq:["$$d","$O_D_ID"]}, {$eq:["$$c","$O_C_ID"]}]}}},
88110
{$group:{_id:0, sum:{$sum:{$sum:"$ORDER_LINE.OL_AMOUNT"}}}}]}},
89-
{$lookup:{from:"HISTORY", as:"h", let:{ w: "$C_W_ID", d:"$C_D_ID", c:"$C_ID"}, pipeline:[
90-
{$match: {$expr: {$and: [ {$eq: ["$$w","$H_W_ID"]},{$eq:["$$d","$H_D_ID"]}, {$eq:["$$c","$H_C_ID"]}]}}},
91-
{$group:{_id:0, sum:{$sum:"$H_AMOUNT"}}}]}},
111+
{$lookup:{from:"HISTORY", as:"h", let:{ w: "$C_W_ID", d:"$C_D_ID", c:"$C_ID"}, pipeline:[
112+
{$match: {$expr: {$and: [ {$eq: ["$$w","$H_W_ID"]},{$eq:["$$d","$H_D_ID"]}, {$eq:["$$c","$H_C_ID"]}]}}},
113+
{$group:{_id:0, sum:{$sum:"$H_AMOUNT"}}}]}},
92114
{$project:{C_BALANCE:1, OSUM:{$ifNull:[{$arrayElemAt:["$o.sum",0]},0]},HSUM:{$arrayElemAt:["$h.sum",0]},_id:0, C_ID:1, C_W_ID:1, C_D_ID:1}},
93115
{$match:{$expr:{$ne:["$C_BALANCE", {$subtract:["$OSUM","$HSUM"]}]}}},
94-
{$count:"c"}
95116
]};
96117
// *** consistency check 11 Correct when first loaded!
97-
c[11]={r:"11", c:"DISTRICT", pipeline:[
98-
{$project:{w:"$D_W_ID", d:"$D_ID"}},
118+
c[11]={r:"11", c:"DISTRICT", pipeline:[
119+
{$project:{w:"$D_W_ID", d:"$D_ID"}},
99120
{$lookup:{from:"ORDERS",as:"o",let:{w:"$w",d:"$d"},pipeline:[
100121
{$match:{$expr:{$and:[{$eq:["$$w","$O_W_ID"]},{$eq:["$$d","$O_D_ID"]}]}}},
101122
{$count:"c"}]}},
102-
{$unwind:"$o"},
123+
{$unwind:"$o"},
103124
{$lookup:{from:"NEW_ORDER",as:"no",let:{w:"$w",d:"$d"},pipeline:[
104125
{$match:{$expr:{$and:[{$eq:["$$w","$NO_W_ID"]},{$eq:["$$d","$NO_D_ID"]}]}}},
105126
{$count:"c"}]}},
106127
{$unwind:"$no"},
107128
{$match:{$expr:{$ne:[2100, {$subtract:["$o.c","$no.c"]}]}}},
108-
{$count:"c"}
109129
]};
110130
// consistency check 12
111-
c[12]={r:"12", c:"CUSTOMER", pipeline:[
112-
{$lookup:{from:"ORDERS", as:"o", let:{ w: "$C_W_ID", d:"$C_D_ID", c:"$C_ID"}, pipeline:[
113-
{$match: {O_CARRIER_ID:{$ne:0},$expr: {$and: [ {$eq: ["$$w","$O_W_ID"]},{$eq:["$$d","$O_D_ID"]}, {$eq:["$$c","$O_C_ID"]}]}}},
131+
c[12]={r:"12", c:"CUSTOMER", pipeline:[
132+
{$lookup:{from:"ORDERS", as:"o", let:{ w: "$C_W_ID", d:"$C_D_ID", c:"$C_ID"}, pipeline:[
133+
{$match: {O_CARRIER_ID:{$ne:0},$expr: {$and: [ {$eq: ["$$w","$O_W_ID"]},{$eq:["$$d","$O_D_ID"]}, {$eq:["$$c","$O_C_ID"]}]}}},
114134
{$group:{_id:0, sum:{$sum:{$sum:"$ORDER_LINE.OL_AMOUNT"}}}}]}},
115135
{$project:{C_BALANCE:1, C_YTD_PAYMENT:1, OLSUM:{$ifNull:[{$arrayElemAt:["$o.sum",0]},0]}}},
116136
{$match:{$expr:{$ne:[0,{$toLong:{$subtract:["$OLSUM", {$add:["$C_BALANCE","$C_YTD_PAYMENT"]}]}}]}}},
117-
{$count:"c"}
118137
]};
119138

120-
for (i=1; i<13; i++) {
139+
for (i=1; i<13; i++) {
121140
print (""+ new ISODate() + " Checking " + i);
122-
checkConsistency(c[i].r, c[i].c, c[i].pipeline);
141+
if (i==10) continue;
142+
checkConsistency(c[i].r, c[i].c, c[i].pipeline,true);
123143
}
124144

0 commit comments

Comments
 (0)