Skip to content

Commit 7682538

Browse files
PercoccoPercocco
committed
owner added to record
1 parent af82012 commit 7682538

File tree

3 files changed

+19
-662
lines changed

3 files changed

+19
-662
lines changed

‎Minio-SQL-connector/api/services/service.js‎

Lines changed: 0 additions & 178 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,6 @@ const Source = require('../models/source')
66
const Value = require('../models/value')
77
const Key = require('../models/key')
88
const Entries = require('../models/entries')
9-
const common = require('../../utils/common')
109
const { sleep, json2csv } = require('../../utils/common')
1110
const { Client } = require('pg');
1211
const config = require('../../config')
@@ -28,12 +27,6 @@ else
2827
for (let bucket of minioConfig.subscribe.buckets)
2928
minioWriter.getNotifications(bucket)
3029

31-
async function save(objects) {
32-
for (let obj of objects)
33-
await minioWriter.insertInDBs(obj.raw, obj.info, true)
34-
return true
35-
}
36-
3730
async function sync() {
3831
try {
3932
if (!syncing) {
@@ -42,7 +35,6 @@ async function sync() {
4235
await Key.deleteMany({})
4336
await Value.deleteMany({})
4437
await Entries.deleteMany({})
45-
//await sleep(10000)
4638
let objects = []
4739
let buckets = await minioWriter.listBuckets()
4840
let bucketIndex = 1
@@ -84,12 +76,6 @@ async function sync() {
8476
logger.error(error)
8577
}
8678

87-
//minioWriter.entities.values = minioWriter.entities.values.map(obj => ({ ...obj, visibility: getVisibility(obj) }))
88-
//minioWriter.entities.keys = minioWriter.entities.keys.map(obj => ({ ...obj, visibility: getVisibility(obj) }))
89-
//minioWriter.entities.entries = minioWriter.entities.entries.map(obj => ({ ...obj, visibility: getVisibility(obj) }))
90-
91-
//const existingEntries = await Entries.find({ key: { $in: Object.keys(minioWriter.entries) } }); //TODO now this line is useless
92-
9379
let entries = Object.entries(minioWriter.entries).map(([key, value]) => ({ [key]: value }));
9480
let entriesInDB = []
9581
for (let key in minioWriter.entries)
@@ -106,7 +92,6 @@ async function sync() {
10692
log(error);
10793
} else {
10894
try {
109-
// Pulizia di `$` solo in nestedKey
11095
entries = entries.map(entry => {
11196
let fixedEntry = {};
11297

@@ -138,7 +123,6 @@ async function sync() {
138123

139124
let valuesToDB = []
140125

141-
//console.log(entries)
142126
for (let entry of entries)
143127
for (let key in entry)
144128
for (let subKeyAliasValue in entry[key]) {
@@ -149,10 +133,6 @@ async function sync() {
149133
valuesToDB.push({ value: subKeyAliasValue, visibility: entry[key][subKeyAliasValue] })
150134
}
151135

152-
/*let valuesToDB = entries.map(obj => ({
153-
visibility: obj[Object.keys(obj).pop()][Object.keys(obj[Object.keys(obj).pop()]).pop()],
154-
value: Object.keys(obj[Object.keys(obj).pop()]).pop()
155-
}))*/
156136
let keysToDB = entries.map(obj => ({
157137
key: Object.keys(obj).pop() || "flag_error_key_missing",
158138
visibility: obj[Object.keys(obj).pop()][Object.keys(obj[Object.keys(obj).pop()]).pop()],
@@ -162,164 +142,6 @@ async function sync() {
162142
await Key.insertMany(keysToDB)
163143
await Value.insertMany(valuesToDB)
164144

165-
166-
/*
167-
let entries = Object.entries(minioWriter.entries)
168-
entries = entries.map(entry => ({ [entry[0]]: [entry[1]] }))
169-
try {
170-
if (entries.length > 0) await Entries.insertMany(entries);
171-
}
172-
catch (error) {
173-
if (!error?.errorResponse?.message?.includes("Document can't have"))
174-
log(error)
175-
else
176-
try {
177-
for (let entry of entries)
178-
for (let key in entry)
179-
for (let value in entry[key])
180-
entry[key] = {
181-
[entry[key][value.replace(/\$/g, '')]]: JSON.parse(JSON.stringify(entry[key][value]))
182-
}
183-
await Entries.insertMany(entries)
184-
}
185-
catch (error) {
186-
log("There are problems inserting object in mongo DB")
187-
log(error)
188-
}
189-
}
190-
191-
let entries = Object.entries(minioWriter.entries)
192-
entries = entries.map(entry => ({ [entry[0]]: entry[1] }))
193-
try {
194-
if (entries.length > 0) await Entries.insertMany(entries);
195-
}
196-
catch (error) {
197-
if (!error?.errorResponse?.message?.includes("Document can't have"))
198-
log(error)
199-
else
200-
try {
201-
for (let entry of entries)
202-
for (let key in entry)
203-
for (let nestedKey in entry[key])
204-
entry[key] = {
205-
[entry[key][nestedKey.replace(/\$/g, '')]]: JSON.parse(JSON.stringify(entry[key][nestedKey]))
206-
}
207-
await Entries.insertMany(entries)
208-
}
209-
catch (error) {
210-
log("There are problems inserting object in mongo DB")
211-
log(error)
212-
}
213-
}
214-
215-
let entries = Object.entries(minioWriter.entries).map(([key, value]) => ({ [key]: value }));
216-
try {
217-
if (entries.length > 0) await Entries.insertMany(entries);
218-
} catch (error) {
219-
if (!error?.errorResponse?.message?.includes("Document can't have")) {
220-
log(error);
221-
} else {
222-
try {
223-
entries = entries.map(entry => {
224-
let fixedEntry = {};
225-
for (let key in entry) {
226-
let sanitizedKey = key.replace(/\$/g, '');
227-
fixedEntry[sanitizedKey] = entry[key];
228-
}
229-
return fixedEntry;
230-
});
231-
232-
await Entries.insertMany(entries);
233-
} catch (error) {
234-
log("There are problems inserting object in MongoDB");
235-
log(error);
236-
}
237-
}
238-
}*/
239-
240-
241-
/*
242-
const existingValues = (await Value.find({ value: { $in: minioWriter.entities.values.map(v => v.value) } }))//.map(v => v.value);
243-
const existingKeys = (await Key.find({ key: { $in: minioWriter.entities.keys.map(k => k.key) } }))//.map(k => k.key); //{ key: 1, _id: 0 }
244-
const existingValuesMap = new Map();
245-
existingValues.forEach(v => {
246-
if (!existingValuesMap.has(v.name)) {
247-
existingValuesMap.set(v.name, new Set());
248-
}
249-
existingValuesMap.get(v.name).add(v.value);
250-
});
251-
const uniqueValues = existingValues[0] ? minioWriter.entities.values.filter(entry => !existingValuesMap.has(entry.name) || !existingValuesMap.get(entry.name).has(entry.value)) : minioWriter.entities.values;
252-
try {
253-
if (uniqueValues.length > 0) await Value.insertMany(uniqueValues);
254-
}
255-
catch (error) {
256-
if (!error?.errorResponse?.message?.includes("Document can't have"))
257-
log(error)
258-
else
259-
try {
260-
await Value.insertMany(uniqueValues.map(v => ({ value: v.value.replace(/\$/g, '') })))
261-
}
262-
catch (error) {
263-
log("There are problems inserting object in mongo DB")
264-
log(error)
265-
}
266-
}
267-
const existingKeysMap = new Map();
268-
existingKeys.forEach(k => {
269-
if (!existingKeysMap.has(k.name)) {
270-
existingKeysMap.set(k.name, new Set());
271-
}
272-
existingKeysMap.get(k.name).add(k.key);
273-
});
274-
const uniqueKeys = existingKeys[0] ? minioWriter.entities.keys.filter(entry => !existingKeysMap.has(entry.name) || !existingKeysMap.get(entry.name).has(entry.key)) : minioWriter.entities.keys;
275-
try {
276-
if (uniqueKeys.length > 0) await Key.insertMany(uniqueKeys);
277-
}
278-
catch (error) {
279-
if (!error?.errorResponse?.message?.includes("Document can't have"))
280-
log(error)
281-
else
282-
try {
283-
await Key.insertMany(uniqueKeys.map(k => ({ key: k.key.replace(/\$/g, '') })))
284-
}
285-
catch (error) {
286-
log("There are problems inserting object in mongo DB")
287-
log(error)
288-
}
289-
}
290-
291-
const existingEntriesMap = new Map();
292-
existingEntries.forEach(e => {
293-
if (!existingEntriesMap.has(e.key)) {
294-
existingEntriesMap.set(e.key, { values: new Set(), names: new Set() });
295-
}
296-
existingEntriesMap.get(e.key).values.add(e.value);
297-
existingEntriesMap.get(e.key).names.add(e.name);
298-
});
299-
const uniqueEntries = existingEntries[0]
300-
? minioWriter.entities.entries.filter(entry => {
301-
const entryMap = existingEntriesMap.get(entry.key);
302-
return (
303-
!entryMap || // Se non esiste una voce con quella 'key', è unica
304-
(!entryMap.values.has(entry.value) && !entryMap.names.has(entry.name)) // Se il valore o il nome non esistono, è unica
305-
);
306-
})
307-
: minioWriter.entities.entries;
308-
try {
309-
if (uniqueEntries.length > 0) await Entries.insertMany(uniqueEntries);
310-
}
311-
catch (error) {
312-
if (!error?.errorResponse?.message?.includes("Document can't have"))
313-
log(error)
314-
else
315-
try {
316-
await Entries.insertMany(uniqueEntries.map(e => ({ key: e.key.replace(/\$/g, ''), value: e.value.replace(/\$/g, '') })))
317-
}
318-
catch (error) {
319-
log("There are problems inserting object in mongo DB")
320-
log(error)
321-
}
322-
}*/
323145
syncing = false
324146
logger.info("Syncing finished")
325147
console.info("Syncing finished")

‎Minio-SQL-connector/config.template.js‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ module.exports = {
2626
delays : 1,
2727
queryAllowedExtensions : ["csv", "json", "geojson"],
2828
parseCompatibilityMode : 0,
29+
ownerInfoEndpoint: "https://platform.beopendep.it/api/owner",
2930
mongo: "mongodb://localhost:27017/Minio-Mongo", // mongo url
3031
authConfig: {
3132
idmHost: "https://platform.beopendep.it/auth",

0 commit comments

Comments
 (0)