Merge pull request #5 from bewai-tech/feature/sirene-data-gouv-dataset

feat: add data.gouv sirene dataset in place of ODS
bewai-tech · Feb 16, 2024 · 9384add · 9384add
2 parents 476a081 + 168f34f
commit 9384add
Showing 11 changed files with 269 additions and 232 deletions.
diff --git a/README.md b/README.md
@@ -22,7 +22,9 @@ These scripts can be found in the `src/utils` directory.
 |--|--|
 | `initTables` | Create all tables, their columns and indexes if not existing |
 | `downloadAssets {entity}` | Download fresh data file for the given entity from configured remote source |
-| `importData` | (Interactive) Imports the data for the given entity from a data file in the `assets` folder. Usually from `.csv` files. A custom CSV separator can be specified in the script file `dataSets` configuration object.
+| `importData` | (Interactive) Imports the data for the given entity from a data file in the `assets` folder. Usually from `.csv` files. A custom CSV separator can be specified in the script file `dataSets` configuration object. |
+
+> `unzip` **needs to be installed directly on the machine**. Because of some NodeJS limitations with Buffer size, extracting large `.zip` files with a NodeJS library can't be done properly.
 
 ## Run the stack
 

diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -87,4 +87,4 @@
         "winston": "^3.8.2",
         "winston-daily-rotate-file": "^4.7.1"
     }
-}
+}
diff --git a/src/config/config.ts b/src/config/config.ts
@@ -21,7 +21,8 @@ export default {
         port: +process.env.DB_PORT
     },
     opendataRemoteSources: {
-        sirene: process.env.SIRENE,
+        sirene_unitelegale: process.env.SIRENE_UNITE_LEGALE,
+        sirene_etablissement: process.env.SIRENE_ETABLISSEMENT,
         departementsfr: process.env.DEPARTEMENTS_FR,
         ban: process.env.BAN
     }

diff --git a/src/controllers/siren.controller.ts b/src/controllers/siren.controller.ts
@@ -5,7 +5,7 @@ class SirenController {
 
     /**
      * Get all sirene paginated data or specific one with siret number
-     * @returns the corresponding item(s)
+     * @returns the corresponding item
      */
     static checkSirene = async (req: Request, res: Response) => {
         const { siret, siren } = req.query as Record<any, string>;
@@ -15,11 +15,22 @@ class SirenController {
         let result;
         try {
             if (siret) {
-                result = await db.one('SELECT * FROM sirene WHERE siret = $1', [siret]);
+                result = await db.one('SELECT * FROM sirene_etablissement WHERE siret = $1', [siret]);
+
+                if (result) {
+                    const unitelegale = await db.one('SELECT * FROM sirene_unitelegale WHERE siren = $1', [result.siren]);
+                    result = {...result, ...unitelegale};
+                }
+
             } else if (siren) {
-                result = await db.one('SELECT * FROM sirene WHERE siren = $1 AND nic = (SELECT MAX(DISTINCT nic) FROM sirene WHERE siren = $1)', [siren]);
+                result = await db.one('SELECT * FROM sirene_unitelegale WHERE siren = $1', [siren]);
+
+                if (result) {
+                    result.siret = await db.manyOrNone('SELECT * FROM sirene_etablissement WHERE siren = $1', [result.siren]);
+                }
             } else {
-                result = await db.any(`SELECT * FROM sirene LIMIT ${limit} OFFSET ${offset}`);
+                // result = await db.any(`SELECT * FROM sirene LIMIT ${limit} OFFSET ${offset}`);
+                res.json({message: 'Please add `siret` or `siren` query parameter.'});
             }
         } catch (error) {
             res.sendStatus(404);
@@ -35,14 +46,26 @@ class SirenController {
 
         try {
 
-            switch (method) {
-                case 'estimate':
-                    result = await db.one('SELECT reltuples AS estimate FROM pg_class where relname = $1', ['sirene']);
-                    break;
+            if (method === 'estimate') {
+                const estimates = await Promise.all([
+                    db.one('SELECT reltuples AS estimate FROM pg_class where relname = $1', ['sirene_unitelegale']),
+                    db.one('SELECT reltuples AS estimate FROM pg_class where relname = $1', ['sirene_etablissement'])
+                ]);
+
+                result = {
+                    sirene_unitelegale: +estimates[0]?.estimate,
+                    sirene_etablissement: +estimates[1]?.estimate
+                };
+            } else {
+                const estimates = await Promise.all([
+                    db.one('SELECT count(*) FROM sirene_unitelegale'),
+                    db.one('SELECT count(*) FROM sirene_etablissement')
+                ]);
 
-                default:
-                    result = await db.one('SELECT count(*) FROM sirene');
-                    break;
+                result = {
+                    sirene_unitelegale: +estimates[0]?.count,
+                    sirene_etablissement: +estimates[1]?.count
+                };
             }
 
         } catch (error) {

diff --git a/src/tables/sirene.ts b/src/tables/sirene.ts
@@ -2,122 +2,103 @@ import { Pg } from '../helpers';
 
 export const initSireneTable = () => {
 
-    const createTable = `
-    CREATE TABLE IF NOT EXISTS "sirene" (
+    const createTableUniteLegale = `
+    CREATE TABLE IF NOT EXISTS "sirene_unitelegale" (
+        "siren" CHARACTER VARYING,
+        "statutDiffusionUniteLegale" CHARACTER VARYING,
+        "unitePurgeeUniteLegale" CHARACTER VARYING,
+        "dateCreationUniteLegale" CHARACTER VARYING,
+        "sigleUniteLegale" CHARACTER VARYING,
+        "sexeUniteLegale" CHARACTER VARYING,
+        "prenom1UniteLegale" CHARACTER VARYING,
+        "prenom2UniteLegale" CHARACTER VARYING,
+        "prenom3UniteLegale" CHARACTER VARYING,
+        "prenom4UniteLegale" CHARACTER VARYING,
+        "prenomUsuelUniteLegale" CHARACTER VARYING,
+        "pseudonymeUniteLegale" CHARACTER VARYING,
+        "identifiantAssociationUniteLegale" CHARACTER VARYING,
+        "trancheEffectifsUniteLegale" CHARACTER VARYING,
+        "anneeEffectifsUniteLegale" CHARACTER VARYING,
+        "dateDernierTraitementUniteLegale" CHARACTER VARYING,
+        "nombrePeriodesUniteLegale" CHARACTER VARYING,
+        "categorieEntreprise" CHARACTER VARYING,
+        "anneeCategorieEntreprise" CHARACTER VARYING,
+        "dateDebut" CHARACTER VARYING,
+        "etatAdministratifUniteLegale" CHARACTER VARYING,
+        "nomUniteLegale" CHARACTER VARYING,
+        "nomUsageUniteLegale" CHARACTER VARYING,
+        "denominationUniteLegale" CHARACTER VARYING,
+        "denominationUsuelle1UniteLegale" CHARACTER VARYING,
+        "denominationUsuelle2UniteLegale" CHARACTER VARYING,
+        "denominationUsuelle3UniteLegale" CHARACTER VARYING,
+        "categorieJuridiqueUniteLegale" CHARACTER VARYING,
+        "activitePrincipaleUniteLegale" CHARACTER VARYING,
+        "nomenclatureActivitePrincipaleUniteLegale" CHARACTER VARYING,
+        "nicSiegeUniteLegale" CHARACTER VARYING,
+        "economieSocialeSolidaireUniteLegale" CHARACTER VARYING,
+        "societeMissionUniteLegale" CHARACTER VARYING,
+        "caractereEmployeurUniteLegale" CHARACTER VARYING
+    );`;
+
+    const createTableEtablissement = `
+    CREATE TABLE IF NOT EXISTS "sirene_etablissement" (
         "siren" CHARACTER VARYING,
         "nic" CHARACTER VARYING,
         "siret" CHARACTER VARYING,
-        "statutdiffusionetablissement" CHARACTER VARYING,
-        "datecreationetablissement" CHARACTER VARYING,
-        "trancheeffectifsetablissement" CHARACTER VARYING,
-        "trancheeffectifsetablissementtriable" CHARACTER VARYING,
-        "anneeeffectifsetablissement" CHARACTER VARYING,
-        "activiteprincipaleregistremetiersetablissement" CHARACTER VARYING,
-        "datederniertraitementetablissement" CHARACTER VARYING,
-        "etablissementsiege" CHARACTER VARYING,
-        "nombreperiodesetablissement" CHARACTER VARYING,
-        "complementadresseetablissement" CHARACTER VARYING,
-        "numerovoieetablissement" CHARACTER VARYING,
-        "indicerepetitionetablissement" CHARACTER VARYING,
-        "typevoieetablissement" CHARACTER VARYING,
-        "libellevoieetablissement" CHARACTER VARYING,
-        "codepostaletablissement" CHARACTER VARYING,
-        "libellecommuneetablissement" CHARACTER VARYING,
-        "libellecommuneetrangeretablissement" CHARACTER VARYING,
-        "distributionspecialeetablissement" CHARACTER VARYING,
-        "codecommuneetablissement" CHARACTER VARYING,
-        "codecedexetablissement" CHARACTER VARYING,
-        "libellecedexetablissement" CHARACTER VARYING,
-        "codepaysetrangeretablissement" CHARACTER VARYING,
-        "libellepaysetrangeretablissement" CHARACTER VARYING,
-        "complementadresse2etablissement" CHARACTER VARYING,
-        "numerovoie2etablissement" CHARACTER VARYING,
-        "indicerepetition2etablissement" CHARACTER VARYING,
-        "typevoie2etablissement" CHARACTER VARYING,
-        "libellevoie2etablissement" CHARACTER VARYING,
-        "codepostal2etablissement" CHARACTER VARYING,
-        "libellecommune2etablissement" CHARACTER VARYING,
-        "libellecommuneetranger2etablissement" CHARACTER VARYING,
-        "distributionspeciale2etablissement" CHARACTER VARYING,
-        "codecommune2etablissement" CHARACTER VARYING,
-        "codecedex2etablissement" CHARACTER VARYING,
-        "libellecedex2etablissement" CHARACTER VARYING,
-        "codepaysetranger2etablissement" CHARACTER VARYING,
-        "libellepaysetranger2etablissement" CHARACTER VARYING,
-        "datedebutetablissement" CHARACTER VARYING,
-        "etatadministratifetablissement" CHARACTER VARYING,
-        "enseigne1etablissement" CHARACTER VARYING,
-        "enseigne2etablissement" CHARACTER VARYING,
-        "enseigne3etablissement" CHARACTER VARYING,
-        "denominationusuelleetablissement" CHARACTER VARYING,
-        "activiteprincipaleetablissement" CHARACTER VARYING,
-        "nomenclatureactiviteprincipaleetablissement" CHARACTER VARYING,
-        "caractereemployeuretablissement" CHARACTER VARYING,
-        "statutdiffusionunitelegale" CHARACTER VARYING,
-        "unitepurgeeunitelegale" CHARACTER VARYING,
-        "datecreationunitelegale" CHARACTER VARYING,
-        "sigleunitelegale" CHARACTER VARYING,
-        "sexeunitelegale" CHARACTER VARYING,
-        "prenom1unitelegale" CHARACTER VARYING,
-        "prenom2unitelegale" CHARACTER VARYING,
-        "prenom3unitelegale" CHARACTER VARYING,
-        "prenom4unitelegale" CHARACTER VARYING,
-        "prenomusuelunitelegale" CHARACTER VARYING,
-        "pseudonymeunitelegale" CHARACTER VARYING,
-        "identifiantassociationunitelegale" CHARACTER VARYING,
-        "trancheeffectifsunitelegale" CHARACTER VARYING,
-        "trancheeffectifsunitelegaletriable" CHARACTER VARYING,
-        "anneeeffectifsunitelegale" CHARACTER VARYING,
-        "datederniertraitementunitelegale" CHARACTER VARYING,
-        "nombreperiodesunitelegale" CHARACTER VARYING,
-        "categorieentreprise" CHARACTER VARYING,
-        "anneecategorieentreprise" CHARACTER VARYING,
-        "datedebutunitelegale" CHARACTER VARYING,
-        "etatadministratifunitelegale" CHARACTER VARYING,
-        "nomunitelegale" CHARACTER VARYING,
-        "nomusageunitelegale" CHARACTER VARYING,
-        "denominationunitelegale" CHARACTER VARYING,
-        "denominationusuelle1unitelegale" CHARACTER VARYING,
-        "denominationusuelle2unitelegale" CHARACTER VARYING,
-        "denominationusuelle3unitelegale" CHARACTER VARYING,
-        "categoriejuridiqueunitelegale" CHARACTER VARYING,
-        "activiteprincipaleunitelegale" CHARACTER VARYING,
-        "nomenclatureactiviteprincipaleunitelegale" CHARACTER VARYING,
-        "nicsiegeunitelegale" CHARACTER VARYING,
-        "economiesocialesolidaireunitelegale" CHARACTER VARYING,
-        "societemissionunitelegale" CHARACTER VARYING,
-        "caractereemployeurunitelegale" CHARACTER VARYING,
-        "codeepcietablissement" CHARACTER VARYING,
-        "epcietablissement" CHARACTER VARYING,
-        "codearrondissementetablissement" CHARACTER VARYING,
-        "codedepartementetablissement" CHARACTER VARYING,
-        "departementetablissement" CHARACTER VARYING,
-        "coderegionetablissement" CHARACTER VARYING,
-        "regionetablissement" CHARACTER VARYING,
-        "sectionetablissement" CHARACTER VARYING,
-        "soussectionetablissement" CHARACTER VARYING,
-        "divisionetablissement" CHARACTER VARYING,
-        "groupeetablissement" CHARACTER VARYING,
-        "classeetablissement" CHARACTER VARYING,
-        "sectionunitelegale" CHARACTER VARYING,
-        "soussectionunitelegale" CHARACTER VARYING,
-        "divisionunitelegale" CHARACTER VARYING,
-        "groupeunitelegale" CHARACTER VARYING,
-        "classeunitelegale" CHARACTER VARYING,
-        "naturejuridiqueunitelegale" CHARACTER VARYING,
-        "l1_adressage_unitelegale" CHARACTER VARYING,
-        "adresseetablissement" CHARACTER VARYING,
-        "siretsiegeunitelegale" CHARACTER VARYING,
-        "datefermetureetablissement" CHARACTER VARYING,
-        "datefermetureunitelegale" CHARACTER VARYING,
-        "geolocetablissement" CHARACTER VARYING
+        "statutDiffusionEtablissement" CHARACTER VARYING,
+        "dateCreationEtablissement" CHARACTER VARYING,
+        "trancheEffectifsEtablissement" CHARACTER VARYING,
+        "anneeEffectifsEtablissement" CHARACTER VARYING,
+        "activitePrincipaleRegistreMetiersEtablissement" CHARACTER VARYING,
+        "dateDernierTraitementEtablissement" CHARACTER VARYING,
+        "etablissementSiege" CHARACTER VARYING,
+        "nombrePeriodesEtablissement" CHARACTER VARYING,
+        "complementAdresseEtablissement" CHARACTER VARYING,
+        "numeroVoieEtablissement" CHARACTER VARYING,
+        "indiceRepetitionEtablissement" CHARACTER VARYING,
+        "typeVoieEtablissement" CHARACTER VARYING,
+        "libelleVoieEtablissement" CHARACTER VARYING,
+        "codePostalEtablissement" CHARACTER VARYING,
+        "libelleCommuneEtablissement" CHARACTER VARYING,
+        "libelleCommuneEtrangerEtablissement" CHARACTER VARYING,
+        "distributionSpecialeEtablissement" CHARACTER VARYING,
+        "codeCommuneEtablissement" CHARACTER VARYING,
+        "codeCedexEtablissement" CHARACTER VARYING,
+        "libelleCedexEtablissement" CHARACTER VARYING,
+        "codePaysEtrangerEtablissement" CHARACTER VARYING,
+        "libellePaysEtrangerEtablissement" CHARACTER VARYING,
+        "complementAdresse2Etablissement" CHARACTER VARYING,
+        "numeroVoie2Etablissement" CHARACTER VARYING,
+        "indiceRepetition2Etablissement" CHARACTER VARYING,
+        "typeVoie2Etablissement" CHARACTER VARYING,
+        "libelleVoie2Etablissement" CHARACTER VARYING,
+        "codePostal2Etablissement" CHARACTER VARYING,
+        "libelleCommune2Etablissement" CHARACTER VARYING,
+        "libelleCommuneEtranger2Etablissement" CHARACTER VARYING,
+        "distributionSpeciale2Etablissement" CHARACTER VARYING,
+        "codeCommune2Etablissement" CHARACTER VARYING,
+        "codeCedex2Etablissement" CHARACTER VARYING,
+        "libelleCedex2Etablissement" CHARACTER VARYING,
+        "codePaysEtranger2Etablissement" CHARACTER VARYING,
+        "libellePaysEtranger2Etablissement" CHARACTER VARYING,
+        "dateDebut" CHARACTER VARYING,
+        "etatAdministratifEtablissement" CHARACTER VARYING,
+        "enseigne1Etablissement" CHARACTER VARYING,
+        "enseigne2Etablissement" CHARACTER VARYING,
+        "enseigne3Etablissement" CHARACTER VARYING,
+        "denominationUsuelleEtablissement" CHARACTER VARYING,
+        "activitePrincipaleEtablissement" CHARACTER VARYING,
+        "nomenclatureActivitePrincipaleEtablissement" CHARACTER VARYING,
+        "caractereEmployeurEtablissement" CHARACTER VARYING
     );`;
 
-    return Pg.execute(createTable).then(async (res) => {
+    return Promise.all([Pg.execute(createTableUniteLegale), Pg.execute(createTableEtablissement)]).then((values) => {
+        const res = values.every(val => val);
+
         if (res) {
-            console.log('sirene table initialized ✓');
+            console.log('sirene tables initialized ✓');
         } else {
-            console.log('sirene table already exist (nothing done)');
+            console.log('sirene tables already exist (nothing done)');
         }
 
         return res;
-Original file line number
+Diff line change
@@ @@ -87,4 +87,4 @@ @@
             "winston": "^3.8.2",
             "winston-daily-rotate-file": "^4.7.1"
         }
-    }
+    }