Add process file and Docker run for batch

Author: kamicut

.yarnrc.yml

@@ -0,0 +1 @@
+nodeLinker: node-modules

Dockerfile

@@ -0,0 +1,30 @@
+# Start with an official Node image
+FROM node:18
+
+# Update the system and install necessary dependencies
+RUN apt-get update && \
+  apt-get install -y parallel jq && \
+  apt-get clean && \
+  rm -rf /var/lib/apt/lists/*
+
+# Create an app directory to hold the application code inside the image
+WORKDIR /usr/src/app
+
+# Copy your package.json and package-lock.json (if you have one) into the container
+COPY package*.json ./
+
+# Install your Node dependencies
+RUN npm install
+
+# Copy your Node scripts into the container
+COPY fetchOsc.js ./
+COPY parser.js ./
+
+# The main script to run the tasks
+COPY process.sh ./
+
+# Give execute permissions to the script
+RUN chmod +x process.sh
+
+# The command to run when the container starts
+ENTRYPOINT [ "./process.sh" ]

fetchOsc.js

@@ -0,0 +1,29 @@
+const maxRetries = 10;
+const retryInterval = 5000;
+const fs = require("fs");
+const axios = require("axios");
+
+const OSC_SOURCE_URL =
+  "http://s3-eu-west-1.amazonaws.com/overpass-db-eu-west-1/augmented-diffs";
+
+const sequenceNumber = process.argv[2];
+const outputFileName = process.argv[3];
+const url = `${OSC_SOURCE_URL}/${sequenceNumber}.osc`;
+
+async function downloadFile(url, outputFileName) {
+  const oscFile = fs.createWriteStream(outputFileName);
+  try {
+    const res = await axios({
+      method: "get",
+      url,
+      responseType: "stream",
+    });
+    res.data.pipe(oscFile);
+  } catch (err) {
+    console.error(err);
+  }
+}
+
+downloadFile(url, outputFileName).then(() => {
+  console.log(`Downloaded ${url} to ${outputFileName}`);
+});

package.json

@@ -9,6 +9,7 @@
   "author": "",
   "license": "ISC",
   "dependencies": {
+    "axios": "^1.5.0",
     "osm-adiff-parser": "^1.1.0",
     "real-changesets-parser": "https://github.com/developmentseed/real-changesets-parser.git"
   }

parser.js

@@ -1,27 +1,29 @@
-const adiffParser = require('osm-adiff-parser');
-const changesetParser = require('real-changesets-parser');
-const fs = require('fs');
+const adiffParser = require("osm-adiff-parser");
+const changesetParser = require("real-changesets-parser");
+const fs = require("fs");
 
 // Read the file name from the command-line argument
 const fileName = process.argv[2];
 
 // Read the file
-fs.readFile(fileName, 'utf8', (err, data) => {
+fs.readFile(fileName, "utf8", (err, data) => {
   if (err) {
-    console.error('Error reading the file:', err);
+    console.error("Error reading the file:", err);
     return;
   }
-  let DONE = false;
+  let ranOnce = false;
   adiffParser(data, null, (err, result) => {
-    if (DONE) { return };
-    DONE = true;
+    if (ranOnce) {
+      return;
+    }
+    ranOnce = true;
     // console.log('keys', Object.keys(result));
     const featureCollection = {
-      'type': 'FeatureCollection',
-      'features': []
+      type: "FeatureCollection",
+      features: [],
     };
-    Object.keys(result).forEach(changesetId => {
-      result[changesetId].forEach(element => {
+    Object.keys(result).forEach((changesetId) => {
+      result[changesetId].forEach((element) => {
         const change = changesetParser.elementParser(element);
         // console.log('change', change);
         featureCollection.features = featureCollection.features.concat(change);
@@ -33,6 +35,4 @@ fs.readFile(fileName, 'utf8', (err, data) => {
     // console.log('changeset', changeset);
     // return result;
   });
-
-
 });

process.sh

@@ -0,0 +1,44 @@
+#!/bin/bash
+
+# Check if the necessary arguments were provided
+if [[ -z "$1" ]] || [[ -z "$2" ]]; then
+    echo "Usage: $0 <start_number> <num_files>"
+    exit 1
+fi
+
+# Starting number provided as the first argument
+start_num=$1
+
+# Number of files to fetch
+num_files=$2
+
+# Calculate the end number
+end_num=$((start_num - num_files))
+
+# Fetch .osc files using fetchOsc.js
+fetchOscFunction() {
+    sequence_num=$1
+    file_path="/tmp/${sequence_num}.osc"
+    node fetchOsc.js $sequence_num $file_path
+}
+
+export -f fetchOscFunction
+
+seq $end_num $start_num | parallel fetchOscFunction
+
+# Parse .osc files and collect into a single geojson using parser.js and jq
+echo '{ "type": "FeatureCollection", "features": [' > /tmp/final_geojson.json
+
+echo 'Parsing files'
+for (( i=start_num; i>end_num; i-- )); do
+    echo $i
+    file_path="/tmp/${i}.osc"
+    node parser.js $file_path | jq '.features[]' >> /tmp/final_geojson.json
+    # Add a comma between geojsons except for the last one
+    if [[ "$i" -ne "$((end_num+1))" ]]; then
+        echo ',' >> /tmp/final_geojson.json
+    fi
+done
+echo 'Done parsing files'
+
+echo '] }' >> /tmp/final_geojson.json