Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 16 additions & 2 deletions bin/importGlossary.js
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,8 @@ const setGlossary = async (metadata) => {
answer?.UIID || "",
definition,
aliases,
imageToUpdate
imageToUpdate,
shouldUpdateImage ? metadata.imageDimensions : undefined
);

if (res.status === 429) {
Expand Down Expand Up @@ -130,7 +131,13 @@ async function main() {
// Fetch Google Doc
const gdocsClient = await getDocsClient();
const doc = await getGoogleDoc({ docID: GLOSSARY_DOC }, gdocsClient);
await replaceImages(doc.inlineObjects, GLOSSARY_DOC);
const imageDimensions = await replaceImages(doc.inlineObjects, GLOSSARY_DOC);
const documentContext = {
footnotes: doc.footnotes || {},
namedStyles: doc.namedStyles,
inlineObjects: doc.inlineObjects,
imageDimensions,
};

const table = doc.body.content.filter(({ table }) => table)[0];

Expand Down Expand Up @@ -211,13 +218,19 @@ async function main() {
const imgMatch = row.image?.match(/!\[\]\((.*?)\)/);
const newImage = imgMatch?.[1];

// Get image dimensions if available
const dimensions = newImage
? documentContext.imageDimensions[newImage]
: null;

// If no existing entry found, mark as new
if (!existingEntry) {
return {
row,
needsUpdate: true,
existsInCoda: false,
parsedImage: newImage,
imageDimensions: dimensions,
};
}

Expand Down Expand Up @@ -263,6 +276,7 @@ async function main() {
existsInCoda: true,
existingEntry,
parsedImage: newImage,
imageDimensions: dimensions,
};
});

Expand Down
1 change: 1 addition & 0 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
"google-auth-library": "^8.7.0",
"googleapis": "^105.0.0",
"html-entities": "^2.4.0",
"image-size": "^2.0.2",
"markdown-table": "^3.0.4",
"node-fetch": "^3.3.1",
"prettier": "^2.8.8"
Expand Down
48 changes: 43 additions & 5 deletions parser/cloudflare.js
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import { randomUUID } from "crypto";
import { withRetry } from "./utils.js";
import imageSize from "image-size";

const sendRequest = (endpoint, method, body) =>
withRetry(
Expand Down Expand Up @@ -30,6 +31,25 @@ const sendRequest = (endpoint, method, body) =>
}
);

const getImageDimensions = async (url) => {
try {
const response = await fetch(url);
if (!response.ok) {
console.warn(
`Failed to fetch image for dimensions: ${response.statusText}`
);
return null;
}

const buffer = Buffer.from(await response.arrayBuffer());
const dimensions = imageSize(buffer);
return { width: dimensions.width, height: dimensions.height };
} catch (error) {
console.warn(`Error getting image dimensions for ${url}:`, error.message);
return null;
}
};

const uploadImage = async (url, metadata) => {
const formData = new FormData();
formData.append("url", url);
Expand All @@ -42,19 +62,37 @@ const uploadImage = async (url, metadata) => {
};

export const replaceImages = async (objects, uiid) => {
if (!process.env.CLOUDFLARE_ACCOUNT_ID) return null;
if (!process.env.CLOUDFLARE_ACCOUNT_ID) return {};

const fingerprint = randomUUID();
const imageDimensions = {};

const updates = Object.entries(objects || {}).map(async ([key, obj]) => {
const img = obj?.inlineObjectProperties?.embeddedObject;
if (img) {
img.imageProperties.contentUri = await uploadImage(
img.imageProperties.contentUri,
{ title: img.title, UIID: uiid, fingerprint }
);
const originalUri = img.imageProperties.contentUri;

// Get dimensions before uploading
const dimensions = await getImageDimensions(originalUri);

// Upload to Cloudflare
const newUri = await uploadImage(originalUri, {
title: img.title,
UIID: uiid,
fingerprint,
});

img.imageProperties.contentUri = newUri;

// Store dimensions mapped by the final URI
if (dimensions && newUri) {
imageDimensions[newUri] = dimensions;
}
}
});

await Promise.all(updates);
return imageDimensions;
};

export const questionImages = async () => {
Expand Down
25 changes: 21 additions & 4 deletions parser/coda.js
Original file line number Diff line number Diff line change
Expand Up @@ -211,9 +211,25 @@ export const updateGlossary = async (
questionUIId,
md,
aliases,
image
) =>
codaUpsert(
image,
imageDimensions
) => {
// Require image dimensions if image is provided
let imageData = undefined;
if (image) {
if (!imageDimensions) {
throw new Error(
`Image dimensions required for glossary entry: ${glossaryWord}`
);
}
imageData = JSON.stringify({
url: image,
width: imageDimensions.width,
height: imageDimensions.height,
});
}

return codaUpsert(
`${glossaryTableURL}/rows/`,
{
glossaryWord,
Expand All @@ -222,7 +238,8 @@ export const updateGlossary = async (
glossaryRichText: md,
glossaryAliases: aliases,
glossaryLastIngested: new Date().toISOString(),
glossaryImage: image || undefined,
glossaryImage: imageData,
},
["glossaryWord"]
);
};