Update neo-scraper to 0.9.0 + post.uploadMode changes

Author: neobooru

.github/workflows/build.yaml

@@ -70,7 +70,7 @@ jobs:
     runs-on: ubuntu-20.04
     # Doesn't really need the Chrome build, but we can reuse its cache.
     needs: build_chrome
-    # Only run on master branch AND if tagging or manually dispatching.
+    # Only run (on master branch && if tagged) or (manually dispatching).
     if: startsWith(github.ref, 'refs/tags/v') || github.event.inputs.publish_amo_listed == true
     steps:
       - name: Checkout

package.json

@@ -32,7 +32,7 @@
     "deepmerge": "^4.3.1",
     "lodash": "^4.17.21",
     "markdown-it": "^14.0.0",
-    "neo-scraper": "^0.8.0",
+    "neo-scraper": "^0.9.0",
     "normalize.css": "^8.0.1",
     "pinia": "^2.1.7",
     "primeflex": "^3.3.1",

pnpm-lock.yaml

@@ -8,7 +8,7 @@ export class TagDetails {
     public names: string[],
     public category?: string,
     public usages?: number,
-  ) {}
+  ) { }
 
   get name() {
     return this.names[0];
@@ -38,7 +38,7 @@ export class PoolDetails {
     public names: string[],
     public category?: string,
     public postCount?: number,
-  ) {}
+  ) { }
 
   get name() {
     return this.names[0];
@@ -73,11 +73,11 @@ export class ScrapedPostDetails {
   contentType: ContentType;
   contentSubType: string | undefined;
   rating: BooruTypes.SafetyRating;
-  source = "";
+  source;
+  uploadMode: UploadMode;
   referrer?: string;
   resolution?: [number, number];
   instanceSpecificData: MappedInstanceSpecificData = {};
-  uploadMode: UploadMode = "url";
 
   constructor(post: ScrapedPost) {
     this.contentUrl = post.contentUrl;
@@ -91,6 +91,7 @@ export class ScrapedPostDetails {
     this.tags = post.tags.map((x) => TagDetails.fromScapedTag(x));
     this.notes = post.notes;
     this.resolution = post.resolution;
+    this.uploadMode = post.uploadMode;
   }
 }
 
@@ -108,7 +109,7 @@ export class SimilarPostInfo {
   constructor(
     public readonly id: number,
     public readonly percentage: number,
-  ) {}
+  ) { }
 }
 
 export type PostUploadState = "uploading" | "uploaded" | "error";
@@ -147,38 +148,38 @@ export class PostUploadCommandData {
   constructor(
     public readonly post: ScrapedPostDetails,
     public readonly selectedSite: SzuruSiteConfig,
-  ) {}
+  ) { }
 }
 
 export class SetPostUploadInfoData {
   constructor(
     public instanceId: string,
     public postId: string,
     public info: PostUploadInfo,
-  ) {}
+  ) { }
 }
 
 export class SetExactPostId {
   constructor(
     public readonly instanceId: string,
     public readonly postId: string,
     public readonly exactPostId: number,
-  ) {}
+  ) { }
 }
 
 export class PostUpdateCommandData {
   constructor(
     public readonly postId: number,
     public readonly updateRequest: UpdatePostRequest,
     public readonly selectedSite: SzuruSiteConfig,
-  ) {}
+  ) { }
 }
 
 export class FetchCommandData {
   constructor(
     public readonly url: string,
     public readonly options: RequestInit | undefined = undefined,
-  ) {}
+  ) { }
 }
 
 export class SzuruSiteConfig {
@@ -192,7 +193,7 @@ export class TagCategoryColor {
   constructor(
     public name: string,
     public color: string,
-  ) {}
+  ) { }
 }
 
 export const getDefaultTagCategories = () => [

src/models/index.ts

@@ -11,6 +11,8 @@ import Column from "primevue/column";
 import { SzuruSiteConfig, TagCategoryColor, getDefaultTagCategories } from "~/models";
 import SzurubooruApi from "~/api";
 
+type StatusType = "success" | "error" | "quiet";
+
 const statusText = ref("");
 const statusType = ref("status-quiet");
 const versionInfo = "Version: " + (import.meta.env.VITE_SZ_VERSION ?? browser.runtime.getManifest().version);
@@ -26,9 +28,7 @@ const selectedSite = computed(() => {
   }
 });
 
-type StatusType = "success" | "error" | "quiet";
-
-let mode = useColorMode({ emitAuto: true });
+const mode = useColorMode({ emitAuto: true });
 
 async function testConnection() {
   if (
@@ -221,8 +221,13 @@ wnd.szc_set_config_version = (v = 0) => (cfg.value.version = v);
             <template v-if="selectedSite">
               <div class="field col-12">
                 <label>URL</label>
-                <input v-if="selectedSite" text="Szurubooru URL" type="text" name="domain"
-                  v-model="selectedSite.domain" />
+                <input
+                  v-if="selectedSite"
+                  text="Szurubooru URL"
+                  type="text"
+                  name="domain"
+                  v-model="selectedSite.domain"
+                />
               </div>
 
               <div class="field col-12 md:col-6">
@@ -251,10 +256,14 @@ wnd.szc_set_config_version = (v = 0) => (cfg.value.version = v);
                   <template v-if="field == 'color'">
                     <div class="color-preview">
                       <input type="text" class="color" v-model="data[field]" />
-                      <div class="preview background-preview"
-                        :style="{ 'border-color': data[field], 'background-color': data[field] }"></div>
-                      <div class="preview text-preview" :style="{ 'border-color': data[field], color: data[field] }">
-                      </div>
+                      <div
+                        class="preview background-preview"
+                        :style="{ 'border-color': data[field], 'background-color': data[field] }"
+                      ></div>
+                      <div
+                        class="preview text-preview"
+                        :style="{ 'border-color': data[field], color: data[field] }"
+                      ></div>
                     </div>
                   </template>
 
@@ -285,6 +294,16 @@ wnd.szc_set_config_version = (v = 0) => (cfg.value.version = v);
           <!-- TODO: Category ignore list -->
         </TabPanel>
 
+        <!-- <TabPanel header="About">
+          <div class="grid">
+            <div v-for="engine in scraper.engines" :key="engine.name" class="col-12 md:col-4 flex flex-column">
+              <span class="font-bold">{{ engine.name }}</span>
+              <span>Features: {{ engine.features.join(", ") }}</span>
+              <span>Hosts: {{ engine.supportedHosts.join(", ") }}</span>
+            </div>
+          </div>
+        </TabPanel> -->
+
         <!--
         <TabPanel header="Developer">
           <div class="flex flex-column gap-2">

src/options/App.vue

@@ -1,7 +1,7 @@
 <script setup lang="ts">
 import { useDark } from "@vueuse/core";
 import { cloneDeep } from "lodash";
-import { NeoScraper, ScrapeResults } from "neo-scraper";
+import { ScrapeResults } from "neo-scraper";
 import { getUrl, encodeTagName, getErrorMessage, getPostInfoSummary } from "~/utils";
 import {
   BrowserCommand,
@@ -14,7 +14,6 @@ import {
   SzuruSiteConfig,
   PoolDetails,
 } from "~/models";
-import { ImageSearchResult } from "~/api/models";
 import { isMobile } from "~/env";
 import { DeepReadonly } from "vue";
 import { cfg, usePopupStore } from "~/stores";
@@ -99,14 +98,11 @@ async function grabPost() {
   // Clear current posts array
   pop.posts.splice(0);
 
-  const scraper = new NeoScraper();
-
   for (const result of res.results) {
-    const engine = scraper.engines.find((x) => x.name == result.engine);
-
     for (const i in result.posts) {
       const vm = new ScrapedPostDetails(result.posts[i]);
-      vm.name = `[${result.engine}] Post ${parseInt(i) + 1}`; // parseInt() is required!
+      const name = result.posts[i].name ?? `Post ${parseInt(i) + 1}`; // parseInt() is required!
+      vm.name = `[${result.engine}] ${name}`;
 
       if (!cfg.value.addAllParsedTags) {
         vm.tags.splice(0);
@@ -125,10 +121,6 @@ async function grabPost() {
         vm.instanceSpecificData[site.id] = {};
       }
 
-      if (engine) {
-        vm.uploadMode = engine.uploadMode;
-      }
-
       pop.posts.push(vm);
     }
   }
@@ -145,7 +137,7 @@ async function grabPost() {
     }
 
     if (cfg.value.fetchPostInfo) {
-      await fetchPostInfo();
+      await fetchPostsInfo();
     }
 
     // Delayed call to findSimilar, as fetchPostInfo might change the post's contentUrl.
@@ -168,7 +160,7 @@ async function upload() {
   }
 
   try {
-    const post: ScrapedPostDetails = cloneDeep(pop.selectedPost);
+    const post: ScrapedPostDetails = cloneDeep(pop.selectedPost)!;
 
     // uploadMode "content" requires a content token to work. So ensure it is set.
     if (post.uploadMode == "content") {
@@ -291,10 +283,10 @@ async function findSimilar(post: ScrapedPostDetails | undefined) {
 
   isSearchingForSimilarPosts.value++;
 
-  await ensurePostHasContentToken(post);
-
   try {
-    const res = await selectedInstance.reverseSearchToken(instanceSpecificData.contentToken);
+    await ensurePostHasContentToken(post);
+
+    const res = await selectedInstance.reverseSearchToken(instanceSpecificData.contentToken!);
 
     instanceSpecificData.reverseSearchResult = {
       exactPostId: res.exactPost?.id,
@@ -331,33 +323,58 @@ async function loadTagCounts() {
   }
 }
 
-async function fetchPostInfo() {
+async function updatePostWithRemoteInfo(post: ScrapedPostDetails, contentUrl: string) {
+  // We are missing cookies/etc which means that the request might fail.
+  // If you want access to the cookies/session/etc then you need to execute this code inside the content script.
+  try {
+    // This request follows redirects.
+    const res = await fetch(contentUrl, { method: "HEAD" });
+    const size = res.headers.get("Content-Length");
+    const type = res.headers.get("Content-Type");
+
+    if (type) {
+      if (type.indexOf("text/html") != -1) {
+        // If we get a HTML page back it usually means that the request failed.
+        // Not all sites (e.g. e-hentai) reply with a 403.
+        throw new Error(
+          "Received a text/html content type. This probably means that we don't have permission to access the resource.",
+        );
+      }
+
+      const [_main, sub] = type.split("/");
+      if (sub) post.contentSubType = sub.toUpperCase();
+    }
+
+    // This should be after the type check, because the type check also checks whether the response we get is valid/unsable.
+    if (size) post.contentSize = parseInt(size);
+
+    // Update url if it changes. The url can change when:
+    // a. the extraContentUrl was successfully loaded
+    // b. because of redirects
+    if (res.url != post.contentUrl) {
+      // The developer needs to make sure that the remote server can load this URL if `post.uploadMode == 'url'`!
+      console.log(`Updating post.contentUrl to '${res.url}'`);
+      post.contentUrl = res.url;
+    }
+
+    return true;
+  } catch (ex) {
+    console.error(ex);
+    return false;
+  }
+}
+
+async function fetchPostsInfo() {
   for (const post of pop.posts) {
     if (!post.contentSize || post.extraContentUrl) {
-      // We are missing cookies/etc which means that the request might fail.
-      // If you want access to the cookies/session/etc then you need to execute this code inside the content script.
-      try {
-        const contentUrl = post.extraContentUrl ?? post.contentUrl;
-
-        // TODO: Check whether we need to call this in the background page, or if it is fine to call it from the popup.
-        const res = await fetch(contentUrl, { method: "HEAD" });
-        const size = res.headers.get("Content-Length");
-        const type = res.headers.get("Content-Type");
-
-        if (size) post.contentSize = parseInt(size);
-
-        if (type) {
-          const [_main, sub] = type.split("/");
-          if (sub) post.contentSubType = sub.toUpperCase();
-        }
-
-        // Resolve extraContentUrl redirects.
-        if (post.extraContentUrl && res.url != post.contentUrl) {
-          post.contentUrl = res.url;
-        }
-      } catch (ex) {
-        // TODO: Maybe display an error in the UI.
-        console.error(ex);
+      let ok = false;
+      if (post.extraContentUrl) {
+        // Prioritize info from extraContentUrl.
+        ok = await updatePostWithRemoteInfo(post, post.extraContentUrl);
+      }
+      if (!ok) {
+        // Fall back to normal contentUrl if we can't get the data from the extraContentUrl.
+        await updatePostWithRemoteInfo(post, post.contentUrl);
       }
     }
   }
@@ -369,7 +386,8 @@ function getUpdatedTagsText(count: number) {
 }
 
 function onResolutionLoaded(res: any) {
-  if (pop.selectedPost && !pop.selectedPost.resolution) {
+  // This no longer checks for `!pop.selectedPost.resolution` because the resolution can change if the contentUrl is updated.
+  if (pop.selectedPost) {
     pop.selectedPost.resolution = res;
   }
 }