wip: read casc file system

packages/casc/README.md

@@ -0,0 +1,4 @@
+# @diablo2/casc
+
+Pure javascript implementation of CASC file readers
+

packages/casc/package.json

@@ -0,0 +1,28 @@
+{
+  "name": "@diablo2/casc",
+  "version": "0.5.0",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/blacha/diablo2.git",
+    "directory": "packages/casc"
+  },
+  "author": {
+    "name": "Blayne Chard",
+    "email": "[email protected]"
+  },
+  "license": "MIT",
+  "main": "./build/index.js",
+  "types": "./build/index.d.ts",
+  "scripts": {
+    "test": "ospec --globs 'build/**/*.test.js'"
+  },
+  "dependencies": {
+    "binparse": "^1.0.1"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "files": [
+    "build/"
+  ]
+}

packages/casc/src/__test__/file.hash.test.ts

@@ -0,0 +1,24 @@
+import { fileNameHash, fileNameHashString } from '../file.hash';
+import o from 'ospec';
+
+o.spec('FileHash', () => {
+  o('should calculate hashes correctly', () => {
+    o(fileNameHash('data')).equals(3034860415959858194);
+  });
+
+  o('should normalise case', () => {
+    o(fileNameHash('data')).equals(3034860415959858194);
+    o(fileNameHash('DATA')).equals(3034860415959858194);
+    o(fileNameHash('DaTa')).equals(3034860415959858194);
+    o(fileNameHashString('Data')).equals('0x2a1dfd780c2e7c12');
+  });
+
+  o('should replace / with \\', () => {
+    o(fileNameHash('data:data/global')).equals(8030390320118729804);
+    o(fileNameHash('data:data\\global')).equals(8030390320118729804);
+    o(fileNameHashString('data:data/global')).equals('0x6f71ad7306698c4c');
+
+    o(fileNameHash('data:data\\global\\chars\\ba\\la\\balahvynustf.dcc')).equals(4906873520881998204);
+    o(fileNameHashString('data:data\\global\\chars\\ba\\la\\balahvynustf.dcc')).equals('0x4418b77831f1197c');
+  });
+});

packages/casc/src/file.hash.ts

@@ -0,0 +1,22 @@
+import { lookup3 } from './lookup3';
+
+const PathReplace = /\//g;
+/**
+ * Calculate a normalised file hash as a number
+ */
+export function fileNameHash(fileName: string): number {
+  const searchName = fileName.toUpperCase().replace(PathReplace, '\\');
+  const [low, high] = lookup3(Buffer.from(searchName), 0, 0);
+  return high * 0x100000000 + low;
+}
+
+/**
+ * Calculate a normalised file hash as a hex string
+ *
+ * @example 0x4418b77831f1197c
+ */
+export function fileNameHashString(fileName: string): string {
+  const searchName = fileName.toUpperCase().replace(PathReplace, '\\');
+  const [low, high] = lookup3(Buffer.from(searchName), 0, 0);
+  return '0x' + high.toString(16) + low.toString(16).padStart(8, '0');
+}

packages/casc/src/idx.ts

@@ -0,0 +1,51 @@
+import { bp } from 'binparse';
+import { BufferLike } from './lookup3';
+
+/** Default number of idx files */
+export const IdxCount = 16;
+
+/**
+ * Reading *.idx Files
+ */
+
+export const Idx = {
+  header: bp.object('IdxHeader', {
+    headerHashSize: bp.lu32,
+    headerHash: bp.lu32,
+    unk0: bp.lu16,
+    bucketIndex: bp.u8,
+    unk1: bp.u8,
+    entrySizeBytes: bp.u8,
+    offsetBytes: bp.u8,
+    keyBytes: bp.u8,
+    fileHeaderBytes: bp.u8,
+    totalSize: bp.bytes(8),
+    padding: bp.skip(8),
+    entrySize: bp.lu32,
+    entryHash: bp.lu32,
+  }),
+  entry: bp.object('IdexEntry', {
+    key: bp.bytes(8),
+    offset: bp.bytes(5).refine((bytes) => {
+      const byteA = bytes[0] * 0x100000000;
+      const byteB = bytes[1] * 0x1000000;
+      const byteC = bytes[2] * 0x10000;
+      const byteD = bytes[3] * 0x100;
+      const byteE = bytes[4];
+      const bigNumber = byteA + byteB + byteC + byteD + byteE;
+
+      return {
+        archiveNumber: bigNumber >> 30,
+        offset: bigNumber & 0x3fffffff,
+      };
+    }),
+    size: bp.lu32,
+  }),
+};
+
+/** Lookup the idx file number for a hash */
+export function getIdxFileIndex(k: BufferLike): number {
+  const i = k[0] ^ k[1] ^ k[2] ^ k[3] ^ k[4] ^ k[5] ^ k[6] ^ k[7] ^ k[8];
+  const res = (i & 0xf) ^ (i >> 4);
+  return (res + 1) % IdxCount;
+}

packages/casc/src/lookup3.ts

@@ -0,0 +1,88 @@
+export interface BufferLike {
+  [i: number]: number;
+  length: number;
+}
+
+export function lookup3(k: BufferLike, init = 0, init2 = 0): [number, number] {
+  let len = k.length,
+    o = 0,
+    a = (0xdeadbeef + len + init) | 0,
+    b = (0xdeadbeef + len + init) | 0,
+    c = (0xdeadbeef + len + init + init2) | 0;
+
+  while (len > 12) {
+    a += k[o] | (k[o + 1] << 8) | (k[o + 2] << 16) | (k[o + 3] << 24);
+    b += k[o + 4] | (k[o + 5] << 8) | (k[o + 6] << 16) | (k[o + 7] << 24);
+    c += k[o + 8] | (k[o + 9] << 8) | (k[o + 10] << 16) | (k[o + 11] << 24);
+
+    a -= c;
+    a ^= (c << 4) | (c >>> 28);
+    c = (c + b) | 0;
+    b -= a;
+    b ^= (a << 6) | (a >>> 26);
+    a = (a + c) | 0;
+    c -= b;
+    c ^= (b << 8) | (b >>> 24);
+    b = (b + a) | 0;
+    a -= c;
+    a ^= (c << 16) | (c >>> 16);
+    c = (c + b) | 0;
+    b -= a;
+    b ^= (a << 19) | (a >>> 13);
+    a = (a + c) | 0;
+    c -= b;
+    c ^= (b << 4) | (b >>> 28);
+    b = (b + a) | 0;
+
+    (len -= 12), (o += 12);
+  }
+
+  if (len > 0) {
+    // final mix only if len > 0
+    switch (
+      len // incorporate trailing bytes before fmix
+    ) {
+      case 12:
+        c += k[o + 11] << 24;
+      case 11:
+        c += k[o + 10] << 16;
+      case 10:
+        c += k[o + 9] << 8;
+      case 9:
+        c += k[o + 8];
+      case 8:
+        b += k[o + 7] << 24;
+      case 7:
+        b += k[o + 6] << 16;
+      case 6:
+        b += k[o + 5] << 8;
+      case 5:
+        b += k[o + 4];
+      case 4:
+        a += k[o + 3] << 24;
+      case 3:
+        a += k[o + 2] << 16;
+      case 2:
+        a += k[o + 1] << 8;
+      case 1:
+        a += k[o];
+    }
+
+    c ^= b;
+    c -= (b << 14) | (b >>> 18);
+    a ^= c;
+    a -= (c << 11) | (c >>> 21);
+    b ^= a;
+    b -= (a << 25) | (a >>> 7);
+    c ^= b;
+    c -= (b << 16) | (b >>> 16);
+    a ^= c;
+    a -= (c << 4) | (c >>> 28);
+    b ^= a;
+    b -= (a << 14) | (a >>> 18);
+    c ^= b;
+    c -= (b << 24) | (b >>> 8);
+  }
+  // use c as 32-bit hash; add b for 64-bit hash. a is not mixed well.
+  return [b >>> 0, c >>> 0];
+}

packages/casc/tsconfig.json

@@ -0,0 +1,9 @@
+{
+    "extends": "../../tsconfig.base.json",
+    "compilerOptions": {
+        "rootDir": "./src",
+        "outDir": "./build"
+    },
+    "include": ["src/**/*"],
+    "references": [ ]
+}

tsconfig.json

@@ -1,6 +1,7 @@
 {
     "files": [],
     "references": [
+        { "path": "./packages/casc" },
         { "path": "./packages/mpq" },
         { "path": "./packages/data" },
         { "path": "./packages/bintools" },