Using rs-0.40 (pola-rs#211)

* Upgrading to rs-0.4.0 and new toolchain

* Fixing read_csv and other errors

* Addressing PR comments

* Refactoring arguments for read_csv to DF

* Upgrading yarn

* Enabling corepack

* Moving corepack up before setup

* Update src/dataframe.rs

Co-authored-by: Cory Grinstead <[email protected]>

* Uncommenting skipped test

* Adding toDummies

* Adding nodeLinker

---------

Co-authored-by: Cory Grinstead <[email protected]>

Author: Bidek56

.github/workflows/docs.yaml

@@ -14,7 +14,7 @@ jobs:
       - name: Install latest Rust nightly
         uses: dtolnay/rust-toolchain@stable
         with:
-          toolchain: nightly-2024-04-15
+          toolchain: nightly-2024-05-14
           components: rustfmt, clippy
       - name: Install ghp-import
         uses: actions/setup-python@v5

.github/workflows/test-js.yaml

@@ -10,6 +10,8 @@ jobs:
         node: ["18", "20"]
     steps:
       - uses: actions/checkout@v4
+      - name: Enable Corepack
+        run: corepack enable
       - uses: actions/setup-node@v4
         with:
           node-version: ${{ matrix.node }}
@@ -18,9 +20,10 @@ jobs:
       - name: Install latest Rust nightly
         uses: dtolnay/rust-toolchain@stable
         with:
-          toolchain: nightly-2024-04-15
+          toolchain: nightly-2024-05-14
           components: rustfmt, clippy
-      - run: yarn --version
+      - name: Check yarn version
+        run: yarn --version
       - name: Install Node Dependencies
         run: yarn install
       - name: Build Binary
@@ -43,7 +46,7 @@ jobs:
       - name: Install latest Rust nightly
         uses: dtolnay/rust-toolchain@stable
         with:
-          toolchain: nightly-2024-04-15
+          toolchain: nightly-2024-05-14
           components: rustfmt, clippy
       - name: Bun version
         uses: oven-sh/setup-bun@v1

.nvmrc

@@ -1 +1 @@
-16
+20

.yarn/releases/yarn-3.6.1.cjs

@@ -1,3 +1,3 @@
-nodeLinker: node-modules
-
-yarnPath: .yarn/releases/yarn-3.6.1.cjs
+compressionLevel: mixed
+enableGlobalCache: false
+nodeLinker: node-modules

.yarnrc.yml

@@ -15,14 +15,14 @@ crate-type = ["cdylib", "lib"]
 [dependencies]
 ahash = "0.8.11"
 bincode = "1.3.3"
-napi = { version = "2.16.4", default-features = false, features = [
+napi = { version = "2.16.6", default-features = false, features = [
     "napi8",
     "serde-json",
 ] }
-napi-derive = { version = "2.16.3", default-features = false }
-polars-core = { git = "https://github.com/pola-rs/polars.git", rev = "4c57688d204fad3d0d5e4586ecd0405ead7baeb2", default-features = false }
-polars-io = { git = "https://github.com/pola-rs/polars.git", rev = "4c57688d204fad3d0d5e4586ecd0405ead7baeb2", default-features = false }
-polars-lazy = { git = "https://github.com/pola-rs/polars.git", rev = "4c57688d204fad3d0d5e4586ecd0405ead7baeb2", default-features = false }
+napi-derive = { version = "2.16.5", default-features = false }
+polars-core = { git = "https://github.com/pola-rs/polars.git", rev = "7bc70141f4dad7863a2026849522551abb274f00", default-features = false }
+polars-io = { git = "https://github.com/pola-rs/polars.git", rev = "7bc70141f4dad7863a2026849522551abb274f00", default-features = false }
+polars-lazy = { git = "https://github.com/pola-rs/polars.git", rev = "7bc70141f4dad7863a2026849522551abb274f00", default-features = false }
 thiserror = "1"
 smartstring = { version = "1" }
 serde_json = { version = "1" }
@@ -160,7 +160,7 @@ features = [
   "azure"
 ]
 git = "https://github.com/pola-rs/polars.git"
-rev = "4c57688d204fad3d0d5e4586ecd0405ead7baeb2"
+rev = "7bc70141f4dad7863a2026849522551abb274f00"
 
 [build-dependencies]
 napi-build = "2.1.3"

Cargo.toml

@@ -57,9 +57,8 @@ describe("dataframe", () => {
       describe: ["mean", "std", "min", "max", "median"],
       a: [2, 1, 1, 3, 2],
       b: [null, null, "a", "c", null],
-      c: [0.6666666666666666, 0.5773502588272095, 0, 1, 1],
+      c: [0.6666666666666666, 0.5773502691896258, 0, 1, 1],
     });
-
     expect(actual).toFrameEqual(expected);
   });
   test("drop", () => {
@@ -1504,8 +1503,9 @@ describe("join", () => {
     const expected = pl.DataFrame({
       foo: [1, 2, 3, null],
       bar: [6, 7, 8, null],
-      ham: ["a", "b", "c", "d"],
+      ham: ["a", "b", "c", null],
       apple: ["x", null, null, "y"],
+      ham_right: ["a", null, null, "d"],
       foo_right: [1, null, null, 10],
     });
     expect(actual).toFrameEqual(expected);

__tests__/dataframe.test.ts

@@ -36,7 +36,7 @@ describe("lazyframe", () => {
       })
       .lazy();
     let actual = df.describeOptimizedPlan().replace(/\s+/g, " ");
-    const expected = `DF ["foo", "bar"]; PROJECT */2 COLUMNS; SELECTION: "None"`;
+    const expected = `DF ["foo", "bar"]; PROJECT */2 COLUMNS; SELECTION: None`;
     expect(actual).toEqual(expected);
     actual = df.describePlan().replace(/\s+/g, " ");
     expect(actual).toEqual(expected);
@@ -601,11 +601,12 @@ describe("lazyframe", () => {
         })
         .collectSync();
       const expected = pl.DataFrame({
-        foo: [1, 2, 3, null],
-        bar: [6, 7, 8, null],
-        ham: ["a", "b", "c", "d"],
-        apple: ["x", null, null, "y"],
-        fooright: [1, null, null, 10],
+        foo: [1, null, 2, 3],
+        bar: [6, null, 7, 8],
+        ham: ["a", null, "b", "c"],
+        apple: ["x", "y", null, null],
+        hamright: ["a", "d", null, null],
+        fooright: [1, 10, null, null],
       });
       expect(actual).toFrameEqualIgnoringOrder(expected);
     });

__tests__/lazyframe.test.ts

@@ -12,7 +12,7 @@ describe("serde", () => {
     expect(actual).toFrameEqual(expected);
   });
 
-  test.skip("lazyframe:bincode", () => {
+  test("lazyframe:bincode", () => {
     const df = pl.scanCSV(csvpath);
     const buf = df.serialize("bincode");
     const deserde = pl.LazyDataFrame.deserialize(buf, "bincode");

__tests__/serde.test.ts

@@ -664,6 +664,21 @@ describe("series", () => {
       new Uint8Array([1, 2, 3, 5]),
     );
   });
+  test("toDummies", () => {
+    const s = pl.Series("a", [1, 2, 3]);
+    let actual = s.toDummies();
+    let expected = pl.DataFrame(
+      { "a_1.0": [1, 0, 0], "a_2.0": [0, 1, 0], "a_3.0": [0, 0, 1] },
+      { schema: { "a_1.0": pl.UInt8, "a_2.0": pl.UInt8, "a_3.0": pl.UInt8 } },
+    );
+    expect(actual).toFrameEqual(expected);
+    actual = s.toDummies(":", true);
+    expected = pl.DataFrame(
+      { "a:2.0": [0, 1, 0], "a:3.0": [0, 0, 1] },
+      { schema: { "a:2.0": pl.UInt8, "a:3.0": pl.UInt8 } },
+    );
+    expect(actual).toFrameEqual(expected);
+  });
 });
 describe("comparators & math", () => {
   test("add/plus", () => {

__tests__/series.test.ts

@@ -54,20 +54,20 @@
     "precommit": "yarn lint && yarn test"
   },
   "devDependencies": {
-    "@biomejs/biome": "^1.7.0",
-    "@napi-rs/cli": "^2.18.2",
+    "@biomejs/biome": "^1.7.3",
+    "@napi-rs/cli": "^2.18.3",
     "@types/chance": "^1.1.6",
     "@types/jest": "^29.5.12",
-    "@types/node": "^20.12.7",
+    "@types/node": "^20.12.12",
     "chance": "^1.1.11",
     "jest": "^29.7.0",
     "source-map-support": "^0.5.21",
-    "ts-jest": "^29.1.2",
+    "ts-jest": "^29.1.3",
     "ts-node": "^10.9.2",
     "typedoc": "^0.25.13",
     "typescript": "5.4.5"
   },
-  "packageManager": "yarn@4.0.2",
+  "packageManager": "yarn@4.2.2",
   "workspaces": [
     "benches"
   ]

package.json

@@ -932,7 +932,7 @@ export const _Expr = (_expr: any): Expr => {
         throw new Error("window size is required");
       }
       const callOpts = {
-        windowSize: `${windowSize}i`,
+        windowSize: windowSize,
         weights: opts?.["weights"] ?? weights,
         minPeriods: opts?.["minPeriods"] ?? minPeriods ?? windowSize,
         center: opts?.["center"] ?? center ?? false,
@@ -1459,7 +1459,7 @@ export const _Expr = (_expr: any): Expr => {
         "rollingQuantile",
         val.quantile,
         val.interpolation ?? "nearest",
-        `${windowSize}i`,
+        windowSize,
         val?.["weights"] ?? weights ?? null,
         val?.["minPeriods"] ?? minPeriods ?? windowSize,
         val?.["center"] ?? center ?? false,

polars/lazy/expr/index.ts

@@ -375,7 +375,7 @@ export const ExprStringFunctions = (_expr: any): StringNamespace => {
       return wrap("strJsonDecode", dtype, inferSchemaLength);
     },
     jsonPathMatch(pat: string) {
-      return wrap("strJsonPathMatch", pat);
+      return wrap("strJsonPathMatch", [pat]);
     },
     lengths() {
       return wrap("strLengths");

polars/lazy/expr/string.ts

@@ -1040,6 +1040,40 @@ export interface Series
    */
   toTypedArray(): any;
 
+  /**
+   * Get dummy/indicator variables.
+   * @param separator: str = "_", 
+   * @param dropFirst: bool = False
+   * 
+   * @example
+   * const s = pl.Series("a", [1, 2, 3])
+    >>> s.toDummies()
+    shape: (3, 3)
+    ┌─────┬─────┬─────┐
+    │ a_1 ┆ a_2 ┆ a_3 │
+    │ --- ┆ --- ┆ --- │
+    │ u8  ┆ u8  ┆ u8  │
+    ╞═════╪═════╪═════╡
+    │ 1   ┆ 0   ┆ 0   │
+    │ 0   ┆ 1   ┆ 0   │
+    │ 0   ┆ 0   ┆ 1   │
+    └─────┴─────┴─────┘
+
+    >>> s.toDummies(":", true)
+    shape: (3, 2)
+    ┌─────┬─────┐
+    │ a:2 ┆ a:3 │
+    │ --- ┆ --- │
+    │ u8  ┆ u8  │
+    ╞═════╪═════╡
+    │ 0   ┆ 0   │
+    │ 1   ┆ 0   │
+    │ 0   ┆ 1   │
+    └─────┴─────┘
+   * 
+   */
+  toDummies(separator?: string, dropFirst?: boolean): DataFrame;
+
   /**
    * _Returns a Javascript object representation of Series_
    * Often this is much faster than the iterator, or `values` method
@@ -1715,6 +1749,9 @@ export function _Series(_s: any): Series {
       }
       throw new Error("data contains nulls, unable to convert to TypedArray");
     },
+    toDummies(separator = "_", dropFirst = false) {
+      return _DataFrame(_s.toDummies(separator, dropFirst));
+    },
     toFrame() {
       return _DataFrame(new pli.JsDataFrame([_s]));
     },

polars/series/index.ts

@@ -328,7 +328,7 @@ export const SeriesStringFunctions = (_s: any): StringNamespace => {
       return wrap("strJsonDecode", dtype, inferSchemaLength);
     },
     jsonPathMatch(pat: string) {
-      return wrap("strJsonPathMatch", pat);
+      return wrap("strJsonPathMatch", [pat]);
     },
     lengths() {
       return wrap("strLengths");

polars/series/string.ts

@@ -1 +1 @@
-nightly-2024-04-15
+nightly-2024-05-14

rust-toolchain

@@ -498,41 +498,20 @@ impl FromNapiValue for Wrap<u64> {
 
 #[napi(object)]
 pub struct JsRollingOptions {
-    pub window_size: String,
+    pub window_size: i16,
     pub weights: Option<Vec<f64>>,
     pub min_periods: i64,
     pub center: bool,
     pub ddof: Option<u8>,
 }
 
-impl From<JsRollingOptions> for RollingOptionsImpl<'static> {
+impl From<JsRollingOptions> for RollingOptionsFixedWindow {
     fn from(o: JsRollingOptions) -> Self {
-        RollingOptionsImpl {
-            window_size: Duration::parse(&o.window_size),
+        RollingOptionsFixedWindow {
+            window_size: o.window_size as usize,
             weights: o.weights,
             min_periods: o.min_periods as usize,
             center: o.center,
-            by: None,
-            tu: None,
-            tz: None,
-            closed_window: None,
-            fn_params: Some(Arc::new(RollingVarParams {
-                ddof: o.ddof.unwrap_or(1),
-            }) as Arc<dyn Any + Send + Sync>),
-            ..Default::default()
-        }
-    }
-}
-
-impl From<JsRollingOptions> for RollingOptions {
-    fn from(o: JsRollingOptions) -> Self {
-        RollingOptions {
-            window_size: Duration::parse(&o.window_size),
-            weights: o.weights,
-            min_periods: o.min_periods as usize,
-            center: o.center,
-            by: None,
-            closed_window: None,
             fn_params: Some(Arc::new(RollingVarParams {
                 ddof: o.ddof.unwrap_or(1),
             }) as Arc<dyn Any + Send + Sync>),
@@ -550,7 +529,7 @@ pub struct JsRowCount {
 impl From<JsRowCount> for RowIndex {
     fn from(o: JsRowCount) -> Self {
         RowIndex {
-            name: o.name,
+            name: o.name.into(),
             offset: o.offset,
         }
     }
@@ -832,7 +811,7 @@ impl FromNapiValue for Wrap<JoinType> {
         let parsed = match s.as_ref() {
             "inner" => JoinType::Inner,
             "left" => JoinType::Left,
-            "outer" => JoinType::Outer { coalesce: true },
+            "outer" => JoinType::Outer,
             "semi" => JoinType::Semi,
             "anti" => JoinType::Anti,
             "cross" => JoinType::Cross,