feat: python API

.github/workflows/test.yml

@@ -49,6 +49,8 @@ jobs:
 
       - name: Format and lint
         run: |
+          python -m pip install --upgrade pip
+          pip install hatch pytest pytest-cov
           uv run ruff check
           uv run ruff format --check
 
@@ -63,6 +65,12 @@ jobs:
           uv run mypy
           uv run --with pytest-cov pytest --cov-report=term-missing --color=yes --cov=pkg
 
+      - name: Build and test Schema Wrapper
+        run: |
+          cd packages/schema_wrapper
+          pip install -e .
+          pytest test/ --cov=schema_wrapper --cov-report=term-missing
+
   rust:
     name: Test in Rust
 

.gitignore

@@ -14,3 +14,22 @@ packages/*/test-output
 **/.mypy_cache
 **/.coverage
 **/.venv
+transitionFolder
+
+# Virtual environment
+venv/
+.env/
+
+# Conda environment
+conda-env/
+
+# Python cache
+__pycache__/
+*.pyc
+
+# Test cache
+.pytest_cache/
+.coverage
+
+# Python API Generated classes
+packages/schema_wrapper/generated_classes.py

bin/prepare-examples.js

@@ -1,8 +1,14 @@
 #!/usr/bin/env node
-import { basename, extname, join, resolve } from 'node:path';
-import { copyFile, readdir, readFile, writeFile } from 'node:fs/promises';
-import { parseSpec, astToESM } from '@uwdata/mosaic-spec';
-import { parse } from 'yaml';
+import { basename, extname, join, resolve } from "node:path";
+import {
+  copyFile,
+  readdir,
+  readFile,
+  writeFile,
+  mkdir,
+} from "node:fs/promises";
+import { parseSpec, astToESM, astToPython } from "@uwdata/mosaic-spec";
+import { parse } from "yaml";
 
 // This script prepares all Mosaic example specifications
 // ...AND WILL OVERWRITE EXISTING TEST CASE DATA AND DOCS!
@@ -11,86 +17,90 @@ import { parse } from 'yaml';
 // - YAML, non-parsed JSON, and ESM code written to /docs/public/specs
 // - Example Markdown pages written to /docs/examples
 
-const specDir = join('specs', 'yaml');
-const esmTestDir = join('specs', 'esm');
-const jsonTestDir = join('specs', 'json');
-const tsTestDir = join('specs', 'ts');
+const specDir = join("specs", "yaml");
+const esmTestDir = join("specs", "esm");
+const jsonTestDir = join("specs", "json");
+const tsTestDir = join("specs", "ts");
+const pythonTestDir = join("specs", "python");
 
-const docsDir = 'docs';
-const yamlDocsDir = join(docsDir, 'public', 'specs', 'yaml');
-const jsonDocsDir = join(docsDir, 'public', 'specs', 'json');
-const esmDocsDir = join(docsDir, 'public', 'specs', 'esm');
-const exampleDir = join(docsDir, 'examples');
+const docsDir = "docs";
+const yamlDocsDir = join(docsDir, "public", "specs", "yaml");
+const jsonDocsDir = join(docsDir, "public", "specs", "json");
+const esmDocsDir = join(docsDir, "public", "specs", "esm");
+const exampleDir = join(docsDir, "examples");
+const pythonDocsDir = join(docsDir, "public", "specs", "python");
 
-const specToTS = spec => {
+const specToTS = (spec) => {
   return `import { Spec } from '@uwdata/mosaic-spec';
 
 export const spec : Spec = ${JSON.stringify(spec, 0, 2)};
 `;
-}
+};
 
-const files = await Promise.allSettled((await readdir(specDir))
-  .filter(name => extname(name) === '.yaml')
-  .map(async name => {
-    const base = basename(name, '.yaml');
-    const file = resolve(specDir, name);
-    const text = await readFile(file, 'utf8');
+// Create directories if they don't exist
+await Promise.all([
+  mkdir(pythonTestDir, { recursive: true }),
+  mkdir(pythonDocsDir, { recursive: true }),
+]).catch(console.error);
 
-    // parse spec and perform code generation
-    // do this first to catch any errors
-    const spec = parse(text);
-    const ast = parseSpec(spec);
-    const code = astToESM(ast);
+const files = await Promise.allSettled(
+  (
+    await readdir(specDir)
+  )
+    .filter((name) => extname(name) === ".yaml")
+    .map(async (name) => {
+      const base = basename(name, ".yaml");
+      const file = resolve(specDir, name);
+      const text = await readFile(file, "utf8");
 
-    try {
-      await Promise.all([
-        // write ESM DSL spec to tests
-        writeFile(resolve(esmTestDir, `${base}.js`), code),
-        // write JSON spec to tests
-        writeFile(
-          resolve(jsonTestDir, `${base}.json`),
-          JSON.stringify(ast.toJSON(), 0, 2)
-        ),
-        // write TS JSON spec to tests
-        writeFile(
-          resolve(tsTestDir, `${base}.ts`),
-          specToTS(spec)
-        ),
-        // copy YAML file to docs
-        copyFile(file, resolve(yamlDocsDir, `${base}.yaml`)),
-        // write JSON spec to docs
-        writeFile(
-          resolve(jsonDocsDir, `${base}.json`),
-          JSON.stringify(spec, 0, 2)
-        ),
-        // write ESM DSL spec to docs
-        writeFile(resolve(esmDocsDir, `${base}.js`), code),
-        // write examples page to docs
-        writeFile(
-          resolve(exampleDir, `${base}.md`),
-          examplePage(base, spec.meta)
-        )
-      ]);
-    } catch (err) {
-      console.error(err);
-    }
+      // parse spec and perform code generation
+      // do this first to catch any errors
+      const spec = parse(text);
+      const ast = parseSpec(spec);
+      const code = astToESM(ast);
 
-    return base;
-  })
-);
+      try {
+        await Promise.all([
+          // write ESM DSL spec to tests
+          writeFile(resolve(esmTestDir, `${base}.js`), code),
+          // write JSON spec to tests
+          writeFile(
+            resolve(jsonTestDir, `${base}.json`),
+            JSON.stringify(ast.toJSON(), 0, 2)
+          ),
+          // write TS JSON spec to tests
+          writeFile(resolve(tsTestDir, `${base}.ts`), specToTS(spec)),
+          // copy YAML file to docs
+          copyFile(file, resolve(yamlDocsDir, `${base}.yaml`)),
+          // write JSON spec to docs
+          writeFile(
+            resolve(jsonDocsDir, `${base}.json`),
+            JSON.stringify(spec, 0, 2)
+          ),
+          // write ESM DSL spec to docs
+          writeFile(resolve(esmDocsDir, `${base}.js`), code),
+          // write examples page to docs
+          writeFile(
+            resolve(exampleDir, `${base}.md`),
+            examplePage(base, spec.meta)
+          ),
+          // write Python spec to tests
+          writeFile(resolve(pythonTestDir, `${base}.py`), astToPython(ast)),
+          // write Python spec to docs
+          writeFile(resolve(pythonDocsDir, `${base}.py`), astToPython(ast)),
+        ]);
+      } catch (err) {
+        console.error(err);
+      }
 
-// output successfully written examples
-console.log(JSON.stringify(
-  files
-    .filter(x => x.status === 'fulfilled')
-    .map(x => x.value),
-  0, 2
-));
+      return base;
+    })
+);
 
 // output unsuccessful example errors
 files
-  .filter(x => x.status === 'rejected')
-  .forEach(x => console.error(x.reason));
+  .filter((x) => x.status === "rejected")
+  .forEach((x) => console.error(x.reason));
 
 function examplePage(spec, { title = spec, description, credit } = {}) {
   return `<script setup>
@@ -99,9 +109,9 @@ function examplePage(spec, { title = spec, description, credit } = {}) {
 </script>
 
 # ${title}
-${description ? `\n${description.trim()}\n` : ''}
+${description ? `\n${description.trim()}\n` : ""}
 <Example spec="/specs/yaml/${spec}.yaml" />
-${credit ? `\n**Credit**: ${credit}\n` : ''}
+${credit ? `\n**Credit**: ${credit}\n` : ""}
 ## Specification
 
 ::: code-group

docs/public/specs/python/aeromagnetic-survey.py

@@ -0,0 +1,19 @@
+from mosaic import *
+from mosaic.spec import *
+from mosaic.generated_classes import *
+from typing import Dict, Any, Union
+
+
+ca55 = DataSource(
+    type="parquet",
+    file="data/ca55-south.parquet",
+    where=""
+)
+
+spec = Plot(
+    plot=[
+        PlotMark(Raster(mark="raster", data=PlotFrom(from_="ca55"), x=ChannelValueSpec(ChannelValue("LONGITUDE")), y=ChannelValueSpec(ChannelValue("LATITUDE")), fill=ChannelValueSpec(ChannelValue({"max":"MAG_IGRF90"}))))
+    ],
+    width=None,
+    height=None
+)

docs/public/specs/python/airline-travelers.py

@@ -0,0 +1,28 @@
+from mosaic import *
+from mosaic.spec import *
+from mosaic.generated_classes import *
+from typing import Dict, Any, Union
+
+
+travelers = DataSource(
+    type="parquet",
+    file="data/travelers.parquet",
+    where=""
+)
+endpoint = DataSource(
+    type="table",
+    file="undefined",
+    where=""
+)
+
+spec = Plot(
+    plot=[
+        PlotMark(RuleY(mark="ruleY")),
+        PlotMark(LineY(mark="lineY", data=PlotFrom(from_="travelers"), x=ChannelValueSpec(ChannelValue("date")), y=ChannelValueSpec(ChannelValue("previous")), strokeOpacity=0.35)),
+        PlotMark(LineY(mark="lineY", data=PlotFrom(from_="travelers"), x=ChannelValueSpec(ChannelValue("date")), y=ChannelValueSpec(ChannelValue("current")))),
+        PlotMark(Text(mark="text", data=PlotFrom(from_="endpoint"), x=ChannelValueSpec(ChannelValue("date")), y=ChannelValueSpec(ChannelValue("previous")), fillOpacity=0.5, dy=-6)),
+        PlotMark(Text(mark="text", data=PlotFrom(from_="endpoint"), x=ChannelValueSpec(ChannelValue("date")), y=ChannelValueSpec(ChannelValue("current")), dy=6))
+    ],
+    width=None,
+    height=None
+)

docs/public/specs/python/athlete-height.py

@@ -0,0 +1,20 @@
+from mosaic import *
+from mosaic.spec import *
+from mosaic.generated_classes import *
+from typing import Dict, Any, Union
+
+
+athletesBatched = DataSource(
+    type="parquet",
+    file="data/athletes.parquet",
+    where="height IS NOT NULL"
+)
+
+spec = Plot(
+    plot=[
+        PlotMark(ErrorbarX(mark="errorbarX", data=PlotFrom(from_="athletesBatched", filterBy=$query), x=ChannelValueSpec(ChannelValue("height")), y=ChannelValueSpec(ChannelValue("sport")), stroke=ChannelValueSpec(ChannelValue("sex")), strokeWidth=1)),
+        PlotMark(Text(mark="text", data=PlotFrom(from_="athletesBatched"), y=ChannelValueSpec(ChannelValue("sport")), fill=ChannelValueSpec(ChannelValue("#999")), frameAnchor="right"))
+    ],
+    width=None,
+    height=420
+)

docs/public/specs/python/athletes.py

@@ -0,0 +1,22 @@
+from mosaic import *
+from mosaic.spec import *
+from mosaic.generated_classes import *
+from typing import Dict, Any, Union
+
+
+athletes = DataSource(
+    type="parquet",
+    file="data/athletes.parquet",
+    where=""
+)
+
+spec = Plot(
+    plot=[
+        PlotMark(Dot(mark="dot", data=PlotFrom(from_="athletes", filterBy=$query), x=ChannelValueSpec(ChannelValue("weight")), y=ChannelValueSpec(ChannelValue("height")), fill=ChannelValueSpec(ChannelValue("sex")), r=2)),
+        PlotMark(RegressionY(mark="regressionY", data=PlotFrom(from_="athletes", filterBy=$query), x=ChannelValueSpec(ChannelValue("weight")), y=ChannelValueSpec(ChannelValue("height")), stroke=ChannelValueSpec(ChannelValue("sex")))),
+        PlotMark(),
+        PlotMark(Dot(mark="dot", data=PlotFrom(from_="athletes", filterBy=$hover), x=ChannelValueSpec(ChannelValue("weight")), y=ChannelValueSpec(ChannelValue("height")), fill=ChannelValueSpec(ChannelValue("sex")), stroke=ChannelValueSpec(ChannelValue("currentColor")), strokeWidth=1, r=3))
+    ],
+    width=570,
+    height=350
+)

docs/public/specs/python/axes.py

@@ -0,0 +1,19 @@
+from mosaic import *
+from mosaic.spec import *
+from mosaic.generated_classes import *
+from typing import Dict, Any, Union
+
+
+
+spec = Plot(
+    plot=[
+        PlotMark(GridY(mark="gridY", strokeOpacity=1)),
+        PlotMark(AxisY(mark="axisY", dy=-4)),
+        PlotMark(AxisY(mark="axisY")),
+        PlotMark(AxisX(mark="axisX")),
+        PlotMark(GridX(mark="gridX")),
+        PlotMark(RuleY(mark="ruleY"))
+    ],
+    width=680,
+    height=None
+)

docs/public/specs/python/bias.py

@@ -0,0 +1,19 @@
+from mosaic import *
+from mosaic.spec import *
+from mosaic.generated_classes import *
+from typing import Dict, Any, Union
+
+
+walk = DataSource(
+    type="parquet",
+    file="data/random-walk.parquet",
+    where=""
+)
+
+spec = Plot(
+    plot=[
+        PlotMark(AreaY(mark="areaY", data=PlotFrom(from_="walk"), x=ChannelValueSpec(ChannelValue("t")), y=ChannelValueSpec(ChannelValue(sql="v + $point")), fill=ChannelValueSpec(ChannelValue("steelblue"))))
+    ],
+    width=680,
+    height=200
+)

docs/public/specs/python/contours.py

@@ -0,0 +1,21 @@
+from mosaic import *
+from mosaic.spec import *
+from mosaic.generated_classes import *
+from typing import Dict, Any, Union
+
+
+penguins = DataSource(
+    type="parquet",
+    file="data/penguins.parquet",
+    where=""
+)
+
+spec = Plot(
+    plot=[
+        PlotMark(Heatmap(mark="heatmap", data=PlotFrom(from_="penguins"), x=ChannelValueSpec(ChannelValue("bill_length")), y=ChannelValueSpec(ChannelValue("bill_depth")), fill=ChannelValueSpec(ChannelValue("species")))),
+        PlotMark(Contour(mark="contour", data=PlotFrom(from_="penguins"), x=ChannelValueSpec(ChannelValue("bill_length")), y=ChannelValueSpec(ChannelValue("bill_depth")), stroke=ChannelValueSpec(ChannelValue("species")))),
+        PlotMark(Dot(mark="dot", data=PlotFrom(from_="penguins"), x=ChannelValueSpec(ChannelValue("bill_length")), y=ChannelValueSpec(ChannelValue("bill_depth")), fill=ChannelValueSpec(ChannelValue("currentColor")), r=1))
+    ],
+    width=700,
+    height=480
+)