Fix build in full nix sandbox

.gitignore

@@ -1,3 +1,4 @@
 .next/
 node_modules/
 out/
+result

app/[slug]/page.tsx

@@ -5,7 +5,7 @@ import MarkdownToJSX from 'markdown-to-jsx'
 
 import styles from "~/styles/post.module.css"
 import { ReactElement } from "react"
-import ScriptLoaderServer from "~/components/ScriptLoaderServer"
+import ScriptLoaderServer from "~/components/ScriptLoader/server"
 
 export async function generateMetadata({ params: { slug } }: { params: { slug: string } }): Promise<Metadata> {
   const post = await loadPageMetada(slug)

components/ScriptLoader/client.tsx

@@ -0,0 +1,68 @@
+'use client'
+
+import type { ContentModule, ScriptModule, WasmModuleParam, ResourceFileParam, StringParam } from "./types"
+import { useEffect, useRef } from "react"
+import { themeSignal } from "../Appearance"
+
+interface ScriptLoaderClientParams {
+  src: ScriptModule,
+  wasm: WasmModuleParam | undefined,
+  rest: {
+    [name: string]: ResourceFileParam | StringParam
+  }
+}
+
+export default function ScriptLoaderClient({ src, wasm, rest }: ScriptLoaderClientParams) {
+  const moduleRef = useRef<ContentModule | null>(null)
+
+  useEffect(() => {
+    (async () => {
+      if (!src) return undefined
+
+      const scriptModule : ContentModule = await import(/*webpackIgnore: true */ src.path)
+
+      let wasmModule: WebAssembly.Instance | undefined = undefined
+
+      if (wasm) {
+        const instantiatedWasm = await WebAssembly.instantiateStreaming(fetch(wasm.binaryPath))
+        wasmModule = instantiatedWasm.instance
+      }
+
+      moduleRef.current = scriptModule
+
+      const restProps = Object.keys(rest).reduce((acc: {[name: string]: string}, propName: string) => {
+        const prop = rest[propName]
+        return {
+          ...acc,
+          [propName]: prop.type === 'string' ? prop.value : prop
+        }
+      }, {})
+
+      if (scriptModule.setup && typeof scriptModule.setup === 'function')
+        scriptModule.setup(restProps, wasmModule)
+    })()
+
+    return () => {
+      const mod = moduleRef.current
+
+      if (mod?.cleanup && typeof mod.cleanup === 'function') {
+        mod.cleanup()
+      }
+    }
+  }, [])
+
+  useEffect(() => {
+    function onThemeChange() {
+      const mod = moduleRef.current
+
+      if (mod?.onThemeChange && typeof mod?.onThemeChange === 'function') {
+        mod.onThemeChange()
+      }
+    }
+
+    themeSignal.addListener('change', onThemeChange)
+    return () => { themeSignal.removeListener('change', onThemeChange) }
+  }, [])
+
+  return null
+}

components/ScriptLoader/server.tsx

@@ -0,0 +1,62 @@
+import ScriptLoaderClient from './client'
+
+import defaultTransformer from './transformers/_default'
+import glslTransformer from './transformers/glsl'
+import javascriptTransformer from './transformers/javascript'
+import webassemblyTransformer from './transformers/webassembly'
+
+import type { ResourceFileParam, StringParam, TransformResult } from './types'
+
+interface Props {
+  [propName: string]: string | undefined
+}
+
+const transformers = [
+  javascriptTransformer,
+  webassemblyTransformer,
+  glslTransformer,
+  defaultTransformer
+]
+
+export default async function ScriptServer(props: Props) {
+  let transformedProps: {[name: string] : TransformResult} = {}
+  propLoop: for (const propName in props) {
+    const propValue = props[propName]
+    if (typeof propValue !== 'string')
+      continue
+
+    const extension = propValue.split('.').at(-1)
+
+    for (const transformer of transformers) {
+      if (transformer.extension === '*' || transformer.extension === extension) {
+        transformedProps[propName] = await transformer.transform(propValue)
+        continue propLoop
+      }
+    }
+
+    throw new Error(`Cannot transform prop ${propName} with value ${propValue}`)
+  }
+
+  if (!transformedProps.src) {
+    throw new Error("No script source specified")
+  }
+
+  if (transformedProps.src.type !== 'script') {
+    throw new Error("Script source is not of type script")
+  }
+
+  if (transformedProps.wasm !== undefined && transformedProps.wasm.type !== 'wasm') {
+    throw new Error(`Wasm prop cannot be of type ${transformedProps.wasm.type}`)
+  }
+
+  const {src, wasm, ...rest} = transformedProps
+
+
+  return (
+    <ScriptLoaderClient
+      src={src}
+      wasm={wasm}
+      rest={rest as {[name: string]: StringParam | ResourceFileParam}}
+    />
+  )
+}

components/ScriptLoader/transformers/_default.ts

@@ -0,0 +1,33 @@
+import type { PostResourceTransformer } from '../types'
+
+import path from 'path'
+import { promises as fs } from 'fs'
+
+const transformer : PostResourceTransformer = {
+  extension: "*",
+  transform: async (value) => {
+    try {
+      const argPath = path.join(process.cwd(), 'scripts', value)
+      const argContents = await fs.readFile(argPath)
+      const argFileName = path.basename(value)
+      const argDest = path.join(process.cwd(), '.next/static/scripts', value)
+      const argDir = path.dirname(argDest)
+      await fs.mkdir(argDir, { recursive: true })
+      await fs.writeFile(argDest, argContents)
+
+      return {
+        type: "resource",
+        name: argFileName,
+        path: path.join('/_next/static/scripts', value)
+      }
+    } catch {
+      return {
+        type: 'string',
+        value
+      }
+    }
+  }
+}
+
+export default transformer
+

components/ScriptLoader/transformers/glsl.ts

@@ -0,0 +1,33 @@
+import { PostResourceTransformer } from "../types"
+
+import path from 'path'
+import { promises as fs } from 'fs'
+import glslx from 'glslx'
+
+const transformer : PostResourceTransformer = {
+  extension: "glsl",
+  transform: async (value) => {
+    const argPath = path.join(process.cwd(), 'scripts', value)
+    const argContents = await fs.readFile(argPath)
+    const argFileName = path.basename(value)
+
+    const {output: glslResult, log} = glslx.compile({name: argFileName, contents: argContents.toString('utf8')})
+    if (!glslResult) {
+      console.error(log)
+      throw new Error("Could not parse GLSL file")
+    }
+
+    const argDest = path.join(process.cwd(), '.next/static/scripts', value)
+    const argDir = path.dirname(argDest)
+    await fs.mkdir(argDir, { recursive: true })
+    await fs.writeFile(argDest, argContents)
+
+    return {
+      type: "resource",
+      name: argFileName,
+      path: path.join('/_next/static/scripts', value)
+    }
+  }
+}
+
+export default transformer

components/ScriptLoader/transformers/javascript.ts

@@ -0,0 +1,25 @@
+import type { PostResourceTransformer } from '../types'
+
+import path from 'path'
+import { promises as fs } from 'fs'
+
+const transformer : PostResourceTransformer = {
+  extension: "js",
+  transform: async (value) => {
+    const scriptPath = path.join(process.cwd(), 'scripts', value)
+    const scriptContents = await fs.readFile(scriptPath)
+    const scriptFileName = path.basename(value)
+    const scriptDest = path.join(process.cwd(), '.next/static/scripts', value)
+    const destDir = path.dirname(scriptDest)
+    await fs.mkdir(destDir, { recursive: true })
+    await fs.writeFile(scriptDest, scriptContents)
+
+    return {
+      type: 'script',
+      name: scriptFileName,
+      path: path.join('/_next/static/scripts', value)
+    }
+  }
+}
+
+export default transformer

components/ScriptLoader/transformers/webassembly.ts

@@ -0,0 +1,37 @@
+import type { PostResourceTransformer } from '../types'
+
+import path from 'path'
+import { promises as fs } from 'fs'
+import wabtModule from 'wabt'
+
+const wabtModulePromise = wabtModule()
+
+const transformer : PostResourceTransformer = {
+  extension: "wat",
+  transform: async (src) => {
+    const wasmPath = path.join(process.cwd(), 'scripts', src)
+    const wasmContents = await fs.readFile(wasmPath)
+
+    const wasmTextFileName = path.basename(src)
+    const wasmBinFileName = wasmTextFileName.replace(/\.wat$/, '.wasm')
+    const wabt = await wabtModulePromise
+
+    const wasmParsed = wabt.parseWat(wasmTextFileName, wasmContents)
+    const { buffer: wasmBinary } = wasmParsed.toBinary({ write_debug_names: true })
+    const wasmDestText = path.join(process.cwd(), '.next/static/scripts', src)
+    const wasmDestBinary = wasmDestText.replace(/\.wat$/, '.wasm')
+    const destDir = path.dirname(wasmDestText)
+    await fs.mkdir(destDir, { recursive: true })
+    await fs.writeFile(wasmDestText, wasmContents)
+    await fs.writeFile(wasmDestBinary, wasmBinary)
+
+    return {
+      type: 'wasm',
+      name: wasmBinFileName,
+      binaryPath: path.join('/_next/static/scripts/', src.replace(/\.wat$/, '.wasm')),
+      textPath: path.join('/_next/static/scripts/', src),
+    }
+  }
+}
+
+export default transformer

components/ScriptLoader/types.ts

@@ -0,0 +1,46 @@
+/**
+ * Content Script Interfaces
+ */
+export interface ContentModule {
+  setup: (args: ContentModuleSetupParams, wasm?: WebAssembly.Instance) => Promise<undefined>,
+  cleanup?: () => Promise<undefined>,
+  onThemeChange?: () => void,
+}
+
+export interface ContentModuleSetupParams {
+  [argName: string]: ResourceFileParam | string | undefined
+}
+
+export interface ScriptModule {
+  type: 'script',
+  name: string,
+  path: string,
+}
+
+export interface WasmModuleParam {
+  type: 'wasm',
+  name: string,
+  binaryPath: string,
+  textPath: string,
+}
+
+export interface ResourceFileParam {
+  type: 'resource',
+  name: string,
+  path: string,
+}
+
+export interface StringParam {
+  type: 'string',
+  value: string,
+}
+
+export type TransformResult = ScriptModule | WasmModuleParam | ResourceFileParam | StringParam
+
+/**
+ * Resource transformer interfaces
+ */
+export interface PostResourceTransformer {
+  extension: string,
+  transform: (value: string) => Promise<TransformResult>
+}

components/ScriptLoaderClient.tsx

@@ -1,58 +0,0 @@
-'use client'
-
-import { useEffect, useRef } from "react"
-import { themeSignal } from "./Appearance"
-
-interface Module {
-  setup?: (params: any, wasmModule: WebAssembly.Instance | undefined) => Promise<undefined>
-  cleanup?: () => Promise<undefined>
-  onThemeChange?: () => void
-}
-
-export default function ScriptLoaderClient({ src, wasmSrc, ...rest }: { src: string, wasmSrc?: string }) {
-  const moduleRef = useRef<Module | null>(null)
-
-  useEffect(() => {
-    if (!src) return undefined;
-
-    (async () => {
-      const scriptModule = await import(/*webpackIgnore: true */ src)
-
-      let wasmModule: WebAssembly.Instance | undefined = undefined;
-
-      if (wasmSrc) {
-        const wasm = await WebAssembly.instantiateStreaming(fetch(wasmSrc))
-        wasmModule = wasm.instance
-      }
-
-      moduleRef.current = scriptModule
-
-
-      if (scriptModule.setup && typeof scriptModule.setup === 'function')
-        scriptModule.setup(rest, wasmModule)
-    })();
-
-    return () => {
-      const mod = moduleRef.current
-
-      if (mod?.cleanup && typeof mod.cleanup === 'function') {
-        mod.cleanup()
-      }
-    }
-  }, [])
-
-  useEffect(() => {
-    function onThemeChange() {
-      const mod = moduleRef.current
-
-      if (mod?.onThemeChange && typeof mod?.onThemeChange === 'function') {
-        mod.onThemeChange()
-      }
-    }
-
-    themeSignal.addListener('change', onThemeChange)
-    return () => { themeSignal.removeListener('change', onThemeChange) }
-  }, [])
-
-  return null
-}

components/ScriptLoaderServer.tsx

@@ -1,95 +0,0 @@
-import path from 'path'
-import { promises as fs } from 'fs'
-import wabtModule from 'wabt'
-import ScriptLoaderClient from './ScriptLoaderClient'
-
-interface Props {
-  src: string,
-  wasm?: string,
-}
-
-export interface ScriptFile {
-  name: string,
-  path: string,
-}
-
-const wabtModulePromise = wabtModule()
-
-export default async function ScriptServer({ src, wasm, ...rest }: Props) {
-  if (!src)
-    throw new Error("ScriptLoader: No src parameter")
-
-  const scriptPath = path.join(process.cwd(), 'scripts', src)
-  const wasmPath = wasm && path.join(process.cwd(), 'scripts', wasm)
-
-  const scriptContents = await fs.readFile(scriptPath)
-  const wasmContents = wasmPath && await fs.readFile(wasmPath)
-
-  let script: ScriptFile | undefined = undefined;
-
-  if (scriptContents) {
-    const scriptFileName = path.basename(src)
-    const scriptDest = path.join(process.cwd(), '.next/static/scripts', src)
-    const destDir = path.dirname(scriptDest)
-    await fs.mkdir(destDir, { recursive: true })
-    await fs.writeFile(scriptDest, scriptContents)
-    script = {
-      name: scriptFileName,
-      path: path.join('/_next/static/scripts', src)
-    }
-  }
-
-  let wasmCompiled: ScriptFile | undefined = undefined;
-  if (wasmContents) {
-    const wasmTextFileName = path.basename(wasm)
-    const wasmBinFileName = wasmTextFileName.replace(/\.wat$/, '.wasm')
-    const wabt = await wabtModulePromise
-
-    const wasmParsed = wabt.parseWat(wasmTextFileName, wasmContents)
-    const { buffer: wasmBinary } = wasmParsed.toBinary({ write_debug_names: true })
-    const wasmDestText = path.join(process.cwd(), '.next/static/scripts', wasm)
-    const wasmDestBinary = wasmDestText.replace(/\.wat$/, '.wasm')
-    const destDir = path.dirname(wasmDestText)
-    await fs.mkdir(destDir, { recursive: true })
-    await fs.writeFile(wasmDestText, wasmContents)
-    await fs.writeFile(wasmDestBinary, wasmBinary)
-
-    wasmCompiled = {
-      name: wasmBinFileName,
-      path: path.join('/_next/static/scripts/', wasm.replace(/\.wat$/, '.wasm'))
-    }
-  }
-
-  let processedArgs = {}
-  for (const argName in rest) {
-    const argValue = rest[argName]
-
-    try {
-      const argPath = path.join(process.cwd(), 'scripts', argValue)
-      const argContents = await fs.readFile(argPath)
-      const argFileName = path.basename(argValue)
-      const argDest = path.join(process.cwd(), '.next/static/scripts', argValue)
-      const argDir = path.dirname(argDest)
-      await fs.mkdir(argDir, { recursive: true })
-      await fs.writeFile(argDest, argContents)
-      processedArgs[argName] = {
-        name: argFileName,
-        path: path.join('/_next/static/scripts', argValue)
-      }
-    } catch {
-      processedArgs[argName] = argValue
-    }
-  }
-
-  if (!script) return null
-
-  return (
-    <ScriptLoaderClient
-      src={script.path}
-      {...(wasmCompiled ? {
-        wasmSrc: wasmCompiled.path
-      } : {})}
-      {...processedArgs}
-    />
-  )
-}

flake.nix

@@ -19,28 +19,51 @@
 
       filter = nix-filter.lib;
 
-      nodejs = pkgs.nodejs-18_x;
+      nodejs = pkgs.nodejs_20;
+
+      src = filter {
+        root = ./.;
+        include = [
+          ./package.json
+          ./package-lock.json
+        ];
+      };
 
-      node_modules = pkgs.stdenv.mkDerivation {
-        name = "node_modules";
+      # Read lockfile
+      packageLock = builtins.fromJSON(builtins.readFile (src + "/package-lock.json"));
+
+      # Convert to tarball dep list
+      deps = builtins.attrValues (removeAttrs packageLock.packages [ "" ])
+        ++ builtins.attrValues (removeAttrs (packageLock.dependencies or {} ) [ "" ])
+      ;
+      depTarballs = map (p: pkgs.fetchurl { url = p.resolved; hash = p.integrity; }) deps;
+      tarballsFile = pkgs.writeTextFile {
+        name = "tarballs";
+        text = (builtins.concatStringsSep "\n" depTarballs) + "\n";
+      };
 
-        src = filter {
-          root = ./.;
-          include = [
-            ./package.json
-            ./package-lock.json
-          ];
-        };
+      # build npm cache
+      ashenearth_modules = pkgs.stdenv.mkDerivation {
+        inherit src;
 
-        __noChroot = true;
+        name = "ashenearth_nodemodules";
 
-        configurePhase = ''
-          export HOME=$TMP
-        '';
+        nativeBuildInputs = [
+          nodejs
+        ];
 
         buildInputs = [ nodejs ];
 
         buildPhase = ''
+          export HOME=$PWD/.home
+          export npm_config_cache=$PWD/.npm
+
+          while read package
+          do
+            echo "caching $(echo $package | sed 's/\/nix\/store\/[^-]*-//')"
+            npm cache add "$package"
+          done <${tarballsFile}
+
           ${nodejs}/bin/npm ci
         '';
 
@@ -70,7 +93,7 @@
           '';
 
           configurePhase = ''
-            ln -sf ${node_modules}/node_modules node_modules
+            ln -sf ${ashenearth_modules}/node_modules node_modules
             export HOME=$TMP
           '';
 

package.json

@@ -12,14 +12,16 @@
     "bright": "^0.8.4",
     "front-matter": "^4.0.2",
     "markdown-to-jsx": "^7.2.0",
-    "next": "^13.4.1",
+    "next": "^13.5.3",
     "react": "^18.2.0",
-    "react-dom": "^18.2.0",
-    "wabt": "1.0.19"
+    "react-dom": "^18.2.0"
   },
   "devDependencies": {
     "@types/node": "20.1.0",
     "@types/react": "^18.2.6",
-    "typescript": "5.0.4"
+    "glslx": "^0.3.0",
+    "swc": "^1.0.11",
+    "typescript": "5.0.4",
+    "wabt": "1.0.19"
   }
 }

posts/2023-09-05_wasm-game-of-life-2.md

@@ -78,7 +78,7 @@ in the table below:
 > | WebGL Drawing | 44.8ms | 0.4ms | 22.1 fps |
 > | Major WASM Changes | 11.8ms | 0.4ms | 81.9\* fps |
 > 
-> _\* This computed framerate is greater than my monitor can display, so the browser caps is to 60 fps_
+> _\* This computed framerate is greater than my monitor can display, so the browser caps it to 60 fps_
 
 Right off the bat, there were no simulation / game tick changes between the second and
 third rows, but we do see a change in the *measured* tick time - this indicates that my measurement method has **at least** a millisecond
@@ -147,8 +147,7 @@ void main() {
 
 There's a correction factor here (multiplying the value by 255), but that's because
 my board data had only 0s and 1s in the bytes for indicating an alive or dead
-cell.  (And on the GPU this kind of data transformation is *very* fast, which is
-why I didn't first correct that in Javascript).
+cell, and the shader expects that texture value to have a range between 0 and 255 (the range of a byte).
 
 The shader also takes two `uniform` parameters for what colors to draw with, so
 I will show you where those get set:
@@ -187,7 +186,7 @@ So I don't know that much about how Webassembly execution is implemented into
 browsers, but I know enough about actual CPU architectures to make some reasonable
 inferences.
 
-With that in mind, as I started thinking about more significant algorithm changes
+As I started thinking about more significant algorithm changes
 I could make, I suspected that the largest contributor to time taken in my old
 algorithm was probably from it taking longer to access the Webassembly linear memory
 than local or global variables.  At an implementation level this could be because
@@ -195,7 +194,7 @@ of how the wasm memory relates to the CPU cache, but I didn't really care about
  \- I was just curious to see if reducing the number of memory operations my algorithm
 took could provide me more of a speed-up.
 
-I knew I was already pretty inefficient in this areay - my previous algorithm had made ***10 memory calls per cell***
+I knew I was already pretty inefficient in this area - my previous algorithm had made ***10 memory calls per cell***
 (8 to check neighbor states, 1 to check the current cell's state, and 1 to store
 the updated cell's state), so I suspected I could get that much lower.
 

utils/post.ts

@@ -1,6 +1,5 @@
 import { promises as fs } from 'fs'
 import path from 'path'
-import wabtModule from 'wabt'
 
 import frontmatter from 'front-matter'
 
@@ -42,7 +41,7 @@ export async function loadPageMetada(slug: string): Promise<PostMeta | null> {
     .filter((regexResult: RegExpExecArray | null) => regexResult !== null)
     .find((regexResult: RegExpExecArray) => regexResult.groups?.slug === slug)
 
-  if (!postMatch) return null;
+  if (!postMatch) return null
 
   const fileName = `${postMatch.groups?.date}_${postMatch.groups?.slug}.md`
   const filePath = path.join(process.cwd(), 'posts', fileName)
@@ -50,7 +49,7 @@ export async function loadPageMetada(slug: string): Promise<PostMeta | null> {
   const { attributes } = frontmatter(fileContents)
   const data: Attributes = attributes as Attributes
 
-  if (!data.title) return null;
+  if (!data.title) return null
 
   const [year, month, day] = postMatch.groups?.date.split('-') || []
   const date = new Date(Date.UTC(parseInt(year), parseInt(month) - 1, parseInt(day)))
@@ -72,8 +71,7 @@ export async function loadSinglePage(slug: string): Promise<Post | null> {
   if (!postMeta) return null
 
   const fileContents = (await fs.readFile(postMeta.filePath)).toString('utf8')
-  const { attributes, body } = frontmatter(fileContents)
-  const data: Attributes = attributes as Attributes
+  const { body } = frontmatter(fileContents)
 
   const { filePath, ...otherMeta } = postMeta