diff --git a/.gitignore b/.gitignore
index 5ffd9c3e..28fda3cb 100644
--- a/.gitignore
+++ b/.gitignore
@@ -8,3 +8,5 @@ dist
!test/reference-test/files/*.parquet
examples/server/package-lock.json
test/browser/*.js
+main.js
+main.js.map
diff --git a/README.md b/README.md
index 749e45a3..11e4982d 100644
--- a/README.md
+++ b/README.md
@@ -484,6 +484,15 @@ var writer = await parquet.ParquetWriter.openFile(schema, 'fruits.parquet');
writer.setRowGroupSize(8192);
```
+## Browser Tests
+
+To run the browser tests (folder: `test/browser`) in a specific browser:
+
+1. `npm i`
+2. `npm run build:browser`
+3. `npx serve .`
+4. `open http://localhost:3000/test/browser/` in your preferred browser (Trailing `/` is required)
+
## Dependencies
Parquet uses [thrift](https://thrift.apache.org/) to encode the schema and other
diff --git a/esbuild-plugins.js b/esbuild-plugins.js
deleted file mode 100644
index a9caefc0..00000000
--- a/esbuild-plugins.js
+++ /dev/null
@@ -1,73 +0,0 @@
-/**
- * this plugin resolves to a browser version of compression.js that
- * does not include LZO or Brötli comprssion.
- */
-const compressionBrowserPlugin = {
- name: 'compressionBrowser',
- setup(build) {
- let path = require('path');
- build.onResolve({ filter: /^\.\/compression$/ }, (_args) => {
- return {
- path: path.resolve(__dirname, 'lib', 'browser', 'compression.js'),
- };
- });
- },
-};
-
-// Lifted from https://esbuild.github.io/plugins/#webassembly-plugin
-const wasmPlugin = {
- name: 'wasm',
- setup(build) {
- let path = require('path');
- let fs = require('fs');
-
- // Resolve ".wasm" files to a path with a namespace
- build.onResolve({ filter: /\.wasm$/ }, (args) => {
- // If this is the import inside the stub module, import the
- // binary itself. Put the path in the "wasm-binary" namespace
- // to tell our binary load callback to load the binary file.
- if (args.namespace === 'wasm-stub') {
- return {
- path: args.path,
- namespace: 'wasm-binary',
- };
- }
-
- // Otherwise, generate the JavaScript stub module for this
- // ".wasm" file. Put it in the "wasm-stub" namespace to tell
- // our stub load callback to fill it with JavaScript.
- //
- // Resolve relative paths to absolute paths here since this
- // resolve callback is given "resolveDir", the directory to
- // resolve imports against.
- if (args.resolveDir === '') {
- return; // Ignore unresolvable paths
- }
- return {
- path: path.isAbsolute(args.path) ? args.path : path.join(args.resolveDir, args.path),
- namespace: 'wasm-stub',
- };
- });
-
- // Virtual modules in the "wasm-stub" namespace are filled with
- // the JavaScript code for compiling the WebAssembly binary. The
- // binary itself is imported from a second virtual module.
- build.onLoad({ filter: /.*/, namespace: 'wasm-stub' }, async (args) => ({
- contents: `import wasm from ${JSON.stringify(args.path)}
- export default (imports) =>
- WebAssembly.instantiate(wasm, imports).then(
- result => result.instance.exports)`,
- }));
-
- // Virtual modules in the "wasm-binary" namespace contain the
- // actual bytes of the WebAssembly file. This uses esbuild's
- // built-in "binary" loader instead of manually embedding the
- // binary data inside JavaScript code ourselves.
- build.onLoad({ filter: /.*/, namespace: 'wasm-binary' }, async (args) => ({
- contents: await fs.promises.readFile(args.path),
- loader: 'binary',
- }));
- },
-};
-
-module.exports = { compressionBrowserPlugin, wasmPlugin };
diff --git a/esbuild-plugins.mjs b/esbuild-plugins.mjs
new file mode 100644
index 00000000..365290b2
--- /dev/null
+++ b/esbuild-plugins.mjs
@@ -0,0 +1,12 @@
+import path from 'node:path';
+/**
+ * this plugin resolves to a browser version of compression.ts that uses different code for browsers
+ */
+export const compressionBrowserPlugin = {
+ name: 'compressionBrowser',
+ setup(build) {
+ build.onResolve({ filter: /^\.\/compression$/ }, (args) => {
+ return { path: path.join(args.resolveDir, args.path.replace('compression', 'browser/compression.ts')) };
+ });
+ },
+};
diff --git a/esbuild-serve.js b/esbuild-serve.mjs
similarity index 71%
rename from esbuild-serve.js
rename to esbuild-serve.mjs
index 343048aa..0359557b 100644
--- a/esbuild-serve.js
+++ b/esbuild-serve.mjs
@@ -3,24 +3,27 @@
* It attaches the parquet.js exports to a "parquetjs" global variable.
* See the example server for how to use it.
*/
-const { compressionBrowserPlugin, wasmPlugin } = require('./esbuild-plugins');
+import { compressionBrowserPlugin } from './esbuild-plugins.mjs';
+import watPlugin from 'esbuild-plugin-wat';
+import esbuild from 'esbuild';
// esbuild has TypeScript support by default. It will use .tsconfig
-require('esbuild')
+esbuild
.context({
entryPoints: ['parquet.ts'],
outfile: 'main.js',
define: { 'process.env.NODE_DEBUG': 'false', 'process.env.NODE_ENV': '"production"', global: 'window' },
platform: 'browser',
- plugins: [compressionBrowserPlugin, wasmPlugin],
+ plugins: [compressionBrowserPlugin, watPlugin()],
sourcemap: 'external',
bundle: true,
+ minify: false,
globalName: 'parquetjs',
- inject: ['./esbuild-shims.js'],
+ inject: ['./esbuild-shims.mjs'],
})
.then((context) => {
context
.serve({
- servedir: __dirname,
+ servedir: './',
})
.then((server) => {
console.log('serving parquetjs', server);
diff --git a/esbuild-shims.js b/esbuild-shims.js
deleted file mode 100644
index c948dc27..00000000
--- a/esbuild-shims.js
+++ /dev/null
@@ -1,2 +0,0 @@
-const buffer = require('buffer/').Buffer;
-export let Buffer = buffer;
diff --git a/esbuild-shims.mjs b/esbuild-shims.mjs
new file mode 100644
index 00000000..cf6459ff
--- /dev/null
+++ b/esbuild-shims.mjs
@@ -0,0 +1,2 @@
+import { Buffer as buffer } from 'buffer/';
+export let Buffer = buffer;
diff --git a/esbuild.js b/esbuild.mjs
similarity index 56%
rename from esbuild.js
rename to esbuild.mjs
index bf2d5250..7c623fc3 100644
--- a/esbuild.js
+++ b/esbuild.mjs
@@ -1,6 +1,6 @@
-const esbuild = require('esbuild');
-const path = require('path');
-const { compressionBrowserPlugin, wasmPlugin } = require('./esbuild-plugins');
+import esbuild from 'esbuild';
+import watPlugin from 'esbuild-plugin-wat';
+import { compressionBrowserPlugin } from './esbuild-plugins.mjs';
// esbuild has TypeScript support by default
const baseConfig = {
bundle: true,
@@ -10,11 +10,11 @@ const baseConfig = {
'process.env.NODE_ENV': '"production"',
global: 'window',
},
- inject: ['./esbuild-shims.js'],
+ inject: ['./esbuild-shims.mjs'],
minify: true,
mainFields: ['browser', 'module', 'main'],
platform: 'browser', // default
- plugins: [compressionBrowserPlugin, wasmPlugin],
+ plugins: [compressionBrowserPlugin, watPlugin()],
target: 'es2020', // default
};
// configuration for generating test code in browser
@@ -26,39 +26,53 @@ const testConfig = {
'process.env.NODE_ENV': '"production"',
global: 'window',
},
- inject: ['./esbuild-shims.js'],
+ inject: ['./esbuild-shims.mjs'],
minify: false,
mainFields: ['browser', 'module', 'main'],
platform: 'browser', // default
- plugins: [compressionBrowserPlugin, wasmPlugin],
+ plugins: [compressionBrowserPlugin, watPlugin()],
target: 'es2020', // default
};
const targets = [
{
...baseConfig,
globalName: 'parquetjs',
- outdir: path.resolve(__dirname, 'dist', 'browser'),
+ outdir: './dist/browser',
},
{
...baseConfig,
format: 'esm',
- outfile: path.resolve(__dirname, 'dist', 'browser', 'parquet.esm.js'),
+ outfile: 'dist/browser/parquet.esm.js',
},
{
...baseConfig,
format: 'cjs',
- outfile: path.resolve(__dirname, 'dist', 'browser', 'parquet.cjs.js'),
+ outfile: 'dist/browser/parquet.cjs.js',
},
- // Browser test code below
+];
+
+// Browser test code below is only in ESM
+const testTargets = [
{
...testConfig,
- outfile: path.resolve(__dirname, 'test', 'browser', 'main.js'),
+ format: 'esm',
+ mainFields: ['module', 'main'],
+ outfile: 'test/browser/main.js',
},
];
+
Promise.all(targets.map(esbuild.build))
.then((results) => {
if (results.reduce((m, r) => m && !r.warnings.length, true)) {
- console.log('built with no errors or warnings');
+ console.log('built dist targets with no errors or warnings');
+ }
+ })
+ .then(() => {
+ return Promise.all(testTargets.map(esbuild.build));
+ })
+ .then((results) => {
+ if (results.reduce((m, r) => m && !r.warnings.length, true)) {
+ console.log('built test targets with no errors or warnings');
}
})
.catch((e) => {
diff --git a/eslint.config.mjs b/eslint.config.mjs
index 7676fb11..2b666d45 100644
--- a/eslint.config.mjs
+++ b/eslint.config.mjs
@@ -10,6 +10,9 @@ export default tseslint.config(
mochaPlugin.configs.flat.recommended,
...tseslint.configs.strict,
...tseslint.configs.stylistic,
+ {
+ ignores: ['dist/*', 'test/browser/main.js'],
+ },
{
rules: {
// TODO: Fix/ignore in tests and remove
diff --git a/examples/server/public/files/gzip-nation.impala.parquet b/examples/server/public/files/gzip-nation.impala.parquet
new file mode 100755
index 00000000..5bbf0d50
Binary files /dev/null and b/examples/server/public/files/gzip-nation.impala.parquet differ
diff --git a/examples/server/public/files/sample_brotli_compressed.parquet b/examples/server/public/files/sample_brotli_compressed.parquet
new file mode 100644
index 00000000..f5d5af8b
Binary files /dev/null and b/examples/server/public/files/sample_brotli_compressed.parquet differ
diff --git a/examples/server/public/files/snappy-compressed.parquet b/examples/server/public/files/snappy-compressed.parquet
new file mode 100644
index 00000000..2d36d52f
Binary files /dev/null and b/examples/server/public/files/snappy-compressed.parquet differ
diff --git a/examples/server/views/parquetFiles.ejs b/examples/server/views/parquetFiles.ejs
index 5477ea28..09866217 100644
--- a/examples/server/views/parquetFiles.ejs
+++ b/examples/server/views/parquetFiles.ejs
@@ -11,31 +11,38 @@
-
-
-
-
-