Skip to content

Commit 91fc71f

Browse files
wilwadewilwade
authored
Update Deprecated Function Calls (#118)
- Buffer.slice -> Buffer.subarray (and correct test that wasn't using buffers) - new Buffer(array) -> Buffer.from(array) - Fix issue with `npm run serve` Via looking into #117 As `subarray` is slightly faster in the browser shim.
1 parent 117e5a5 commit 91fc71f

9 files changed

+94
-93
lines changed

‎esbuild-serve.js

Lines changed: 18 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -3,21 +3,22 @@
33
* It attaches the parquet.js exports to a "parquetjs" global variable.
44
* See the example server for how to use it.
55
*/
6-
const {compressionBrowserPlugin, wasmPlugin} = require("./esbuild-plugins");
6+
const { compressionBrowserPlugin, wasmPlugin } = require("./esbuild-plugins");
77
// esbuild has TypeScript support by default. It will use .tsconfig
8-
require('esbuild')
9-
.serve({
10-
servedir: __dirname,
11-
}, {
12-
entryPoints: ['parquet.ts'],
13-
outfile: 'main.js',
14-
define: {"process.env.NODE_DEBUG": "false", "process.env.NODE_ENV": "\"production\"", global: "window" },
15-
platform: 'browser',
16-
plugins: [compressionBrowserPlugin,wasmPlugin],
17-
sourcemap: "external",
18-
bundle: true,
19-
globalName: 'parquetjs',
20-
inject: ['./esbuild-shims.js']
21-
}).then(server => {
22-
console.log("serving parquetjs", server)
23-
})
8+
require('esbuild').context({
9+
entryPoints: ['parquet.ts'],
10+
outfile: 'main.js',
11+
define: { "process.env.NODE_DEBUG": "false", "process.env.NODE_ENV": "\"production\"", global: "window" },
12+
platform: 'browser',
13+
plugins: [compressionBrowserPlugin, wasmPlugin],
14+
sourcemap: "external",
15+
bundle: true,
16+
globalName: 'parquetjs',
17+
inject: ['./esbuild-shims.js']
18+
}).then(context => {
19+
context.serve({
20+
servedir: __dirname,
21+
}).then(server => {
22+
console.log("serving parquetjs", server)
23+
})
24+
})

‎examples/server/README.md

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
# Example Server
22
This is a toy server that illustrates how to use the parquetjs library built with esbuild.
3-
To run it:
3+
To run it:
44
1. npm install
5-
1. View and edit the files in `views` to taste
5+
1. View and edit the files in `views` to taste
66
1. node app.js
7-
1. Build and serve the parquetjs bundle in the main parquetjs directory: `npm run serve`
7+
1. Build and serve the parquetjs bundle in the main parquetjs directory: `npm run serve`
88
1. visit `http://localhost:3000` and click buttons, do things in the console.

‎lib/bufferReader.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -72,7 +72,7 @@ export default class BufferReader {
7272
const buffer = await this.envelopeReader.readFn(start, finish - start);
7373

7474
processQueue.forEach(async d => {
75-
d.resolve(buffer.slice(d.offset - start, d.offset + d.length - start));
75+
d.resolve(buffer.subarray(d.offset - start, d.offset + d.length - start));
7676
});
7777
};
7878

‎lib/codec/plain.ts

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -205,7 +205,6 @@ function decodeValues_DOUBLE(cursor: Cursor, count: number) {
205205
return values;
206206
}
207207

208-
// Waylands reminder to check again
209208
function encodeValues_BYTE_ARRAY(values: Array<Uint8Array>) {
210209
let buf_len = 0;
211210
const returnedValues: Array<Buffer> = [];
@@ -231,7 +230,7 @@ function decodeValues_BYTE_ARRAY(cursor: Cursor, count: number) {
231230
for (let i = 0; i < count; ++i) {
232231
let len = cursor.buffer.readUInt32LE(cursor.offset);
233232
cursor.offset += 4;
234-
values.push(cursor.buffer.slice(cursor.offset, cursor.offset + len));
233+
values.push(cursor.buffer.subarray(cursor.offset, cursor.offset + len));
235234
cursor.offset += len;
236235
}
237236

@@ -272,7 +271,7 @@ function decodeValues_FIXED_LEN_BYTE_ARRAY(
272271

273272
for (let i = 0; i < count; ++i) {
274273
values.push(
275-
cursor.buffer.slice(cursor.offset, cursor.offset + typeLength)
274+
cursor.buffer.subarray(cursor.offset, cursor.offset + typeLength)
276275
);
277276
cursor.offset += typeLength;
278277
}

‎lib/codec/plain_dictionary.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ import * as rle from './rle'
22
import { Cursor, Options } from './types'
33

44
export const decodeValues = function(type: string, cursor: Cursor, count: number, opts: Options) {
5-
const bitWidth = cursor.buffer.slice(cursor.offset, cursor.offset+1).readInt8(0);
5+
const bitWidth = cursor.buffer.subarray(cursor.offset, cursor.offset+1).readInt8(0);
66
cursor.offset += 1;
77
return rle.decodeValues(type, cursor, count, Object.assign({}, opts, { disableEnvelope: true, bitWidth }));
88
};

‎lib/reader.ts

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -441,7 +441,7 @@ export class ParquetEnvelopeReader {
441441
return Promise.reject('external references are not supported');
442442
}
443443

444-
return Promise.resolve(buffer.slice(offset, offset + length));
444+
return Promise.resolve(buffer.subarray(offset, offset + length));
445445
};
446446

447447
let closeFn = () => ({});
@@ -503,7 +503,7 @@ export class ParquetEnvelopeReader {
503503
if (blob.arrayBuffer !== undefined) {
504504
const arrayBuffer = await blob.arrayBuffer();
505505
const uint8Array: Uint8Array = new Uint8Array(arrayBuffer);
506-
return new Buffer(uint8Array);
506+
return Buffer.from(uint8Array);
507507
}
508508

509509
//Assumed to be a Readable like object
@@ -834,7 +834,7 @@ async function decodePage(cursor: Cursor, opts: Options): Promise<PageData> {
834834
const pageHeader = new NewPageHeader();
835835

836836
const headerOffset = cursor.offset;
837-
const headerSize = parquet_util.decodeThrift(pageHeader, cursor.buffer.slice(cursor.offset));
837+
const headerSize = parquet_util.decodeThrift(pageHeader, cursor.buffer.subarray(cursor.offset));
838838
cursor.offset += headerSize;
839839

840840
const pageType = parquet_util.getThriftEnum(
@@ -926,7 +926,7 @@ async function decodeDictionaryPage(cursor: Cursor, header: parquet_thrift.PageH
926926

927927
let dictCursor = {
928928
offset: 0,
929-
buffer: cursor.buffer.slice(cursor.offset,cursorEnd),
929+
buffer: cursor.buffer.subarray(cursor.offset,cursorEnd),
930930
size: cursorEnd - cursor.offset
931931
};
932932

@@ -935,7 +935,7 @@ async function decodeDictionaryPage(cursor: Cursor, header: parquet_thrift.PageH
935935
if (opts.compression && opts.compression !== 'UNCOMPRESSED') {
936936
let valuesBuf = await parquet_compression.inflate(
937937
opts.compression,
938-
dictCursor.buffer.slice(dictCursor.offset,cursorEnd));
938+
dictCursor.buffer.subarray(dictCursor.offset,cursorEnd));
939939

940940
dictCursor = {
941941
buffer: valuesBuf,
@@ -963,7 +963,7 @@ async function decodeDataPage(cursor: Cursor, header: parquet_thrift.PageHeader,
963963
if (opts.compression && opts.compression !== 'UNCOMPRESSED') {
964964
let valuesBuf = await parquet_compression.inflate(
965965
opts.compression,
966-
cursor.buffer.slice(cursor.offset, cursorEnd));
966+
cursor.buffer.subarray(cursor.offset, cursorEnd));
967967

968968
valuesBufCursor = {
969969
buffer: valuesBuf,
@@ -1090,7 +1090,7 @@ async function decodeDataPageV2(cursor: Cursor, header: parquet_thrift.PageHeade
10901090
if (dataPageHeaderV2.is_compressed) {
10911091
let valuesBuf = await parquet_compression.inflate(
10921092
opts.compression!,
1093-
cursor.buffer.slice(cursor.offset, cursorEnd));
1093+
cursor.buffer.subarray(cursor.offset, cursorEnd));
10941094

10951095
valuesBufCursor = {
10961096
buffer: valuesBuf,

‎test/bloomFilterIntegration.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ const sampleColumnHeaders = async (filename: string) => {
3232

3333
while (cursor.offset < cursor.size) {
3434
const pageHeader = new parquet_thrift.PageHeader();
35-
cursor.offset += decodeThrift(pageHeader, cursor.buffer.slice(cursor.offset));
35+
cursor.offset += decodeThrift(pageHeader, cursor.buffer.subarray(cursor.offset));
3636
pages.push(pageHeader);
3737
cursor.offset += pageHeader.compressed_page_size;
3838
}

‎test/integration.js

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -152,7 +152,7 @@ async function sampleColumnHeaders() {
152152

153153
while (cursor.offset < cursor.size) {
154154
const pageHeader = new parquet_thrift.PageHeader();
155-
cursor.offset += parquet_util.decodeThrift(pageHeader, cursor.buffer.slice(cursor.offset));
155+
cursor.offset += parquet_util.decodeThrift(pageHeader, cursor.buffer.subarray(cursor.offset));
156156
pages.push(pageHeader);
157157
cursor.offset += pageHeader.compressed_page_size;
158158
}

‎test/lib/bufferReader.test.js

Lines changed: 60 additions & 59 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
import chai, {expect} from "chai"
1+
import chai, { expect } from "chai"
22
import sinon from "sinon"
33
import sinonChai from "sinon-chai";
44
import sinonChaiInOrder from 'sinon-chai-in-order';
@@ -30,7 +30,7 @@ describe("bufferReader", () => {
3030
it("only enqueues an item and reads on flushing the queue", async () => {
3131
const mockResolve = sinon.spy();
3232
const mockResolve2 = sinon.spy();
33-
reader.envelopeReader = {readFn: sinon.fake.returns("buffer")}
33+
reader.envelopeReader = { readFn: sinon.fake.returns(Buffer.from("buffer", "utf8")) }
3434

3535
reader.queue = [{
3636
offset: 1,
@@ -44,15 +44,16 @@ describe("bufferReader", () => {
4444

4545
await reader.processQueue();
4646

47-
sinon.assert.calledWith(mockResolve, "b")
48-
sinon.assert.calledWith(mockResolve2, "uffe")
47+
48+
sinon.assert.calledWith(mockResolve, Buffer.from("b", "utf8"));
49+
sinon.assert.calledWith(mockResolve2, Buffer.from("uffe", "utf8"));
4950
})
5051

5152
it("enqueues items and then reads them", async () => {
5253
const mockResolve = sinon.spy();
5354
const mockResolve2 = sinon.spy();
5455
reader.maxLength = 1;
55-
reader.envelopeReader = {readFn: sinon.fake.returns("buffer")}
56+
reader.envelopeReader = { readFn: sinon.fake.returns(Buffer.from("buffer", "utf8")) }
5657

5758
reader.queue = [{
5859
offset: 1,
@@ -66,81 +67,81 @@ describe("bufferReader", () => {
6667

6768
await reader.processQueue();
6869

69-
sinon.assert.calledWith(mockResolve, "b")
70-
sinon.assert.calledWith(mockResolve2, "uffe")
70+
sinon.assert.calledWith(mockResolve, Buffer.from("b", "utf8"));
71+
sinon.assert.calledWith(mockResolve2, Buffer.from("uffe", "utf8"));
7172
})
7273

7374
it("enqueues items and reads them in order", async () => {
7475
const mockResolve = sinon.spy();
75-
reader.envelopeReader = {readFn: sinon.fake.returns("thisisalargebuffer")}
76+
reader.envelopeReader = { readFn: sinon.fake.returns(Buffer.from("thisisalargebuffer", "utf8")) }
7677

7778
reader.queue = [{
78-
offset: 1,
79-
length: 4,
80-
resolve: mockResolve,
81-
}, {
82-
offset: 5,
83-
length: 2,
84-
resolve: mockResolve,
85-
}, {
86-
offset: 7,
87-
length: 1,
88-
resolve: mockResolve,
89-
}, {
90-
offset: 8,
91-
length: 5,
92-
resolve: mockResolve,
93-
}, {
94-
offset: 13,
95-
length: 6,
96-
resolve: mockResolve,
97-
}
79+
offset: 1,
80+
length: 4,
81+
resolve: mockResolve,
82+
}, {
83+
offset: 5,
84+
length: 2,
85+
resolve: mockResolve,
86+
}, {
87+
offset: 7,
88+
length: 1,
89+
resolve: mockResolve,
90+
}, {
91+
offset: 8,
92+
length: 5,
93+
resolve: mockResolve,
94+
}, {
95+
offset: 13,
96+
length: 6,
97+
resolve: mockResolve,
98+
}
9899
];
99100

100101
await reader.processQueue();
101102

102-
expect(mockResolve).inOrder.to.have.been.calledWith("this")
103-
.subsequently.calledWith("is")
104-
.subsequently.calledWith("a")
105-
.subsequently.calledWith("large")
106-
.subsequently.calledWith("buffer");
103+
expect(mockResolve).inOrder.to.have.been.calledWith(Buffer.from("this", "utf8"))
104+
.subsequently.calledWith(Buffer.from("is", "utf8"))
105+
.subsequently.calledWith(Buffer.from("a", "utf8"))
106+
.subsequently.calledWith(Buffer.from("large", "utf8"))
107+
.subsequently.calledWith(Buffer.from("buffer", "utf8"));
107108
})
108109

109110
it("should read even if the maxSpan has been exceeded", async () => {
110111
const mockResolve = sinon.spy();
111112
reader.maxSpan = 5;
112-
reader.envelopeReader = {readFn: sinon.fake.returns("willslicefrombeginning")}
113+
reader.envelopeReader = { readFn: sinon.fake.returns(Buffer.from("willslicefrombeginning", "utf8")) }
113114

114115
reader.queue = [{
115-
offset: 1,
116-
length: 4,
117-
resolve: mockResolve,
118-
}, {
119-
offset: 10,
120-
length: 4,
121-
resolve: mockResolve,
122-
}, {
123-
offset: 10,
124-
length: 9,
125-
resolve: mockResolve,
126-
}, {
127-
offset: 10,
128-
length: 13,
129-
resolve: mockResolve,
130-
}, {
131-
offset: 10,
132-
length: 22,
133-
resolve: mockResolve,
134-
}
116+
offset: 1,
117+
length: 4,
118+
resolve: mockResolve,
119+
}, {
120+
offset: 10,
121+
length: 4,
122+
resolve: mockResolve,
123+
}, {
124+
offset: 10,
125+
length: 9,
126+
resolve: mockResolve,
127+
}, {
128+
offset: 10,
129+
length: 13,
130+
resolve: mockResolve,
131+
}, {
132+
offset: 10,
133+
length: 22,
134+
resolve: mockResolve,
135+
}
135136
];
136137

137138
await reader.processQueue();
138139

139-
expect(mockResolve).inOrder.to.have.been.calledWith("will")
140-
.subsequently.calledWith("will")
141-
.subsequently.calledWith("willslice")
142-
.subsequently.calledWith("willslicefrom")
143-
.subsequently.calledWith("willslicefrombeginning");
140+
expect(mockResolve).inOrder.to.have.been.calledWith(Buffer.from("will", "utf8"))
141+
.subsequently.calledWith(Buffer.from("will", "utf8"))
142+
.subsequently.calledWith(Buffer.from("willslice", "utf8"))
143+
.subsequently.calledWith(Buffer.from("willslicefrom", "utf8"))
144+
.subsequently.calledWith(Buffer.from("willslicefrombeginning", "utf8"));
144145
})
145146
})
146147
})

0 commit comments

Comments
 (0)