Types Cleanup (#67)

* Switch to import
* Better optional types
* Remove duplicated types
* types/types -> declare for less confusion between types and types/types
* Make sure thrift types pass through
* Minimal WriteStream Interface
* Fix an unknown type
* Better openUrl Interface

Author: wilwade

lib/bloom/sbbf.ts

@@ -1,7 +1,7 @@
 import parquet_thrift from "../../gen-nodejs/parquet_types";
 import Long from 'long';
 import XxHasher from "./xxhasher"
-import {Block} from "../types/types";
+import {Block} from "../declare";
 
 /**
  * @class SplitBlockBloomFilter

lib/bloomFilterIO/bloomFilterReader.ts

@@ -2,7 +2,7 @@ import * as parquet_util from "../util";
 import parquet_thrift from "../../gen-nodejs/parquet_types";
 import sbbf from "../bloom/sbbf";
 import { ParquetEnvelopeReader } from "../reader"
-import { ColumnChunkData } from "../types/types";
+import { ColumnChunkData } from "../declare";
 
 const filterColumnChunksWithBloomFilters = (
   columnChunkDataCollection: Array<ColumnChunkData>

lib/bloomFilterIO/bloomFilterWriter.ts

@@ -2,7 +2,7 @@ import * as parquet_util from "../util";
 import parquet_thrift from "../../gen-nodejs/parquet_types";
 import SplitBlockBloomFilter from "../bloom/sbbf";
 
-import { Block } from "../types/types";
+import { Block } from "../declare";
 import Int64 from 'node-int64'
 
 export type createSBBFParams = {

lib/bufferReader.ts

@@ -1,6 +1,6 @@
 import { Statistics } from "../gen-nodejs/parquet_types"
 import { ParquetEnvelopeReader } from "./reader"
-import { FileMetaDataExt } from "./types/types"
+import { FileMetaDataExt } from "./declare"
 
 export interface BufferReaderOptions {
   maxSpan?: number,

lib/codec/types.ts

@@ -1,5 +1,5 @@
-import { PrimitiveType } from "../types/types";
-import { ParquetCodec, OriginalType, ParquetField } from "../types/types";
+import { PrimitiveType } from "../declare";
+import { ParquetCodec, OriginalType, ParquetField } from "../declare";
 import { Statistics } from "../../gen-nodejs/parquet_types";
 
 export interface Options {

lib/types/types.ts renamed to lib/declare.ts

@@ -1,9 +1,9 @@
-// Lifted from https://github.com/kbajalc/parquets
+// Thanks to https://github.com/kbajalc/parquets
 
-import parquet_thrift from "../../gen-nodejs/parquet_types";
-import { Statistics, OffsetIndex, ColumnIndex, PageType, DataPageHeader, DataPageHeaderV2, DictionaryPageHeader, IndexPageHeader, Type, ColumnMetaData } from "../../gen-nodejs/parquet_types";
-import SplitBlockBloomFilter from "../bloom/sbbf";
-import { createSBBFParams } from "../bloomFilterIO/bloomFilterWriter";
+import parquet_thrift from "../gen-nodejs/parquet_types";
+import { Statistics, OffsetIndex, ColumnIndex, PageType, DataPageHeader, DataPageHeaderV2, DictionaryPageHeader, IndexPageHeader, Type, ColumnMetaData } from "../gen-nodejs/parquet_types";
+import SplitBlockBloomFilter from "./bloom/sbbf";
+import { createSBBFParams } from "./bloomFilterIO/bloomFilterWriter";
 import Int64 from 'node-int64'
 
 export type ParquetCodec = 'PLAIN' | 'RLE';
@@ -27,8 +27,8 @@ export type OriginalType =
     | 'UTF8' // 0
     | 'MAP' // 1
     // | 'MAP_KEY_VALUE' // 2
-     | 'LIST' // 3
-    // | 'ENUM' // 4
+    | 'LIST' // 3
+    | 'ENUM' // 4
     // | 'DECIMAL' // 5
     | 'DATE' // 6
     | 'TIME_MILLIS' // 7
@@ -59,15 +59,15 @@ export interface FieldDefinition {
     optional?: boolean;
     repeated?: boolean;
     fields?: SchemaDefinition;
-    statistics?: Statistics
+    statistics?: Statistics | false;
     parent?: ParentField
     num_children?: NumChildrenField
 }
 
 export interface ParquetField {
     name: string;
     path: string[];
-    statistics?: Statistics
+    statistics?: Statistics | false;
     primitiveType?: PrimitiveType;
     originalType?: OriginalType;
     repetitionType: RepetitionType;

lib/reader.ts

@@ -9,7 +9,7 @@ import * as parquet_types from './types';
 import BufferReader , { BufferReaderOptions } from './bufferReader';
 import * as bloomFilterReader from './bloomFilterIO/bloomFilterReader';
 import fetch from 'cross-fetch';
-import { ParquetCodec, Parameter,PageData, SchemaDefinition, ParquetType, FieldDefinition, ParquetField, ClientS3, ClientParameters, FileMetaDataExt, NewPageHeader, RowGroupExt, ColumnChunkExt } from './types/types';
+import { ParquetCodec, Parameter,PageData, SchemaDefinition, ParquetType, FieldDefinition, ParquetField, ClientS3, ClientParameters, FileMetaDataExt, NewPageHeader, RowGroupExt, ColumnChunkExt } from './declare';
 import { Cursor, Options } from './codec/types';
 
 const {
@@ -111,12 +111,12 @@ export class ParquetReader {
    * Open the parquet file pointed to by the specified path and return a new
    * parquet reader
    */
-  static async openFile(filePath: string | Buffer | URL, options: BufferReaderOptions) {
+  static async openFile(filePath: string | Buffer | URL, options?: BufferReaderOptions) {
     let envelopeReader = await ParquetEnvelopeReader.openFile(filePath, options);
     return this.openEnvelopeReader(envelopeReader, options);
   }
 
-  static async openBuffer(buffer: Buffer, options: BufferReaderOptions) {
+  static async openBuffer(buffer: Buffer, options?: BufferReaderOptions) {
     let envelopeReader = await ParquetEnvelopeReader.openBuffer(buffer, options);
     return this.openEnvelopeReader(envelopeReader, options);
   }
@@ -126,7 +126,7 @@ export class ParquetReader {
    * The params have to include `Bucket` and `Key` to the file requested
    * This function returns a new parquet reader
    */
-  static async openS3(client: ClientS3, params: ClientParameters, options: BufferReaderOptions) {
+  static async openS3(client: ClientS3, params: ClientParameters, options?: BufferReaderOptions) {
     let envelopeReader = await ParquetEnvelopeReader.openS3(client, params, options);
     return this.openEnvelopeReader(envelopeReader, options);
   }
@@ -137,13 +137,13 @@ export class ParquetReader {
    * a `url` property.
    * This function returns a new parquet reader
    */
-  static async openUrl(params: Parameter, options: BufferReaderOptions) {
+  static async openUrl(params: Parameter | URL | string, options?: BufferReaderOptions) {
     let envelopeReader = await ParquetEnvelopeReader.openUrl(params, options);
     return this.openEnvelopeReader(envelopeReader, options);
   }
 
-  static async openEnvelopeReader(envelopeReader: ParquetEnvelopeReader, opts: BufferReaderOptions) {
-    if (opts && opts.metadata) {
+  static async openEnvelopeReader(envelopeReader: ParquetEnvelopeReader, opts?: BufferReaderOptions) {
+    if (opts?.metadata) {
       return new ParquetReader(opts.metadata, envelopeReader, opts);
     }
     try {
@@ -164,7 +164,7 @@ export class ParquetReader {
    * and internal use cases. Consider using one of the open{File,Buffer} methods
    * instead
    */
-  constructor(metadata: FileMetaDataExt, envelopeReader: ParquetEnvelopeReader, opts: BufferReaderOptions) {
+  constructor(metadata: FileMetaDataExt, envelopeReader: ParquetEnvelopeReader, opts?: BufferReaderOptions) {
     opts = opts || {};
     if (metadata.version != PARQUET_VERSION) {
       throw 'invalid parquet version';
@@ -266,7 +266,7 @@ export class ParquetReader {
 
   async getBloomFiltersFor(columnNames: string[]) {
     const bloomFilterData = await getBloomFiltersFor(columnNames, this.envelopeReader!);
-    return bloomFilterData.reduce((acc: Record<string, Array<unknown>>, value) => {
+    return bloomFilterData.reduce((acc: Record<string, typeof bloomFilterData>, value) => {
       if (acc[value.columnName]) acc[value.columnName].push(value)
       else acc[value.columnName] = [value]
       return acc;
@@ -384,7 +384,7 @@ export class ParquetEnvelopeReader {
   metadata?: FileMetaDataExt;
   schema?: parquet_schema.ParquetSchema
 
-  static async openFile(filePath: string | Buffer | URL, options: BufferReaderOptions) {
+  static async openFile(filePath: string | Buffer | URL, options?: BufferReaderOptions) {
     let fileStat = await parquet_util.fstat(filePath);
     let fileDescriptor = await parquet_util.fopen(filePath);
 
@@ -401,7 +401,7 @@ export class ParquetEnvelopeReader {
     return new ParquetEnvelopeReader(readFn, closeFn, fileStat.size, options);
   }
 
-  static async openBuffer(buffer: Buffer, options: BufferReaderOptions) {
+  static async openBuffer(buffer: Buffer, options?: BufferReaderOptions) {
     let readFn = (offset: number, length: number, file?: string) => {
       if (file) {
         return Promise.reject('external references are not supported');
@@ -414,7 +414,7 @@ export class ParquetEnvelopeReader {
     return new ParquetEnvelopeReader(readFn, closeFn, buffer.length, options);
   }
 
-  static async openS3(client: ClientS3, params: ClientParameters, options: BufferReaderOptions) {
+  static async openS3(client: ClientS3, params: ClientParameters, options?: BufferReaderOptions) {
     let fileStat = async () => client.headObject(params).promise().then((d: {ContentLength: number}) => d.ContentLength);
 
     let readFn = async (offset: number, length: number, file?: string) => {
@@ -432,9 +432,12 @@ export class ParquetEnvelopeReader {
     return new ParquetEnvelopeReader(readFn, closeFn, fileStat, options);
   }
 
-  static async openUrl(params: Parameter, options: BufferReaderOptions) {
-     if (typeof params === 'string')
-      params = {url: params};
+  static async openUrl(url: Parameter | URL | string, options?: BufferReaderOptions) {
+    let params: Parameter;
+    if (typeof url === 'string') params = { url };
+    else if(url instanceof URL) params = { url: url.toString() }
+    else params = url;
+
     if (!params.url)
       throw new Error('URL missing');
 
@@ -465,7 +468,7 @@ export class ParquetEnvelopeReader {
     return new ParquetEnvelopeReader(readFn, closeFn, filesize, options);
   }
 
-  constructor(readFn: (offset: number, length: number, file?: string) => Promise<Buffer> , closeFn: () => unknown, fileSize: Function | number, options: BufferReaderOptions, metadata?: FileMetaDataExt) {
+  constructor(readFn: (offset: number, length: number, file?: string) => Promise<Buffer> , closeFn: () => unknown, fileSize: Function | number, options?: BufferReaderOptions, metadata?: FileMetaDataExt) {
     options = options || {};
     this.readFn = readFn;
     this.id = ++ParquetEnvelopeReaderIdCounter;

lib/schema.ts

@@ -1,7 +1,7 @@
 import * as parquet_codec from './codec';
 import * as parquet_compression from './compression'
 import * as parquet_types from './types'
-import { SchemaDefinition, ParquetField, RepetitionType } from './types/types'
+import { SchemaDefinition, ParquetField, RepetitionType } from './declare'
 
 const PARQUET_COLUMN_KEY_SEPARATOR = '.';
 
@@ -106,7 +106,7 @@ function buildFields(schema: SchemaDefinition, rLevelParentMax?: number, dLevelP
     }
 
     /* nested field */
-    
+
     if (opts.fields) {
       fieldList[name] = {
         name: name,
@@ -123,7 +123,7 @@ function buildFields(schema: SchemaDefinition, rLevelParentMax?: number, dLevelP
               dLevelMax,
               path.concat(name))
       };
-      
+
       if (opts.type == 'LIST' || opts.type == 'MAP') fieldList[name].originalType = opts.type;
 
       continue;

lib/shred.ts

@@ -1,6 +1,6 @@
 import * as parquet_types from './types'
 import { ParquetSchema } from './schema'
-import { Page, PageData, ParquetField } from './types/types';
+import { Page, PageData, ParquetField } from './declare';
 
 /**
  * 'Shred' a record into a list of <value, repetition_level, definition_level>