Feature: Parquet Schema from JSON Schema (#82) with

Problem
=======

We want it to be easier to build Parquet files (and thus the schema for
those files) from existing schemas. In this case JSON Schemas

Solution
========

Change summary:
---------------
* Added field creation helper functions
* Added static function `ParquetSchema.fromJsonSchema` that takes in a
Json Schema

Steps to Verify:
----------------
1. Examples are in the tests, but take a JSON schema and call
`ParquetSchema.fromJsonSchema` with it and see what it does

---------

Co-authored-by: Shannon Wells <[email protected]>
With assistance from @noxify

Author: wilwade

README.md

@@ -87,6 +87,8 @@ Parquet files have a strict schema, similar to tables in a SQL database. So,
 in order to produce a Parquet file we first need to declare a new schema. Here
 is a simple example that shows how to instantiate a `ParquetSchema` object:
 
+### Native Schema Definition
+
 ``` js
 // declare a schema for the `fruits` table
 var schema = new parquet.ParquetSchema({
@@ -98,6 +100,46 @@ var schema = new parquet.ParquetSchema({
 });
 ```
 
+### Helper Functions
+
+```js
+var schema = new parquet.ParquetSchema({
+  name: parquet.ParquetFieldBuilder.createStringField(),
+  quantity: parquet.ParquetFieldBuilder.createIntField(64),
+  price: parquet.ParquetFieldBuilder.createDoubleField(),
+  date: parquet.ParquetFieldBuilder.createTimestampField(),
+  in_stock: parquet.ParquetFieldBuilder.createBooleanField()
+});
+```
+
+### JSON Schema
+
+``` js
+// declare a schema for the `fruits` JSON Schema
+var schema = new parquet.ParquetSchema.fromJsonSchema({
+  "type": "object",
+  "properties": {
+    "name": {
+      "type": "string"
+    },
+    "quantity": {
+      "type": "integer"
+    },
+    "price": {
+      "type": "number"
+    },
+    "date": {
+      "type": "string"
+    },
+    "in_stock": {
+      "type": "boolean"
+    }
+  },
+  "required": ["name", "quantity", "price", "date", "in_stock"]
+});
+```
+
+
 Note that the Parquet schema supports nesting, so you can store complex, arbitrarily
 nested records into a single row (more on that later) while still maintaining good
 compression.

lib/fields.ts

@@ -0,0 +1,74 @@
+// Helper functions for creating fields
+
+import { FieldDefinition, ParquetType, SchemaDefinition } from "./declare";
+
+export function createStringField(optional = true, fieldOptions: FieldDefinition = {}): FieldDefinition {
+    return { ...fieldOptions, optional, type: 'UTF8' };
+}
+
+export function createBooleanField(optional = true, fieldOptions: FieldDefinition = {}): FieldDefinition {
+    return { ...fieldOptions, optional, type: 'BOOLEAN' };
+}
+
+export function createIntField(size: 32 | 64, optional = true, fieldOptions: FieldDefinition = {}): FieldDefinition {
+    return { ...fieldOptions, optional, type: `INT${size}` };
+}
+
+export function createFloatField(optional = true, fieldOptions: FieldDefinition = {}): FieldDefinition {
+    return { ...fieldOptions, optional, type: 'FLOAT' };
+}
+
+export function createDoubleField(optional = true, fieldOptions: FieldDefinition = {}): FieldDefinition {
+    return { ...fieldOptions, optional, type: 'DOUBLE' };
+}
+
+export function createDecimalField(precision: number, optional = true, fieldOptions: FieldDefinition = {}): FieldDefinition {
+    return { ...fieldOptions, precision, optional, type: 'FLOAT' };
+}
+
+export function createTimestampField(optional = true, fieldOptions: FieldDefinition = {}): FieldDefinition {
+    return { ...fieldOptions, optional, type: 'TIMESTAMP_MILLIS' };
+}
+
+export function createStructField(fields: SchemaDefinition, optional = true): FieldDefinition {
+    return {
+        optional,
+        fields,
+    }
+}
+
+export function createStructListField(fields: SchemaDefinition, optional = true): FieldDefinition {
+    return {
+        type: 'LIST',
+        optional,
+        fields: {
+            list: {
+                repeated: true,
+                fields: {
+                    element: {
+                        fields,
+                    },
+                }
+            },
+        },
+    }
+}
+
+export function createListField(type: ParquetType, optional = true, elementOptions: FieldDefinition = { optional: true }): FieldDefinition {
+    return {
+        type: 'LIST',
+        optional,
+        fields: {
+            list: {
+                repeated: true,
+                fields: {
+                    element: {
+                        optional: true,
+                        ...elementOptions,
+                        type,
+                    },
+                },
+            },
+        },
+    }
+}

lib/jsonSchema.ts

@@ -0,0 +1,127 @@
+// Support json-schema.org schema conversion to a Parquet file
+import { JSONSchema4 } from 'json-schema';
+import { FieldDefinition, SchemaDefinition } from './declare';
+import * as fields from './fields';
+
+type SupportedJSONSchema4 = Omit<JSONSchema4, '$ref' | 'multipleOf' | 'allOf' | 'anyOf' | 'oneOf' | 'not' | 'additionalItems' | 'enum' | 'extends'>
+
+/**
+ * Simple check to make sure that `SupportedJSONSchema4` is correct.
+ * There are a lot of JSON schema stuff we just don't support for now.
+ */
+const isJsonSchemaSupported = (js: JSONSchema4): js is SupportedJSONSchema4 => {
+    const unsupportedFields = [
+        "$ref",
+        "multipleOf",
+        "allOf",
+        "anyOf",
+        "oneOf",
+        "not",
+        "additionalItems",
+        "enum",
+        "extends",
+    ];
+    for (const field in unsupportedFields) {
+        if (!(js[field] === undefined || js[field] === false)) {
+            return false;
+        }
+    }
+    return true;
+}
+
+/**
+ * Error to capture all the unsupported edge cases
+ */
+export class UnsupportedJsonSchemaError extends Error {
+    constructor(msg: string) {
+        const message = `Unsupported JSON schema: ${msg}`;
+        super(message);
+        this.name = 'UnsupportedJsonSchemaError';
+    }
+}
+
+/**
+ * Json Schema has required at the top level instead of field level
+ */
+const isJsonSchemaRequired = (jsonSchema: SupportedJSONSchema4) => (field: string): boolean => {
+    switch (jsonSchema.required) {
+        case true: return true;
+        case undefined:
+        case false:
+            return false;
+    }
+
+    return jsonSchema.required.includes(field);
+}
+
+/**
+ * Converts the Array field type into the correct Field Definition
+ */
+const fromJsonSchemaArray = (fieldValue: SupportedJSONSchema4, optionalFieldList: boolean): FieldDefinition => {
+    if (!fieldValue.items || !fieldValue.items.type) {
+        throw new UnsupportedJsonSchemaError("Array field with no values found.");
+    }
+
+    switch (fieldValue.items.type) {
+        case 'string':
+            return fields.createListField('UTF8', optionalFieldList);
+        case 'integer':
+        case 'number':
+            return fields.createListField('INT64', optionalFieldList);
+        case 'boolean':
+            return fields.createListField('BOOLEAN', optionalFieldList);
+        case 'object':
+            return fields.createStructListField(fromJsonSchema(fieldValue.items), optionalFieldList);
+        default:
+            throw new UnsupportedJsonSchemaError(`Array field type ${JSON.stringify(fieldValue.items)} is unsupported.`);
+    }
+}
+
+/**
+ * Converts a field from a JSON Schema into a Parquet Field Definition
+ */
+const fromJsonSchemaField = (jsonSchema: JSONSchema4) => (fieldName: string, fieldValue: JSONSchema4): FieldDefinition => {
+    if (!isJsonSchemaSupported(fieldValue)) {
+        throw new UnsupportedJsonSchemaError(`Field: ${fieldName} has an unsupported schema`);
+    }
+    const optional = !isJsonSchemaRequired(jsonSchema)(fieldName);
+
+    switch (fieldValue.type) {
+        case 'string':
+            return fields.createStringField(optional);
+        case 'integer':
+        case 'number':
+            return fields.createIntField(64, optional);
+        case 'boolean':
+            return fields.createBooleanField(optional);
+        case 'array':
+            return fromJsonSchemaArray(fieldValue, optional);
+        case 'object':
+            return fields.createStructField(fromJsonSchema(fieldValue), optional);
+        default:
+            throw new UnsupportedJsonSchemaError(
+                `Unable to convert "${fieldName}" with JSON Schema type "${fieldValue.type}" to a Parquet Schema.`,
+            )
+    }
+}
+
+/**
+ * Converts supported Json Schemas into Parquet Schema Definitions
+ */
+export const fromJsonSchema = (jsonSchema: JSONSchema4): SchemaDefinition => {
+    if (!isJsonSchemaSupported(jsonSchema)) {
+        throw new UnsupportedJsonSchemaError("Unsupported fields found");
+    }
+
+    const schema: SchemaDefinition = {};
+
+    const fromField = fromJsonSchemaField(jsonSchema)
+
+    for (const [fieldName, fieldValue] of Object.entries(
+        jsonSchema.properties || {},
+    )) {
+        schema[fieldName] = fromField(fieldName, fieldValue);
+    }
+
+    return schema;
+}

lib/schema.ts

@@ -2,9 +2,12 @@ import * as parquet_codec from './codec';
 import * as parquet_compression from './compression'
 import * as parquet_types from './types'
 import { SchemaDefinition, ParquetField, RepetitionType, FieldDefinition } from './declare'
+import { JSONSchema4 } from 'json-schema'
+import { fromJsonSchema } from './jsonSchema';
 
 const PARQUET_COLUMN_KEY_SEPARATOR = '.';
 
+
 /**
  * A parquet file schema
  */
@@ -13,6 +16,14 @@ export class ParquetSchema {
   fields: Record<string, ParquetField>
   fieldList: Array<ParquetField>
 
+  /**
+   * Create a new schema from JSON Schema (json-schema.org)
+   */
+  static fromJsonSchema(jsonSchema: JSONSchema4) {
+    const schema: SchemaDefinition = fromJsonSchema(jsonSchema);
+    return new ParquetSchema(schema);
+  }
+
   /**
    * Create a new schema from a JSON schema definition
    */