Merge pull request #161 from linkml/fix_dates_160

Fix date range without times

Author: amc-corey-cox

.github/workflows/check-pull-request.yaml

@@ -16,14 +16,18 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ ubuntu-latest, windows-latest ]
+        # os: [ ubuntu-latest, windows-latest ] - Exclude windows for now
+        os: [ ubuntu-latest ]
         python-version: [ "3.9", "3.10" ]
         exclude:
           - os: windows-latest
             python-version: "3.9"
 
     runs-on: ${{ matrix.os }}
 
+    # Allow Python 3.13 to fail due to scipy not being available
+    continue-on-error: ${{ matrix.python-version == '3.13' }}
+
     steps:
 
       #----------------------------------------------
@@ -57,7 +61,7 @@ jobs:
       #----------------------------------------------
       - name: Load cached venv
         id: cached-poetry-dependencies
-        uses: actions/cache@v2
+        uses: actions/cache@v3
         with:
           path: .venv
           key: venv-${{ runner.os }}-${{ hashFiles('**/poetry.lock') }}

.gitignore

@@ -14,4 +14,5 @@ tests/outputs/*
 venv/
 .venv/
 target/
-local/
+local/
+.python-version

poetry.lock

@@ -60,7 +60,7 @@ pydbml = "^1.1.2"
 pyyaml = "^6.0.2"
 llm = {version = "^0.21", optional = true}
 
-[tool.poetry.dev-dependencies]
+[tool.poetry.group.dev.dependencies]
 pytest = ">=7.1.1"
 Sphinx = ">=4.4.0"
 sphinx-pdj-theme = ">=0.2.1"

pyproject.toml

@@ -1,3 +1,4 @@
+import datetime
 import click
 import logging
 import yaml
@@ -644,7 +645,11 @@ def infer_range(slot: dict, vals: set, types: dict, coerce=True) -> str:
             return 'boolean'
         if all(isfloat(v) for v in nn_vals):
             return 'float'
-        if all(is_date(v) for v in nn_vals):
+        parsed_datetimes = [is_date_or_datetime(v) for v in nn_vals]
+        if all(pd == 'date' for pd in parsed_datetimes):
+            return 'date'
+        if all(pd in ('date', 'datetime') for pd in parsed_datetimes):
+            # This selects datetime when values are mixed which may fail validation
             return 'datetime'
     if is_all_measurement(nn_vals):
         return 'measurement'
@@ -691,6 +696,24 @@ def is_date(string, fuzzy=False):
         return False
 
 
+def is_date_or_datetime(string, fuzzy=False):
+    """
+    Return whether the string can be interpreted as a date or datetime.
+
+    :param string: str, string to check for date
+    :param fuzzy: bool, ignore unknown tokens in string if True
+    """
+    try:
+        dt = parse(string, fuzzy=fuzzy)
+        if dt.hour == 0 and dt.minute == 0 and dt.second == 0:
+            return 'date'
+        return 'datetime'
+    except Exception:
+        # https://stackoverflow.com/questions/4990718/how-can-i-write-a-try-except-block-that-catches-all-exceptions
+        # we don't know all the different parse exceptions, we assume any error means this is not a date
+        return False
+
+
 @dataclass
 class Hit:
     term_id: str

schema_automator/generalizers/csv_data_generalizer.py

@@ -1,8 +1,9 @@
 import logging
 from pathlib import Path
-from typing import Dict, Iterable, List, Any, Mapping, TextIO
+from typing import Any, Dict, Iterable, List, Mapping, Optional, TextIO, Union
 import typing
 from collections import defaultdict, Counter
+import warnings
 
 from jsonasobj2 import JsonObj
 from linkml.utils.schema_builder import SchemaBuilder
@@ -51,7 +52,7 @@ class RdfsImportEngine(ImportEngine):
     #: Mapping from field names in this RDF schema (e.g. `price`) to IRIs (e.g. `http://schema.org/price`)
     mappings: Dict[str, URIRef] = field(default_factory=dict)
     #: User-defined mapping from LinkML metamodel slots (such as `domain_of`) to RDFS IRIs (such as http://schema.org/domainIncludes)
-    initial_metamodel_mappings: Dict[str, URIRef | List[URIRef]] = field(default_factory=dict)
+    initial_metamodel_mappings: Dict[str, Union[URIRef, List[URIRef]]] = field(default_factory=dict)
     #: Combined mapping from LinkML metamodel slots to RDFS IRIs
     metamodel_mappings: Dict[str, List[URIRef]] = field(default_factory=lambda: defaultdict(list))
     #: Reverse of `metamodel_mappings`, but supports multiple terms mapping to the same IRI
@@ -97,12 +98,12 @@ def __post_init__(self):
 
     def convert(
         self,
-        file: str | Path | TextIO,
-        name: str | None = None,
-        format: str | None="turtle",
-        default_prefix: str | None = None,
-        model_uri: str | None = None,
-        identifier: str | None = None,
+        file: Union[str, Path, TextIO],
+        name: Optional[str] = None,
+        format: Optional[str] = "turtle",
+        default_prefix: Optional[str] = None,
+        model_uri: Optional[str] = None,
+        identifier: Optional[str] = None,
         **kwargs: Any,
     ) -> SchemaDefinition:
         """
@@ -130,7 +131,10 @@ def convert(
         cls_slots = defaultdict(list)
 
         for slot in self.generate_rdfs_properties(g, cls_slots):
-            sb.add_slot(slot)
+            if slot.name in sb.schema.slots:
+                logging.warning(f"Slot '{slot.name}' already exists in schema; skipping duplicate.")
+            else:
+                sb.add_slot(slot)
         for cls in self.process_rdfs_classes(g, cls_slots):
             sb.add_class(cls)
 
@@ -151,9 +155,16 @@ def convert(
         schema.prefixes = {key: value for key, value in schema.prefixes.items() if key in self.seen_prefixes}
         self.infer_metadata(schema, name, default_prefix, model_uri)
         self.fix_missing(schema)
+        self._normalize_slot_ranges(schema)
         return schema
 
-    def infer_metadata(self, schema: SchemaDefinition, name: str | None, default_prefix: str | None = None, model_uri: str | None = None):
+    def infer_metadata(
+        self,
+        schema: SchemaDefinition,
+        name: Optional[str] = None,
+        default_prefix: Optional[str] = None,
+        model_uri: Optional[str] = None,
+    ):
         top_count = self.prefix_counts.most_common(1)
         if len(top_count) == 0:
             raise ValueError("No prefixes found in the graph")
@@ -313,7 +324,7 @@ def _dict_for_subject(self, g: Graph, s: URIRef, subject_type: typing.Literal["s
     def _rdfs_metamodel_iri(self, name: str) -> List[URIRef]:
         return self.metamodel_mappings.get(name, [])
 
-    def _element_from_iri(self, iri: URIRef) -> str | None:
+    def _element_from_iri(self, iri: URIRef) -> Optional[str]:
         r = self.reverse_metamodel_mappings.get(iri, [])
         if len(r) > 0:
             if len(r) > 1:
@@ -341,3 +352,25 @@ def _as_name(self, v: URIRef) -> str:
             if sep in v_str:
                 return v_str.split(sep)[-1]
         return v_str
+
+    def _normalize_slot_ranges(self, schema: SchemaDefinition) -> None:
+        """
+        Normalize slot ranges to valid LinkML scalars where needed.
+        Currently supports remapping RDF types like 'langString'.
+        """
+        RDF_DATATYPE_MAP = {
+            "langString": "string",
+            "Text": "string",
+            "Thing": "string",
+            "landingPage": "string",
+            "Boolean": "boolean",
+            "Number": "integer",
+            "URL": "uri",
+        }
+
+        for slot in schema.slots.values():
+            if slot.range in RDF_DATATYPE_MAP:
+                warnings.warn(
+                    f"Slot '{slot.name}' has unsupported range '{slot.range}'; mapping to '{RDF_DATATYPE_MAP[slot.range]}'."
+                )
+                slot.range = RDF_DATATYPE_MAP[slot.range]

schema_automator/importers/rdfs_import_engine.py

@@ -68,6 +68,10 @@ def test_infer_range(self):
             (['5.999', '7.955', '7.990', '6.990'], "float"),
             (["2mm", "3m", "4 mm"], "measurement"),
             (["true", "false"], "boolean"),
+            (["2024-01-01", "2023-12-31"], "date"),
+            (["2024-01-01T12:30:00", "2023-12-31T08:15:00"], "datetime"),
+            (["2024-01-01", "2023-12-31T08:15:00"], "datetime"),
+            (["2024-01-01", "not-a-date"], "string"),
         ]
         for values, expected in cases:
             self.assertEqual(infer_range({}, values, {}), expected, f"Failed on {values}")

tests/test_generalizers/test_csv_data_generalizer.py

@@ -5,6 +5,7 @@
 from io import StringIO
 import unittest
 import os
+import pytest
 import yaml
 from linkml_runtime import SchemaView
 
@@ -80,6 +81,6 @@ def test_from_rdfs():
     assert activity.name == "Activity"
     assert activity.is_a == "CreativeWork"
     slots = sv.class_induced_slots(activity.name)
-    assert len(slots) == 1
-    slot = slots[0]
-    assert slot.name == "id"
+    assert len(slots) == 18
+    slot_names = [s.name for s in slots]
+    assert "messages" in slot_names