[Bug]: pixel mask gets duplicated every time it is written

[pauladkisson]

What happened?

Follow up to #272

Every time pixel mask is written is expands in shape with duplicates -- see example below.

Steps to Reproduce

from pynwb.ophys import PlaneSegmentation
from pynwb.testing.mock.file import mock_NWBFile
from pynwb.testing.mock.ophys import mock_ImagingPlane
from hdmf_zarr import NWBZarrIO
import numpy as np

def test_export_pixel_mask():
    nwbfile = mock_NWBFile()
    # Add PlaneSegmentation with pixel_mask
    n_rois = 10
    plane_segmentation = PlaneSegmentation(
        description="no description.",
        imaging_plane=mock_ImagingPlane(nwbfile=nwbfile),
        name="PlaneSegmentation",
    )

    for _ in range(n_rois):
        pixel_mask = [(x, x, 1.0) for x in range(10)]
        plane_segmentation.add_roi(pixel_mask=pixel_mask)

    if "ophys" not in nwbfile.processing:
        nwbfile.create_processing_module("ophys", "ophys")
    nwbfile.processing["ophys"].add(plane_segmentation)

    print("Before write")
    print(f"{np.array(nwbfile.processing['ophys'].data_interfaces['PlaneSegmentation'].pixel_mask.data[:]).shape = }") # (100, 3)
    print(f"{nwbfile.processing['ophys'].data_interfaces['PlaneSegmentation'].pixel_mask.data[:3] = }") # [(0, 0, 1.0), (1, 1, 1.0), (2, 2, 1.0)]

    # write it to disk
    nwbfile_path = "pixel_mask_export_bug.nwb"
    with NWBZarrIO(nwbfile_path, "w") as read_io:
        read_io.write(nwbfile)

    # read it back
    with NWBZarrIO(nwbfile_path, "r") as read_io:
        nwbfile = read_io.read()

        print("After write, before export")
        print(f"{nwbfile.processing['ophys'].data_interfaces['PlaneSegmentation'].pixel_mask.data[:].shape = }") # (100, 3)
        print(f"{nwbfile.processing['ophys'].data_interfaces['PlaneSegmentation'].pixel_mask.data[:3] = }")
        # array([[(0, 0, 1.), (0, 0, 1.), (0, 0, 1.)],
        # [(1, 1, 1.), (1, 1, 1.), (1, 1, 1.)],
        # [(2, 2, 1.), (2, 2, 1.), (2, 2, 1.)]],
        # dtype=[('x', '<u4'), ('y', '<u4'), ('weight', '<f4')])

        # Export to a new path
        export_path = "pixel_mask_export_bug_exported.nwb"
        with NWBZarrIO(export_path, "w") as export_io:
            nwbfile.set_modified()
            export_io.export(nwbfile=nwbfile, src_io=read_io, write_args=dict(link_data=False))
    
    # export again
    with NWBZarrIO(export_path, "r") as export_io:
        nwbfile = export_io.read()
        print("After export")
        print(f"{nwbfile.processing["ophys"].data_interfaces["PlaneSegmentation"].pixel_mask.data[:].shape = }") # (100, 3, 3)
        print(f"{nwbfile.processing["ophys"].data_interfaces["PlaneSegmentation"].pixel_mask.data[:3] = }")
        # array([[[(0, 0, 0.), (0, 0, 0.), (1, 1, 1.)],
    #     [(0, 0, 0.), (0, 0, 0.), (1, 1, 1.)],
    #     [(0, 0, 0.), (0, 0, 0.), (1, 1, 1.)]],

    #    [[(1, 1, 1.), (1, 1, 1.), (1, 1, 1.)],
    #     [(1, 1, 1.), (1, 1, 1.), (1, 1, 1.)],
    #     [(1, 1, 1.), (1, 1, 1.), (1, 1, 1.)]],

    #    [[(2, 2, 2.), (2, 2, 2.), (1, 1, 1.)],
    #     [(2, 2, 2.), (2, 2, 2.), (1, 1, 1.)],
    #     [(2, 2, 2.), (2, 2, 2.), (1, 1, 1.)]]],
    #   dtype=[('x', '<u4'), ('y', '<u4'), ('weight', '<f4')])

        double_export_path = "pixel_mask_export_bug_exported_double.nwb"
        with NWBZarrIO(double_export_path, "w") as double_export_io:
            nwbfile.set_modified()
            double_export_io.export(nwbfile=nwbfile, src_io=export_io, write_args=dict(link_data=False)) # This line throws an error

if __name__ == "__main__":
    test_export_pixel_mask()

Traceback

TypeError: only length-1 arrays can be converted to Python scalars

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "/Users/pauladkisson/Documents/CatalystNeuro/HDMF/LBNL/hdmf-zarr/src/hdmf_zarr/backend.py", line 1402, in __list_fill__
    dset[:] = np.array(data, dtype=dtype)
              ^^^^^^^^^^^^^^^^^^^^^^^^^^^
ValueError: setting an array element with a sequence.

During handling of the above exception, another exception occurred:

TypeError: only length-1 arrays can be converted to Python scalars

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "/Users/pauladkisson/Documents/CatalystNeuro/Neuroconv/neuroconv/pixel_mask_export_bug.py", line 71, in <module>
    test_export_pixel_mask()
  File "/Users/pauladkisson/Documents/CatalystNeuro/Neuroconv/neuroconv/pixel_mask_export_bug.py", line 62, in test_export_pixel_mask
    double_export_io.export(nwbfile=nwbfile, src_io=export_io, write_args=dict(link_data=False))
  File "/opt/anaconda3/envs/repack_env/lib/python3.12/site-packages/hdmf/utils.py", line 577, in func_call
    return func(args[0], **pargs)
           ^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/pauladkisson/Documents/CatalystNeuro/HDMF/LBNL/hdmf-zarr/src/hdmf_zarr/nwb.py", line 71, in export
    super().export(**kwargs)
  File "/opt/anaconda3/envs/repack_env/lib/python3.12/site-packages/hdmf/utils.py", line 577, in func_call
    return func(args[0], **pargs)
           ^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/pauladkisson/Documents/CatalystNeuro/HDMF/LBNL/hdmf-zarr/src/hdmf_zarr/backend.py", line 438, in export
    super().export(**ckwargs)
  File "/opt/anaconda3/envs/repack_env/lib/python3.12/site-packages/hdmf/utils.py", line 577, in func_call
    return func(args[0], **pargs)
           ^^^^^^^^^^^^^^^^^^^^^^
  File "/opt/anaconda3/envs/repack_env/lib/python3.12/site-packages/hdmf/backends/io.py", line 166, in export
    self.write_builder(builder=bldr, **write_args)
  File "/opt/anaconda3/envs/repack_env/lib/python3.12/site-packages/hdmf/utils.py", line 577, in func_call
    return func(args[0], **pargs)
           ^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/pauladkisson/Documents/CatalystNeuro/HDMF/LBNL/hdmf-zarr/src/hdmf_zarr/backend.py", line 522, in write_builder
    self.write_group(
  File "/opt/anaconda3/envs/repack_env/lib/python3.12/site-packages/hdmf/utils.py", line 577, in func_call
    return func(args[0], **pargs)
           ^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/pauladkisson/Documents/CatalystNeuro/HDMF/LBNL/hdmf-zarr/src/hdmf_zarr/backend.py", line 621, in write_group
    self.write_group(
  File "/opt/anaconda3/envs/repack_env/lib/python3.12/site-packages/hdmf/utils.py", line 577, in func_call
    return func(args[0], **pargs)
           ^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/pauladkisson/Documents/CatalystNeuro/HDMF/LBNL/hdmf-zarr/src/hdmf_zarr/backend.py", line 621, in write_group
    self.write_group(
  File "/opt/anaconda3/envs/repack_env/lib/python3.12/site-packages/hdmf/utils.py", line 577, in func_call
    return func(args[0], **pargs)
           ^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/pauladkisson/Documents/CatalystNeuro/HDMF/LBNL/hdmf-zarr/src/hdmf_zarr/backend.py", line 632, in write_group
    self.write_dataset(
  File "/opt/anaconda3/envs/repack_env/lib/python3.12/site-packages/hdmf/utils.py", line 577, in func_call
    return func(args[0], **pargs)
           ^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/pauladkisson/Documents/CatalystNeuro/HDMF/LBNL/hdmf-zarr/src/hdmf_zarr/backend.py", line 1186, in write_dataset
    dset = self.__list_fill__(parent, name, data, options)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/pauladkisson/Documents/CatalystNeuro/HDMF/LBNL/hdmf-zarr/src/hdmf_zarr/backend.py", line 1408, in __list_fill__
    dset[i] = data[i]
    ~~~~^^^
  File "/opt/anaconda3/envs/repack_env/lib/python3.12/site-packages/zarr/core.py", line 1451, in __setitem__
    self.set_basic_selection(pure_selection, value, fields=fields)
  File "/opt/anaconda3/envs/repack_env/lib/python3.12/site-packages/zarr/core.py", line 1547, in set_basic_selection
    return self._set_basic_selection_nd(selection, value, fields=fields)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/opt/anaconda3/envs/repack_env/lib/python3.12/site-packages/zarr/core.py", line 1937, in _set_basic_selection_nd
    self._set_selection(indexer, value, fields=fields)
  File "/opt/anaconda3/envs/repack_env/lib/python3.12/site-packages/zarr/core.py", line 1990, in _set_selection
    self._chunk_setitem(chunk_coords, chunk_selection, chunk_value, fields=fields)
  File "/opt/anaconda3/envs/repack_env/lib/python3.12/site-packages/zarr/core.py", line 2263, in _chunk_setitem
    self._chunk_setitem_nosync(chunk_coords, chunk_selection, value, fields=fields)
  File "/opt/anaconda3/envs/repack_env/lib/python3.12/site-packages/zarr/core.py", line 2267, in _chunk_setitem_nosync
    cdata = self._process_for_setitem(ckey, chunk_selection, value, fields=fields)
            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/opt/anaconda3/envs/repack_env/lib/python3.12/site-packages/zarr/core.py", line 2328, in _process_for_setitem
    chunk[chunk_selection] = value
    ~~~~~^^^^^^^^^^^^^^^^^
ValueError: setting an array element with a sequence.

Operating System

MacOS Intel

Python Version

3.12

Package Versions

annotated-types==0.7.0
asciitree==0.3.3
attrs==25.3.0
click==8.2.0
Deprecated==1.2.18
docstring_parser==0.16
fasteners==0.19
h5py==3.13.0
hdmf==4.0.0
-e git+https://github.com/hdmf-dev/hdmf-zarr.git@f67131484ae3ba1a9e24fa0aed43549b275d4f19#egg=hdmf_zarr
iniconfig==2.1.0
jsonschema==4.23.0
jsonschema-specifications==2025.4.1
-e git+https://github.com/catalystneuro/neuroconv@61bce6612a1cadda5d9f0e9269844b48d89c7b13#egg=neuroconv
numcodecs==0.15.1
numpy==2.2.5
packaging==25.0
pandas==2.2.3
parse==1.20.2
pluggy==1.5.0
psutil==7.0.0
pydantic==2.11.4
pydantic_core==2.33.2
pynwb==3.0.0
pytest==8.3.5
python-dateutil==2.9.0.post0
pytz==2025.2
PyYAML==6.0.2
referencing==0.36.2
rpds-py==0.24.0
ruamel.yaml==0.18.10
ruamel.yaml.clib==0.2.12
scipy==1.15.3
setuptools==78.1.1
six==1.17.0
threadpoolctl==3.6.0
tqdm==4.67.1
typing-inspection==0.4.0
typing_extensions==4.13.2
tzdata==2025.2
wheel==0.45.1
wrapt==1.17.2
zarr==2.18.7

[mavaylon1]

Odd. I can take a look and give an update next Tuesday.

[pauladkisson]

@mavaylon1 have you had a chance to take a look at this?

[pauladkisson]

@mavaylon1 Any update?

[mavaylon1]

@pauladkisson No update yet. I will see if I can pin point the problem Monday. If not, I will pick this up in the following week.

[pauladkisson]

@mavaylon1 checking back in on this.

[mavaylon1]

@pauladkisson After some digging, it isn't clear to me what the issue is. I will bring this up with the team.

[pauladkisson]

Thanks for taking a look!