Skip to content

Filter failure segfaults instead of returning an error #6601

Open

Description

@ArvidJB

⛔ IMPORTANT ⛔
Security issues should not be reported here! Instead, please
report them privately as security vulnerabilities; see for more
information https://github.com/HDFGroup/hdf5/security/policy


Describe the bug

When a data filter fails during a flush-on-close of a chunked dataset, the
library corrupts internal dataset/chunk state and later segfaults (in
H5D_close / H5D_flush_all) instead of propagating a clean error to the
caller.

The failure is easy to hit with the new (and correct) filter buffer-size check
added in #6184: any filter whose write callback returns a data size larger than
the *buf_size it reports now legitimately fails with "buffer size is too
small after filter callback"
— but that failure, when it occurs during
flush-on-close, brings down the whole process.

This is not a bug in the check itself (the check is correct). It is a bug in
the library's handling of a filter/pipeline failure on the flush-on-close path:
a recoverable, expected error condition results in memory corruption and a
crash.

Reproduces on HDF5 2.2.0 and on current develop (2.3.0) — including the
July chunk-buffer failure-path fixes (#6526 H5D__chunk_unlock and #6389).

Expected behavior
It should report and error with segfaulting.

Platform (please complete the following information)

Additional context

Reproducer from Python (with h5py):

>>> import numpy as np, h5py, tables
... base=[b'data1',b'data1',b'data3',b'data4',b' data5 ']
... arr=np.array([base[i%5] for i in range(1250)],dtype='S8').reshape(125,10)
... with h5py.File('/tmp/x2.h5','w') as f:
...     f.create_dataset('d', data=arr, chunks=(10,5), compression=32026)   # -> "buffer size is too small after filter callback"
...
<HDF5 dataset "d": shape (125, 10), type "|S8">
Traceback (most recent call last):
  File "<python-input-0>", line 4, in <module>
    with h5py.File('/tmp/x2.h5','w') as f:
         ~~~~~~~~~^^^^^^^^^^^^^^^^^^
  File "h5py/_objects.pyx", line 54, in h5py._objects.with_phil.wrapper
  File "h5py/_objects.pyx", line 55, in h5py._objects.with_phil.wrapper
  File "/usr/local/python/python-3.14/std/lib64/python3.14/site-packages/h5py/_hl/files.py", line 638, in __exit__
    self.close()
    ~~~~~~~~~~^^
  File "/usr/local/python/python-3.14/std/lib64/python3.14/site-packages/h5py/_hl/files.py", line 620, in close
    self.id._close_open_objects(h5f.OBJ_LOCAL | ~h5f.OBJ_FILE)
    ~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "h5py/_objects.pyx", line 54, in h5py._objects.with_phil.wrapper
  File "h5py/_objects.pyx", line 55, in h5py._objects.with_phil.wrapper
  File "h5py/h5f.pyx", line 359, in h5py.h5f.FileID._close_open_objects
RuntimeError: Output pipeline failed (buffer size is too small after filter callback)
>>>
Segmentation fault (core dumped)

I asked Claude to translate that Python reproducer to a standalone HDF5 reproducer:

/*
 * Standalone HDF5 reproducer using the REAL blosc2 filter (id 32026) -- the
 * C equivalent of the original Python/h5py reproducer:
 *
 *     import numpy as np, h5py, tables
 *     base=[b'data1',b'data1',b'data3',b'data4',b' data5 ']
 *     arr=np.array([base[i%5] for i in range(1250)],dtype='S8').reshape(125,10)
 *     with h5py.File('x2.h5','w') as f:
 *         f.create_dataset('d', data=arr, chunks=(10,5), compression=32026)
 *
 * Point HDF5_PLUGIN_DIR (below) at the directory containing libh5blosc2.so, or
 * override at runtime with the HDF5_PLUGIN_PATH environment variable.
 *
 * Exit status:
 *   0  -> H5Fclose returned an error cleanly (no crash)
 *   2  -> H5Fclose unexpectedly succeeded
 *   (segfault / core dump -> the bug is present)
 */

#include "hdf5.h"

#include <stdio.h>
#include <stdlib.h>
#include <string.h>

#define BLOSC2_FILTER_ID ((H5Z_filter_t)32026)

#ifndef HDF5_PLUGIN_DIR
#define HDF5_PLUGIN_DIR \
    "/usr/local/python/python-3.14/std/lib/python3.14/site-packages/hdf5plugin/plugins"
#endif

int
main(void)
{
    /* 5 base strings, cycled -- matches the numpy S8 test array. */
    static const char *const base[5] = {"data1", "data1", "data3", "data4", " data5 "};

    const hsize_t dims[2]  = {125, 10};
    const hsize_t chunk[2] = {10, 5};
    const size_t  N        = (size_t)dims[0] * (size_t)dims[1]; /* 1250 */
    const size_t  S        = 8;                                  /* 'S8' */

    printf("HDF5 version: %u.%u.%u\n",
           H5_VERS_MAJOR, H5_VERS_MINOR, H5_VERS_RELEASE);

    /* Make the blosc2 filter plugin discoverable, like `import tables`. */
    if (H5PLappend(HDF5_PLUGIN_DIR) < 0) {
        fprintf(stderr, "H5PLappend failed for %s\n", HDF5_PLUGIN_DIR);
        return 1;
    }

    htri_t avail = H5Zfilter_avail(BLOSC2_FILTER_ID);
    printf("blosc2 filter (32026) available before create: %d\n", (int)avail);

    /* Fixed-length 8-byte, null-padded string type == numpy dtype 'S8'. */
    hid_t st = H5Tcopy(H5T_C_S1);
    H5Tset_size(st, S);
    H5Tset_strpad(st, H5T_STR_NULLPAD);

    /* Build the data: row i is base[i % 5], null-padded to 8 bytes. */
    char *data = (char *)calloc(N, S);
    if (data == NULL)
        return 1;
    for (size_t i = 0; i < N; i++) {
        const char *s   = base[i % 5];
        size_t      len = strlen(s);
        if (len > S)
            len = S;
        memcpy(data + i * S, s, len); /* remaining bytes stay 0 (null pad) */
    }

    /* Strong close degree: H5Fclose flushes & closes the still-open dataset,
     * driving the filter during flush-on-close -- the crash path. */
    hid_t fapl = H5Pcreate(H5P_FILE_ACCESS);
    H5Pset_fclose_degree(fapl, H5F_CLOSE_STRONG);

    hid_t fid = H5Fcreate("/tmp/hdf5_repro/out_blosc2.h5", H5F_ACC_TRUNC,
                          H5P_DEFAULT, fapl);
    if (fid < 0) {
        fprintf(stderr, "H5Fcreate failed\n");
        return 1;
    }

    hid_t sid = H5Screate_simple(2, dims, NULL);

    hid_t dcpl = H5Pcreate(H5P_DATASET_CREATE);
    H5Pset_chunk(dcpl, 2, chunk);
    /* Mandatory (non-optional) filter, no client cd_values -- blosc2's
     * set_local callback fills in the required values, exactly as when h5py
     * is called with compression=32026 and no options. */
    if (H5Pset_filter(dcpl, BLOSC2_FILTER_ID, 0 /* not optional */, 0, NULL) < 0) {
        fprintf(stderr, "H5Pset_filter failed\n");
        return 1;
    }

    hid_t did = H5Dcreate2(fid, "d", st, sid,
                           H5P_DEFAULT, dcpl, H5P_DEFAULT);
    if (did < 0) {
        fprintf(stderr, "H5Dcreate2 failed (filter plugin not loaded?)\n");
        return 1;
    }

    printf("blosc2 filter available after create: %d\n",
           (int)H5Zfilter_avail(BLOSC2_FILTER_ID));

    /* Data lands in the chunk cache; the filter runs at flush. */
    herr_t status = H5Dwrite(did, st, H5S_ALL, H5S_ALL, H5P_DEFAULT, data);
    printf("H5Dwrite returned %d\n", (int)status);

    H5Sclose(sid);
    H5Pclose(dcpl);
    H5Pclose(fapl);
    H5Tclose(st);
    /* Dataset intentionally left OPEN so the flush happens in H5Fclose. */

    printf("Closing file (flush-on-close runs the blosc2 filter)...\n");
    fflush(stdout);

    status = H5Fclose(fid); /* <-- crash path in the original report */

    printf("H5Fclose returned %d\n", (int)status);
    free(data);

    if (status >= 0) {
        printf("UNEXPECTED: H5Fclose succeeded.\n");
        return 2;
    }
    printf("OK: filter failure surfaced as a clean error, no crash.\n");
    return 0;
}

Activity

  1. ArvidJB commented on Aug 7, 2026

    @ArvidJB
    Author

    Note that the problem of blosc2 returning the wrong buffer size looks like it was fixed in this commit:
    HDFGroup/hdf5_plugins@a6502a9

    So to reproduce you probably need an older release of hdf5_plugins?

  2. added theissue type on Aug 10, 2026
  3. self-assigned this
    on Aug 11, 2026
  4. added this to the milestone on Aug 11, 2026
  5. moved this from To be triaged to Planning in HDF5 - TRIAGE & TRACKon Aug 11, 2026
  6. updated Priority P2 - Medium and Effort Medium on Sep 11, 2026
  7. modified the milestones: , HDF5 2.3.0 on Sep 11, 2026
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment

Metadata

Metadata

Assignees

Labels

Component - C LibraryCore C library issues (usually in the src directory)

Type

Fields

Priority

P2 - Medium

Effort

Medium

Projects

Relationships

None yet

Development

No branches or pull requests

Issue actions