diff --git a/.github/actions/run-hsds/action.yml b/.github/actions/run-hsds/action.yml new file mode 100644 index 00000000..33dab9bd --- /dev/null +++ b/.github/actions/run-hsds/action.yml @@ -0,0 +1,107 @@ +name: "Start HSDS and run its test suite" +description: > + Installs HSDS, starts it either as a bare process ("manual") or via + Docker Compose ("docker"), waits for node startup, then runs testall.py + against it. Shared by any job that needs a running HSDS instance. +inputs: + build-method: + description: "How to start HSDS: 'manual' or 'docker'" + required: true + os: + description: "matrix.os of the calling job (used to skip docker on windows)" + required: true +runs: + using: "composite" + steps: + - name: Install dependencies + shell: bash + run: | + python -m pip install --upgrade pip + python -m pip install flake8 pytest + if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + + - name: Install package + shell: bash + run: | + pip install -e . + + - name: Run unit tests + shell: bash + run: | + pytest + + - name: Start HSDS (Manual) + shell: bash + if: ${{ inputs.build-method == 'manual' }} + env: + ADMIN_PASSWORD: admin + ADMIN_USERNAME: admin + USER_NAME: test_user1 + USER_PASSWORD: test + USER2_NAME: test_user2 + USER2_PASSWORD: test + HSDS_ENDPOINT: http://127.0.0.1:5101 + BUCKET_NAME: hsdstest + LOG_LEVEL: DEBUG + run: | + cp admin/config/groups.default admin/config/groups.txt + cp admin/config/passwd.default admin/config/passwd.txt + hsds --root_dir data --host localhost --port 5101 --password_file admin/config/passwd.txt --logfile hs.log --loglevel DEBUG --config_dir=admin/config --count=4 & + + - name: Start HSDS (Docker) + shell: bash + if: ${{ inputs.build-method == 'docker' && inputs.os != 'windows-latest' }} + env: + ADMIN_PASSWORD: admin + ADMIN_USERNAME: admin + USER_NAME: test_user1 + USER_PASSWORD: test + USER2_NAME: test_user2 + USER2_PASSWORD: test + HSDS_ENDPOINT: http://127.0.0.1:5101 + BUCKET_NAME: hsdstest + LOG_LEVEL: DEBUG + run: | + cp admin/config/groups.default admin/config/groups.txt + cp admin/config/passwd.default admin/config/passwd.txt + ./build.sh --nolint + ./runall.sh + + - name: Wait for node startup + shell: bash + run: | + sleep 45 + + - name: Run HSDS tests + if: ${{ !(inputs.build-method == 'docker' && inputs.os == 'windows-latest') }} + id: hsds-tests + shell: bash + env: + ADMIN_PASSWORD: admin + ADMIN_USERNAME: admin + USER_NAME: test_user1 + USER_PASSWORD: test + USER2_NAME: test_user2 + USER2_PASSWORD: test + BUCKET_NAME: hsdstest + LOG_LEVEL: DEBUG + run: | + python testall.py + + - name: Show HSDS Logs on Fail (Docker) + # Only run if the whole workflow failed due to HSDS tests + if: ${{ failure() && steps.hsds-tests.outcome == 'failure' && (inputs.build-method == 'docker' && inputs.os != 'windows-latest') }} + shell: bash + run: | + docker logs hsds-sn-1 >&1 + docker logs hsds-dn-1 >&1 + docker logs hsds-dn-2 >&1 + docker logs hsds-dn-3 >&1 + docker logs hsds-dn-4 >&1 + + - name: Show HSDS Logs on Fail (Manual) + # Only run if the whole workflow failed due to HSDS tests + if: ${{ failure() && steps.hsds-tests.outcome == 'failure' && (inputs.build-method == 'manual') }} + shell: bash + run: | + cat hs.log diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 7e24ec14..a4d17535 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -17,7 +17,7 @@ jobs: fail-fast: false matrix: os: [ubuntu-22.04, ubuntu-latest, windows-latest] - python-version: ["3.9", "3.10", "3.11", "3.12"] + python-version: ["3.11", "3.12", "3.13"] build-method: ["manual", "docker"] runs-on: ${{ matrix.os }} @@ -28,119 +28,56 @@ jobs: with: python-version: ${{ matrix.python-version }} - - name: Install dependencies - shell: bash - run: | - python -m pip install --upgrade pip - python -m pip install flake8 pytest - if [ -f requirements.txt ]; then pip install -r requirements.txt; fi - - name: Lint with flake8 shell: bash run: | + python -m pip install --upgrade pip + python -m pip install flake8 # stop the build if there are Python syntax errors or undefined names flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide flake8 . --count --max-complexity=10 --max-line-length=127 --statistics - - - name: Install package - shell: bash - run: | - pip install -e . - - name: Run unit tests - shell: bash - run: | - pytest - - name: Start HSDS (Manual) - shell: bash - if: ${{matrix.build-method == 'manual'}} - env: - ADMIN_PASSWORD: admin - ADMIN_USERNAME: admin - USER_NAME: test_user1 - USER_PASSWORD: test - USER2_NAME: test_user2 - USER2_PASSWORD: test - HSDS_ENDPOINT: http://127.0.0.1:5101 - BUCKET_NAME: hsdstest - LOG_LEVEL: DEBUG - run: | - cp admin/config/groups.default admin/config/groups.txt - cp admin/config/passwd.default admin/config/passwd.txt - hsds --root_dir data --host localhost --port 5101 --password_file admin/config/passwd.txt --logfile hs.log --loglevel DEBUG --config_dir=admin/config --count=4 & + - name: Start HSDS and run its tests + uses: ./.github/actions/run-hsds + with: + build-method: ${{ matrix.build-method }} + os: ${{ matrix.os }} - - name: Start HSDS (Docker) - shell: bash + - name: Shut down Docker if: ${{matrix.build-method == 'docker' && matrix.os != 'windows-latest'}} - env: - ADMIN_PASSWORD: admin - ADMIN_USERNAME: admin - USER_NAME: test_user1 - USER_PASSWORD: test - USER2_NAME: test_user2 - USER2_PASSWORD: test - HSDS_ENDPOINT: http://127.0.0.1:5101 - BUCKET_NAME: hsdstest - LOG_LEVEL: DEBUG run: | - cp admin/config/groups.default admin/config/groups.txt - cp admin/config/passwd.default admin/config/passwd.txt - ./build.sh --nolint - ./runall.sh + ./stopall.sh - - name: Wait for node startup - shell: bash - run: | - sleep 45 + h5pyd-integration: + # Combined HSDS + h5pyd test. Unlike build-and-test, a failure here can be + # a real h5pyd bug (not just HSDS), so it's kept as its own job/check. + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" - - name: Run HSDS tests - if: ${{!(matrix.build-method == 'docker' && matrix.os == 'windows-latest')}} - id: hsds-tests - shell: bash - env: - ADMIN_PASSWORD: admin - ADMIN_USERNAME: admin - USER_NAME: test_user1 - USER_PASSWORD: test - USER2_NAME: test_user2 - USER2_PASSWORD: test - BUCKET_NAME: hsdstest - LOG_LEVEL: DEBUG - run: | - python testall.py - - - name: Show HSDS Logs on Fail (Docker) - # Only run if the whole workflow failed due to HSDS tests - if: ${{failure() && steps.hsds-tests.outcome == 'failure' && (matrix.build-method == 'docker' && matrix.os != 'windows-latest')}} - run: | - docker logs hsds-sn-1 >&1 - docker logs hsds-dn-1 >&1 - docker logs hsds-dn-2 >&1 - docker logs hsds-dn-3 >&1 - docker logs hsds-dn-4 >&1 - - - name: Show HSDS Logs on Fail (Manual) - # Only run if the whole workflow failed due to HSDS tests - if: ${{failure() && steps.hsds-tests.outcome == 'failure' && (matrix.build-method == 'manual')}} - run: | - cat hs.log + - name: Start HSDS and run its tests + uses: ./.github/actions/run-hsds + with: + build-method: docker + os: ubuntu-latest - name: Checkout h5pyd - if: ${{ ( matrix.os != 'windows-latest' ) }} uses: actions/checkout@v4 with: repository: HDFGroup/h5pyd path: ${{github.workspace}}/h5pyd - name: Install h5pyd - if: ${{ ( matrix.os != 'windows-latest' ) }} working-directory: ${{github.workspace}}/h5pyd run: | pip install . -v - + - name: Create h5pyd test folder - if: ${{ ( matrix.os != 'windows-latest' ) }} working-directory: ${{github.workspace}}/h5pyd env: HS_USERNAME: test_user1 @@ -153,7 +90,6 @@ jobs: hstouch -e $HS_ENDPOINT /home/test_user1/h5pyd_tests/ - name: Run h5pyd tests - if: ${{ ( matrix.os != 'windows-latest' ) }} working-directory: ${{github.workspace}}/h5pyd env: HS_USERNAME: test_user1 @@ -166,10 +102,70 @@ jobs: python testall.py - name: Shut down Docker - if: ${{matrix.build-method == 'docker' && matrix.os != 'windows-latest'}} + if: always() + run: | + ./stopall.sh + + test-data-setup: + # Loads the sample test data described in docs/post_install.md + # ("Test Data Setup") so integration tests that depend on it actually + # exercise their assertions instead of skipping with + # "WARNING: Failed to get domain: ... Is test data setup?" + needs: h5pyd-integration + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Start HSDS + uses: ./.github/actions/run-hsds + with: + build-method: docker + os: ubuntu-latest + + - name: Checkout h5pyd + uses: actions/checkout@v4 + with: + repository: HDFGroup/h5pyd + path: ${{github.workspace}}/h5pyd + + - name: Install h5pyd + working-directory: ${{github.workspace}}/h5pyd + run: | + pip install . -v + + - name: Load test data + env: + HS_USERNAME: test_user1 + HS_PASSWORD: test + HS_ENDPOINT: http://127.0.0.1:5101 + run: | + hstouch -e $HS_ENDPOINT /home/test_user1/test/ + wget https://s3.amazonaws.com/hdfgroup/data/hdf5test/tall.h5 + hsload -v -e $HS_ENDPOINT tall.h5 /home/test_user1/test/ + hsls -r -e $HS_ENDPOINT /home/test_user1/test/tall.h5 + + - name: Run HSDS integration tests (with test data loaded) + env: + ADMIN_PASSWORD: admin + ADMIN_USERNAME: admin + USER_NAME: test_user1 + USER_PASSWORD: test + USER2_NAME: test_user2 + USER2_PASSWORD: test + BUCKET_NAME: hsdstest + LOG_LEVEL: DEBUG + run: | + python testall.py --skip_unit + + - name: Shut down Docker + if: always() run: | ./stopall.sh - + build-and-test-socket: env: ADMIN_PASSWORD: admin @@ -178,12 +174,7 @@ jobs: USER_PASSWORD: test USER2_NAME: test_user2 USER2_PASSWORD: test - HSDS_USERNAME: test_user1 - HSDS_PASSWORD: test - HSDS_PATH: /home/test_user1/ - HDF5_API_TEST_PATH_PREFIX: /home/test_user1/ HSDS_ENDPOINT: http+unix://%2Ftmp%2Fhs%2Fsn_1.sock - HDF5_VOL_CONNECTOR: REST ROOT_DIR: ${{github.workspace}}/hsdsdata BUCKET_NAME: hsdstest diff --git a/Dockerfile.lambda b/Dockerfile.lambda deleted file mode 100644 index f9ec122d..00000000 --- a/Dockerfile.lambda +++ /dev/null @@ -1,20 +0,0 @@ -FROM public.ecr.aws/lambda/python:3.9 -RUN yum -y install git - - -# Install numcodecs from the specific commit since we need the brand new shuffle codec... -#RUN DISABLE_NUMCODECS_AVX2=1 CFLAGS=-DHAVE_UNISTD_H \ -#` pip install --no-cache-dir \ -# git+https://github.com/zarr-developers/numcodecs.git@d16d1eac5198166a24726ffe808e3dcfcab9700d#egg=numcodecs - -LABEL maintainer="John Readey " - -# Install HSDS -RUN mkdir /usr/local/src/hsds/ /etc/hsds/ -COPY . /usr/local/src/hsds -COPY admin/config/config.yml /etc/hsds/ -COPY entrypoint.sh / -RUN /bin/bash -c "pip install /usr/local/src/hsds" - -COPY lambda_function.py ./ -CMD ["lambda_function.lambda_handler"] diff --git a/README.md b/README.md index 586ddccf..a92b5108 100755 --- a/README.md +++ b/README.md @@ -10,12 +10,47 @@ Data can be stored in either a POSIX files system, or using object-based storage AWS S3, Azure Blob Storage, or [MinIO](https://min.io). HSDS can be run a single machine with or without Docker or on a cluster using Kubernetes (or AKS on Microsoft Azure). -## Quick Start +## What's New in HSDS v1.0.0 + +### New Features + +- **Query improvements**: added a dedicated `/datasets/{id}/query` path for read-only queries, restored `query`-param support on `PUT /datasets/{id}/value` (query-based conditional update), updated the query syntax and evaluation engine (now backed by the `h5json` library), and extended query support to multi-dimensional datasets (previously limited to 1-D). +- **Region reference support**: `GET`/`PUT` value requests can now read and write HDF5 region references. +- **Consolidated domain metadata**: added support for generating and serving a consolidated summary of all objects in a domain, reducing the number of requests needed to inspect a domain's full structure. +- **Client-provided object IDs and timestamps**: `POST` requests for datasets, groups, and datatypes can now specify the object's ID and creation timestamp directly (subject to a configurable `max_timestamp_drift`), rather than always having the server generate them - useful for replication/migration scenarios. +- **Richer object-creation payloads**: `POST` requests for datasets and groups can now initialize attributes, links, and (for datasets) initial data values in the same request that creates the object. +- **Batch object creation**: added multi-object creation support for datasets, groups, and datatypes (create several objects in a single `POST`), backed by a new async `DomainCrawler`/`PostCrawler`-based implementation. +- **Improved array (`H5T_ARRAY`) dtype handling**: fixed selection/read/write handling for datasets whose own type is an array (subarray) dtype, not just array-typed fields nested in a compound type. (Note: this covers binary reads/writes; JSON-encoded writes to a top-level `H5T_ARRAY` dataset are still tracked as a known issue.) +- **Formal OpenAPI specification**: added `openapi.yml`, a full OpenAPI 3 description of the HSDS REST API, plus new `/about` and `/info` endpoints. + +### Notable Bug Fixes + +- Fixed a race condition in node "ready" state handling that could cause requests to be routed to a node before it was fully initialized. +- Fixed a hang in the `DomainCrawler`'s data-write handler. +- Fixed handling of `H5S_UNLIMITED` in dataset `maxdims`. +- Fixed scalar-dataset value access, uninitialized attribute values, and a `chunkref`-indirect-layout bug. +- Fixed binary field-selection reads/writes (selecting a subset of compound-type fields). +- `runall.sh` now waits until the service reports a `READY` state before returning, instead of a fixed sleep. + +### Other Changes -### With Github codespaces +- Internal data-type, array, object-ID, shape, dataset, filter, link, and time utilities have been migrated to the new standalone [h5json](https://github.com/HDFGroup/hdf5-json) library, replacing several local `hsds/util/*` modules (see Breaking Changes below). +- Removed AWS Lambda support (`Dockerfile.lambda`, `lambda_function.py`, and related docs/config). +- Minimum supported Python version is now 3.11 (up from 3.10). -Launch a Codespaces environment by clicking the banner __["Open in GitHub Codespaces"](https://codespaces.new/HDFGroup/hsds)__. Once the codespace is ready, type: -`python testall.py` in the terminal window to run the test suite. +## Breaking Changes in v1.0.0 + +v1.0.0 is a major version bump and includes some deliberate, non-backward-compatible changes. If you're upgrading from a 0.x release, be aware of the following: + +- **Query response shape changed**: previously, using the `query` parameter on a `PUT /datasets/{id}/value` request returned the *updated values* for the matching rows in a `"value"` field. Now that same request returns the matching row **indices** in an `"indices"` field instead (no `"value"` key). To retrieve matches read-only (without updating), use the new dedicated `GET /datasets/{id}/query` endpoint, which also returns `"indices"`. `GET /datasets/{id}/value?query=...` (read, non-update) is unchanged and still returns `"value"`. +- **Dataset chunk layout moved in the JSON schema**: `GET /datasets/{id}` no longer returns a top-level `"layout"` key. Layout information (whether client-specified or server-generated) now always appears nested under `"creationProperties"."layout"`. Clients that read `dataset_json["layout"]` directly need to switch to `dataset_json["creationProperties"]["layout"]`. +- **External link field renamed**: external link objects now report the target file/domain under a `"file"` key instead of `"h5domain"` in API responses. Creating a link with `"h5domain"` in the request body is still accepted for backward compatibility, but it will no longer be echoed back that way - expect `"file"` in the response. +- **Status code change for duplicate object IDs**: `POST` requests that specify a client-provided object ID which already exists now return `400 Bad Request` (previously `500 Internal Server Error`). +- **AWS Lambda support removed**: HSDS can no longer be deployed as an AWS Lambda function; `Dockerfile.lambda`, `lambda_function.py`, `hsds/util/awsLambdaClient.py`, and the associated setup docs have been removed. +- **New required dependency**: HSDS now depends on the [h5json](https://github.com/HDFGroup/hdf5-json) package for core type/array/object-ID/shape utilities. Code that imported HSDS's own `hsds.util.idUtil`, `hsds.util.timeUtil`, `hsds.util.hdf5dtype`, or `hsds.util.arrayUtil` modules directly will break, as those modules have been removed in favor of `h5json` equivalents. +- **Minimum Python version raised to 3.11** (from 3.10). + +## Quick Start ### On your desktop/laptop @@ -51,9 +86,6 @@ For complete instructions to install on AWS Kubernetes Service (EKS): - See: [docs/kubernetes_install_aws.md](docs/kubernetes_install_aws.md) -For complete instructions to install on AWS Lambda: - -- See: [docs/aws_lambda_setup.md](docs/aws_lambda_setup.md). ### On Azure diff --git a/admin/config/config.yml b/admin/config/config.yml index 756be465..32cc3bd2 100755 --- a/admin/config/config.yml +++ b/admin/config/config.yml @@ -13,8 +13,6 @@ aws_dynamodb_gateway: null # use for dynamodb endpint, e.g. 'https://dynamodb.us aws_dynamodb_users_table: null # set to table name if dynamodb is used to store usernames and passwords azure_connection_string: null # use for connecting to Azure blob storage azure_resource_id: null # resource id for use with Azure Active Directory -azure_storage_account: null # storage account to use on Azure -azure_resource_group: null # Azure resource group the container (BUCKET_NAME) belongs to root_dir: null # base directory to use for Posix storage password_salt: null # salt value to generate password based on username. Not recommended for public deployments bucket_name: hsdstest # set to use a default bucket, otherwise bucket param is needed for all requests @@ -47,6 +45,7 @@ flush_sleep_interval: 1 # time to wait between checking on dirty objects flush_timeout: 10 # max time to wait on all I/O operations to complete for a flush min_chunk_size: 1m # 1 MB max_chunk_size: 4m # 4 MB +default_vlen_type_size: 128 # guess for average size of variable length types max_request_size: 100m # 100 MB - should be no smaller than client_max_body_size in nginx tmpl (if using nginx) max_chunks_per_folder: 0 # max number of chunks per s3 folder. 0 for unlimiited max_task_count: 100 # maximum number of concurrent tasks per node before server will return 503 error @@ -70,15 +69,13 @@ admin_group: null # enable admin privileges for any user in this group openid_provider: azure # OpenID authentication provider openid_url: null # OpenID connect endpoint if provider is not azure or google openid_audience: null # OpenID audience. This is synonymous with azure_resource_id for azure -openid_claims: unique_name,appid,roles # Comma seperated list of claims to resolve to usernames. +openid_claims: unique_name,appid,roles # Comma separated list of claims to resolve to usernames. chaos_die: 0 # if > 0, have nodes randomly die after n seconds (for testing) -standalone_app: false # True when run as a single application blosc_nthreads: 2 # number of threads to use for blosc compression. Set to 0 to have blosc auto-determine thread count http_compression: false # Use HTTP compression http_max_url_length: 512 # Limit http request url + params to be less than this http_streaming: true # enable HTTP streaming k8s_dn_label_selector: app=hsds # Selector for getting data node pods from a k8s deployment (https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#label-selectors) -k8s_namespace: null # Specifies if a the client should be limited to a specific namespace. Useful for some RBAC configurations. restart_policy: on-failure # Docker restart policy # the following two values with give backoff times of approx: 0.2, 0.4, 0.8, 1.6, 3.2, 6.4, 12.8 dn_max_retries: 7 # number of time to retry DN requests @@ -88,11 +85,10 @@ allow_any_bucket_read: true # enable reads to buckets other than default bucket allow_any_bucket_write: true # enable writes to buckets other than default bucket bit_shuffle_default_blocksize: 2048 # default blocksize for bitshuffle filter max_rangeget_gap: 1024 # max gap in byte for intelligent range get requests +predate_max_time: 10.0 # max delta between object created timestamp in request and actual time +posix_delay: 0.0 # delay for POSIX IO operations for simulating cloud storage latencies +max_timestamp_drift: 300 # number of seconds a client-based timestamp can differ from current time # DEPRECATED - the remaining config values are not used in currently but kept for backward compatibility with older container images -aws_lambda_chunkread_function: null # name of aws lambda function for chunk reading -aws_lambda_threshold: 4 # number of chunks per node per request to reach before using lambda -aws_lambda_max_invoke: 1000 # max number of lambda functions to invoke simultaneously -aws_lambda_gateway: null # use lambda endpoint for region HSDS is running in k8s_app_label: null # The app label for k8s deployments (use k8s_dn_label_selector instead) write_zero_chunks: False # write chunk to storage even when it's all zeros (or in general equal to the fill value) max_chunks_per_request: 1000 # maximum number of chunks to be serviced by one request @@ -103,4 +99,9 @@ data_cache_max_req_size: 128k # max size for rangeget fetches data_cache_expire_time: 3600 # expire cache items after one hour data_cache_page_size: 4m # page size for range get cache, set to zero to disable proxy data_cache_max_concurrent_read: 16 # maximum number of inflight storage read requests -domain_req_max_objects_limit: 500 # maximum number of objects to return in GET domain request with use_cache +domain_req_max_objects_limit: 500 # maximum number of objects to return in GET domain request +azure_storage_account: null # storage account to use on Azure +azure_resource_group: null # Azure resource group the container (BUCKET_NAME) belongs to with use_cache +standalone_app: false # True when run as a single application +k8s_namespace: null # Specifies if a the client should be limited to a specific namespace. Useful for some RBAC configurations. +max_compact_dset_size: 65536 # size in bytes for maximum compact storage size \ No newline at end of file diff --git a/admin/docker/docker-compose.swagger.yml b/admin/docker/docker-compose.swagger.yml new file mode 100644 index 00000000..8deaf24f --- /dev/null +++ b/admin/docker/docker-compose.swagger.yml @@ -0,0 +1,10 @@ +services: + swagger-ui: + image: swaggerapi/swagger-ui + restart: ${RESTART_POLICY} + environment: + - SWAGGER_JSON=/spec/openapi.yml + ports: + - ${SWAGGER_PORT}:8080 + volumes: + - ${PWD}/openapi.yml:/spec/openapi.yml:ro diff --git a/docs/aws_lambda_setup.md b/docs/aws_lambda_setup.md deleted file mode 100644 index 3d7e553a..00000000 --- a/docs/aws_lambda_setup.md +++ /dev/null @@ -1,143 +0,0 @@ -# HSDS for AWS Lambda - -AWS Lambda is a technology that enables code to be run without the need to provision a server. For AWS deployments, HSDS can be deployed as a AWS Lambda function to provide more scalability and parallelism than would be practical compared with containers running in Docker or Kubernetes (AWS Lambda supports up to 1000-way parallelism by default). - -Each Lambda invocation will be charged based on how long the code took to execute (typically 2-4 seconds per request) and memory used (can be configured to anything between 1G and 10GB). This is especially attractive for deployments were the service will be used intermittantly, as there is no charge unless the Lambda function is invoked. - -Compared with a traiditional deployment, a Lambda deployment is not optimal for situtaions where the lowest possible latency is desired. Since the Lambda function takes a certain amount of time to "spin up", -the average latency will be higher compared to a lightly loaded server deployment. On the other hand, Lambda invocations generally have a more consistent latency. Even with a high request rate, the latency should be the same or lower since there is little contention among the executing Lambda functions. - -HSDS for AWS Lambda supports the full HDF Rest API, though some care should be taken if Lambda is -used for operations that modify storage. If multiple update requests are sent simultaneously with Lambda it is possible to have a race condition where some udpates will get overwritten. For example, if two Lambda functions are invoked simultaneouly, and attempt to modify the same chunk, -it is possible that one function will overwrite the results of the other. - -# Function Creation - -To use HSDS for Lambda, follow these steps: - -1. In the AWS Management Console, select the Lambda service for your desired region -2. Click the "Create Function" button -3. Choose the "Container Image" option -4. Enter a function name (e.g. "hslambda") -5. Click the "Browse Image" button and select the image you uploaded to ECR. Use one of the images "hslambda..." image tags in https://gallery.ecr.aws/w7l0z8b2/hdfgroup, or upload your own image to ECR. For the later, refer to "Building Lambda Images" below -6. Wait for the image to load -7. Click "Create Function". Wait for image to load -8. Select the "Configuration" tab and change the memory value to at least 1024MB and Timeout of 30 sec (later try out different values for these to see which works best for your workload) -9. Also in the "Configuration" tab, select "Environment variables", click the "Edit" button, and then the "Add environment variable" button. Enter a key of "AWS_S3_GATEWAY" and a value corresponding to the S3 endpoint for your region. E.g. "http://s3.us-west-2.amazonaws.com" for us-west-2 -10. Next select "Permissions" and click the "Edit" button for "Execution Role". Select (or create) a role that includes at least the policies: "AWSLambdaBasicExecutionRole" and "AmazonS3ReadOnlyAccess". If desired, you may use "AmazonS3FullAccess" (for read-write applications), and/or restrict the resource to a given S3 bucket -11. Select the "Test" tab and press the "Test" button using the default event -12. Function should succeed returning a JSON response with "status_code" of 200, and an "output" value containing general information about HSDS. If a different status_code is returned, review the Log output to determine the nature of the error - -# General Usage - -HSDS for AWS Lambda supports the complete REST API supported by HSDS running as a service. Since AWS Lambda currently doesn't support HTTP requests, it's necessary to "package" the components of a typical -http request into the Lambda event structure. Each event sent to the Lambda function should have key values of "method", "request", and "params" as explained below: - -- "method": one of the values "GET", "PUT", "POST", "DELETE" corresponding to the typical http verbs. -- "request": the api to invoke. This is the part of the url that would normally be placed after the http endpoint. -- "params": a dictionary of query params to be sent to the function - -The following is an example of how a hyperslab selection is packaged into an event: - - { - "method": "GET", - "path": "/datasets/d-d29fda32-85f3-11e7-bf89-0242ac110008/value", - "headers": { - "accept": "application/octet-stream" - }, - "params": { - "domain": "/nrel/wtk-us.h5", - "select": "[0:100,620,1401]", - "bucket": "nrel-pds-hsds" - } - } - -When the accept header with octet-stream is used with dataset value requests, -the body of the response will be hex-encoded (otherwise JSON data is used). - -With the request above, the following response should be returned: - - { - "isBase64Encoded": true, - "statusCode": 200, - "headers": "{\"Content-Type\": \"application/octet-stream\", \"Content-Length\": \"4000\", \"Date\": \"Fri, 16 Sep 2022 03:19:21 GMT\", \"Server\": \"Python/3.9 aiohttp/3. 8.1\"}", - "body": "a8bb1241d0f7...e0bf6b40" - } - -Where the "body" key will consist of 8000 hex characters. - -See: https://github.com/HDFGroup/hdf-rest-api, for a complete description of the HDF REST API. - -# Access Control - -On execution the Lambda function will act using the Lambda function name as username. -Publicly readable domains (a "Default" ACL with read permission set) will be accessible -via the Lambda function without any further action. To enable the Lambda function to -read non-publicly readable domains, use the hsacl tool to add read permission with -the Lambda function name as username. - -For example, if the Lambda function is named "hslambda", and the domain is "/shared/data.h5", -run: - - hsacl /shared/data.h5 +r hslambda - -Note: you'll need to at least briefly run a Docker or Kubernetes-based version of HSDS to -run the above command. - -Similarly, if the Lambda function will be modifying data, add permissions to the Lambda -user for update, write, or delete as needed. The following would give the Lambda function -full control of the domain (other than reading or modifying ACLs): - - hsacl /shared/data.h5 +crud hslambda - -# Building the Lambda Image - -If you wish to build the Lambda image from source, clone this repository and run the script: "lambda_build.sh". This will create a docker image that you can then push to ECR. - -# API Gateway - -Amazon API Gateway is the service for managing HTTP endpoints. By forwarding http requests to AWS Lambda, -API Gateway can be used to provide web server functionality without the need to provision a server instance. -When API Gateway is used with the HSDS Lambda function, the functionality of HSDS will be available for HSDS -clients such as h5pyd and HDF REST VOL (to the clients it will be the same as connecting with a regular HSDS -instance). - -To setup API Gateway with HSDS Lambda, follow these steps: - -1. In the AWS Management Console, select the API Gateway service for you desired region -2. Click the "Create API" button -3. Click the "Build" button in the "HTTP API" type box -4. Click the "Add Integration" button -5. In the drop down, choose "Lambda" -6. Leave the "Version" as 2.0 -7. In the "Lambda function" box, chose the HSDS Lambda function you created earlier and click the "Add Integration" button -8. Enter an API name and click "Next" -9. In the "Configure routes" page, use "$default" for the "ANY" method, click "Next" -10. In the "Configure stages", accept the defaults, click "Next" -11. In the "Review and Create" page, click "Create" -12. Upon successful creation of the gateway, you will see a "Invoke URL". Entering that url in a browser and adding "/about" should return the contents of the about method -13. The same "Invoke URL" can be used as the endpoint for HSDS clients. By default that expected username will be the name of the lamba function and that password will be "lambda" - -# Using h5pyd with HSDS Lambda - -You can use h5pyd (version 0.10.4 and higher) to invoke HSDS Lambda directly (i.e. no need to setup API Gateway). To use, install the Lambda function as described above. Then use the following values for endpoint, username, and password: - - hs_endpoint = http+lambda://hslambda - hs_username = hslambda - hs_password = lambda - -If you named your Lambda function something different than "hslambda", replace hslambda with the function name. Make sure the following environment variables are set as appropriatte for you AWS account and region ("us-west-2" is used below) in which the Lambda function is installed: - - export AWS_ACCESS_KEY_ID=YOUR_ACCESS_KEY_ID - export AWS_SECRET_ACCESS_KEY=YOUR_SECRET_KEY - export AWS_REGION=us-west-2 - export AWS_LAMBDA_GATEWAY=https://lambda.us-west-2.amazonaws.com - -Now you can use h5pyd in the same was as you would with an HSDS server -running. Each REST request that h5pyd would normally send to the server -will be replaced by a Lambda invocation. Depending on the program, -the can be quite a bit slower than running with a server (each Lambda -invocation will take 2 seconds or so). - -Use the AWS Management Console to monitor Lambda usage and view logs -as needed. diff --git a/docs/docker_install_aws.md b/docs/docker_install_aws.md index 3378a77f..63e31349 100644 --- a/docs/docker_install_aws.md +++ b/docs/docker_install_aws.md @@ -13,12 +13,11 @@ Export environment variables as shown in "Sample .bashrc" below. 8. Copy the file "passwd.default" to "passwd.txt". Add any usernames/passwords you wish. Modify existing passwords (for admin, test_user1, test_user2) for security. 9. If group-level permissions are desired (See [Authorization](authorization.md)), copy the file "groups.default" to "groups.txt". Modify existing groups as needed 10. Create environment variables as in "Sample .bashrc" below -11. Setup Lambda if desired. See [AWS Lambda Setup](aws_lambda_setup.md) -12. Create the file **admin/config/override.yml** for deployment specific settings (see "Sample override.yml") -13. Start the service `$./runall.sh ` where n is the number of containers desired (defaults to 4) -14. Run `$ docker ps` and verify that the containers are running: hsds_head, hsds_sn_[1-n], hsds_dn_[1-n] -15. Run `$ curl http://127.0.0.1:${SN_PORT}/about` and verify that "cluster_state" is "READY" (might need to give it a minute or two) -16. Perform post install configuration. See: [Post Install Configuration](post_install.md) +11. Create the file **admin/config/override.yml** for deployment specific settings (see "Sample override.yml") +12. Start the service `$./runall.sh ` where n is the number of containers desired (defaults to 4) +13. Run `$ docker ps` and verify that the containers are running: hsds_head, hsds_sn_[1-n], hsds_dn_[1-n] +14. Run `$ curl http://127.0.0.1:${SN_PORT}/about` and verify that "cluster_state" is "READY" (might need to give it a minute or two) +15. Perform post install configuration. See: [Post Install Configuration](post_install.md) Sample .bashrc diff --git a/hsds/async_lib.py b/hsds/async_lib.py index e749e8a2..b6203725 100755 --- a/hsds/async_lib.py +++ b/hsds/async_lib.py @@ -15,18 +15,24 @@ from aiohttp.client_exceptions import ClientError from aiohttp.web_exceptions import HTTPNotFound, HTTPInternalServerError from aiohttp.web_exceptions import HTTPForbidden -from .util.idUtil import isValidUuid, isSchema2Id, getS3Key, isS3ObjKey -from .util.idUtil import getObjId, isValidChunkId, getCollectionForId -from .util.chunkUtil import getDatasetId, getNumChunks, ChunkIterator -from .util.hdf5dtype import getItemSize, createDataType -from .util.arrayUtil import getNumElements, bytesToArray -from .util.dsetUtil import getHyperslabSelection, getFilterOps, getChunkDims, getFilters -from .util.dsetUtil import getDatasetLayoutClass, getDatasetLayout, getShapeDims +from h5json.hdf5dtype import getItemSize +from h5json.hdf5dtype import createDataType +from h5json.array_util import getNumElements, bytesToArray, bytesArrayToList +from h5json.objid import isValidUuid, isSchema2Id, getS3Key, isS3ObjKey +from h5json.objid import getObjId, isValidChunkId, getCollectionForId +from h5json.filters import getFilters +from h5json.shape_util import getShapeDims, getDataSize +from h5json.dset_util import getDatasetLayoutClass, getDatasetLayout, getChunkDims +from h5json.time_util import getNow +from h5json import selections + +from .util.chunkUtil import getDatasetId, getNumChunks, ChunkIterator, getChunkIndex, getChunkIds from .util.storUtil import getStorKeys, putStorJSONObj, getStorJSONObj from .util.storUtil import deleteStorObj, getStorBytes, isStorObj +from .datanode_lib import getFilterOps from . import hsds_logger as log from . import config -import time + # List all keys under given root and optionally update info.json # Note: only works with schema v2 domains! @@ -71,9 +77,10 @@ async def updateDatasetInfo(app, dset_id, dataset_info, bucket=None): msg += f"{dset_id}" log.warn(msg) return + type_json = dset_json["type"] item_size = getItemSize(type_json) - if "layout" not in dset_json: + if not getDatasetLayout(dset_json): msg = "updateDatasetInfo - expected to find layout in dataset_json " msg += f"for {dset_id}" log.warn(msg) @@ -88,7 +95,7 @@ async def updateDatasetInfo(app, dset_id, dataset_info, bucket=None): return # null dataspace if item_size == "H5T_VARIABLE": - # arbitrary lgoical size for vaariable, so just set to allocated size + # arbitrary logical size for variable, so just set to allocated size logical_bytes = dataset_info["allocated_bytes"] else: num_elements = getNumElements(dims) @@ -99,14 +106,14 @@ async def updateDatasetInfo(app, dset_id, dataset_info, bucket=None): layout_class = getDatasetLayoutClass(dset_json) msg = f"updateDatasetInfo - {dset_id} has layout_class: {layout_class}" log.debug(msg) - selection = getHyperslabSelection(dims) # select entire datashape + selection = selections.select(tuple(dims), ...) # select entire datashape linked_bytes = 0 num_linked_chunks = 0 if layout_class == "H5D_CONTIGUOUS_REF": # In H5D_CONTIGUOUS_REF a non-compressed part of the HDF5 is divided # into equal size chunks, so we can just compute link bytes and num - # chunks based on the size of the coniguous dataset + # chunks based on the size of the contiguous dataset layout_dims = getChunkDims(dset_json) num_chunks = getNumChunks(selection, layout_dims) chunk_size = item_size @@ -170,7 +177,7 @@ async def updateDatasetInfo(app, dset_id, dataset_info, bucket=None): # read chunktable one chunk at a time - this can be slow if there # are a lot of chunks, but this is only used by the async bucket # scan task - sel = getHyperslabSelection(chunktable_dims) + sel = selections.select(tuple(chunktable_dims), ...) it = ChunkIterator(chunktable_id, sel, dims) msg = f"updateDatasetInfo - iterating over chunks in {chunktable_id}" log.debug(msg) @@ -262,20 +269,26 @@ def scanRootCallback(app, s3keys): results = app["scanRoot_results"] scanRoot_keyset = app["scanRoot_keyset"] checksums = results["checksums"] + for s3key in s3keys.keys(): if not isS3ObjKey(s3key): - log.info(f"not s3obj key, ignoring: {s3key}") + log.info(f"scanRoot -not s3obj key, ignoring: {s3key}") continue if s3key in scanRoot_keyset: - log.warn(f"scanRoot - dejavu for key: {s3key}") + log.warn(f"scanRoot -scanRoot - dejavu for key: {s3key}") continue scanRoot_keyset.add(s3key) - msg = f"scanRoot adding key: {s3key} to keyset, " + msg = f"scanRoot - adding key: {s3key} to keyset, " msg += f"{len(scanRoot_keyset)} keys" log.debug(msg) objid = getObjId(s3key) + + if objid in app["deleted_ids"]: + log.debug(f"scanRoot - skipping deleted id: {objid}") + continue + etag = None obj_size = None lastModified = None @@ -300,8 +313,15 @@ def scanRootCallback(app, s3keys): is_chunk = True results["num_chunks"] += 1 results["allocated_bytes"] += obj_size + chunk_index = getChunkIndex(objid) + if max(chunk_index) == 0: + # save the first chunk if present + # this will be used to save dataset values to + # the the obj_ids set for small datasets + results["obj_ids"].add(objid) else: results["metadata_bytes"] += obj_size + results["obj_ids"].add(objid) if is_chunk or getCollectionForId(objid) == "datasets": if is_chunk: @@ -339,6 +359,144 @@ def scanRootCallback(app, s3keys): log.error(msg) +async def _getDatsetValueJson(app, dset_id, dset_json, obj_ids, size_limit=None, bucket=None): + """ If the dataset size is less than size_limit, and the chunk_ids for the dataset are + available, return a JSON representation of the dataset values. Otherwise, return None """ + + dims = getShapeDims(dset_json) + if dims is None: + return None # null dataspace + if "type" not in dset_json: + msg = f"_getDatsetValueJson - expected to find type in dataset_json for {dset_id}" + log.warn(msg) + return None + type_json = dset_json["type"] + item_size = getItemSize(type_json) + if item_size == "H5T_VARIABLE": + item_size = 1024 # make a guess for variable length types + dataset_size = getDataSize(dims, item_size) + if dataset_size > size_limit: + log.debug(f"_getDatasetValueJson - dataset size {dataset_size} exceeds limit {size_limit}") + return None + + chunk_dims = getChunkDims(dset_json) + if not chunk_dims: + log.warning(f"_getDatasetValueJson - no layout found for dataset: {dset_id}") + return None + if chunk_dims != dims: + msg = f"_getDatasetValueJson - dataset layout {chunk_dims} does not match dims {dims} " + msg += f"for dataset: {dset_id}, ignoring" + log.warning(msg) + return None + select_all = selections.select(tuple(dims), ...) # select entire datashape + chunk_ids = getChunkIds(dset_id, select_all, dims) + if len(chunk_ids) == 0: + log.debug(f"_getDatasetValueJson - no chunk ids found for dataset: {dset_id}") + return None + if len(chunk_ids) > 1: + log.debug(f"_getDatasetValueJson - more than one chunk id found for dataset: {dset_id}") + return None + chunk_id = chunk_ids[0] + if chunk_id not in obj_ids: + log.debug(f"_getDatasetValueJson - chunk id {chunk_id} not in scanned obj_ids") + return None + log.debug(f"using chunk: {chunk_id} to get dataset value for {dset_id}") + + # fetch the chunk - using getStoreBytes since this will not be used with + # chunk cache or chunk crawlers + # TBD: need parameters for s3path, s3offset, s3size for ref layouts + # regular store read + + filters = getFilters(dset_json) + dt = createDataType(type_json) + filter_ops = getFilterOps(app, dset_id, filters, dtype=dt, chunk_shape=chunk_dims) + + kwargs = { + "filter_ops": filter_ops, + "offset": None, + "length": None, + "bucket": bucket + } + s3key = getS3Key(chunk_id) + + try: + chunk_bytes = await getStorBytes(app, s3key, **kwargs) + except HTTPNotFound: + log.warning(f"_getDatasetValueJson - HTTPNotFound for chunk {chunk_id} bucket:{bucket}") + return None + except HTTPForbidden: + log.warning(f"_getDatasetValueJson - HTTPForbidden for chunk {chunk_id} bucket:{bucket}") + return None + except HTTPInternalServerError: + msg = "_getDatasetValueJson - " + msg += f"HTTPInternalServerError for chunk {chunk_id} bucket:{bucket}" + log.warning(msg) + return None + + if chunk_bytes is None: + msg = f"_getDatasetValueJson -read {chunk_id} bucket: {bucket} returned None" + log.warning(msg) + return None + + arr = bytesToArray(chunk_bytes, dt, chunk_dims) + + json_value = bytesArrayToList(arr) + log.debug(f"_getDatsetValueJson - returning {json_value}") + + return json_value + + +async def getConsolidatedMetaData(app, obj_ids, bucket=None): + # create a consolidated metadata summary for all objects in the domain + # return a dict of obj_ids to their metadata summaries + log.info("getConsolidatedMetaData - creating consolidated metadata summary") + consolidated_metadata = {} + for obj_id in obj_ids: + if isValidChunkId(obj_id): + # skip chunks - we may use the chunk later when processing it's dataset object + continue + s3_key = getS3Key(obj_id) + try: + obj_json = await getStorJSONObj(app, s3_key, bucket=bucket) + except HTTPNotFound: + log.warn(f"HTTPNotFound for {s3_key} bucket:{bucket}") + continue + except HTTPForbidden: + log.warn(f"HTTPForbidden error for {s3_key} bucket:{bucket}") + continue + except HTTPInternalServerError: + msg = f"HTTPInternalServerError error for {s3_key} bucket:{bucket}" + log.warn(msg) + continue + log.debug(f"getConsolidatedMetaData - got json for obj_id: {obj_id}: {obj_json}") + # extract relevant metadata + metadata_summary = {} + if "type" in obj_json: + metadata_summary["type"] = obj_json["type"] + if "shape" in obj_json: + metadata_summary["shape"] = obj_json["shape"] + if "attributes" in obj_json: + metadata_summary["attributes"] = obj_json["attributes"] + if "links" in obj_json: + metadata_summary["links"] = obj_json["links"] + if "creationProperties" in obj_json: + metadata_summary["creationProperties"] = obj_json["creationProperties"] + if getCollectionForId(obj_id) == "datasets": + log.debug("getConsolidatedMetaData - got dataset") + size_limit = 4096 # TBD - make this a config + kwargs = {"size_limit": size_limit, "bucket": bucket} + json_value = await _getDatsetValueJson(app, obj_id, obj_json, obj_ids, **kwargs) + if json_value is not None: + log.debug(f"adding dataset value to metadata summary for dataset: {obj_id}") + metadata_summary["value"] = json_value + else: + log.debug("getConsolidatedMetaData - not a dataset") + + consolidated_metadata[obj_id] = metadata_summary + log.info("getConsolidatedMetaData - done creating consolidated metadata summary") + return consolidated_metadata + + async def scanRoot(app, rootid, update=False, bucket=None): # iterate through all s3 keys under the given root. @@ -380,9 +538,10 @@ async def scanRoot(app, rootid, update=False, bucket=None): results["num_linked_chunks"] = 0 results["linked_bytes"] = 0 results["logical_bytes"] = 0 - results["checksums"] = {} # map of objid to checksums + results["obj_ids"] = set() # map of object ids scanned (and first chunk id for datasets) + results["checksums"] = {} # map of objid to checksums results["bucket"] = bucket - results["scan_start"] = time.time() + results["scan_start"] = getNow(app=app) app["scanRoot_results"] = results app["scanRoot_keyset"] = set() @@ -399,6 +558,9 @@ async def scanRoot(app, rootid, update=False, bucket=None): num_objects += len(results["datasets"]) num_objects += results["num_chunks"] log.info(f"scanRoot - got {num_objects} keys for rootid: {rootid}") + obj_ids = results["obj_ids"] + log.info(f"scanRoot - got {len(obj_ids)} unique object ids") + log.debug(f"scanRoot - obj_ids: {obj_ids}") dataset_results = results["datasets"] for dsetid in dataset_results: @@ -437,7 +599,12 @@ async def scanRoot(app, rootid, update=False, bucket=None): # free up memory used by the checksums del results["checksums"] - results["scan_complete"] = time.time() + results["scan_complete"] = getNow(app=app) + + # extract the obj_ids set, that won't go into .info.json + obj_ids = results["obj_ids"] + del results["obj_ids"] + log.debug(f"obj_ids set: {obj_ids}") if update: # write .info object back to S3 @@ -446,6 +613,17 @@ async def scanRoot(app, rootid, update=False, bucket=None): msg += f"{results}" log.info(msg) await putStorJSONObj(app, info_key, results, bucket=bucket) + + # create a json summary of objects in ths domain + log.debug(f"Creating consolidated metadata summary for root {rootid}") + summary_key = root_prefix + ".summary.json" + summary_data = await getConsolidatedMetaData(app, obj_ids, bucket=bucket) + if summary_data: + log.info(f"Got consolidated metadata summary for root {rootid}") + log.debug(f"Summary data: {summary_data}") + await putStorJSONObj(app, summary_key, summary_data, bucket=bucket) + else: + log.info(f"No consolidated metadata summary for root {rootid}") return results diff --git a/hsds/attr_dn.py b/hsds/attr_dn.py index 456e9854..a4b69c3b 100755 --- a/hsds/attr_dn.py +++ b/hsds/attr_dn.py @@ -12,21 +12,24 @@ # # attribute handling routines # -import time from bisect import bisect_left from aiohttp.web_exceptions import HTTPBadRequest, HTTPConflict, HTTPNotFound, HTTPGone from aiohttp.web_exceptions import HTTPInternalServerError from aiohttp.web import json_response +from h5json.hdf5dtype import getItemSize, createDataType +from h5json.array_util import arrayToBytes, jsonToArray, decodeData +from h5json.array_util import bytesToArray, bytesArrayToList, getNumElements +from h5json.shape_util import getShapeDims +from h5json.time_util import getNow + from .util.attrUtil import validateAttributeName, isEqualAttr -from .util.hdf5dtype import getItemSize, createDataType from .util.globparser import globmatch -from .util.dsetUtil import getShapeDims -from .util.arrayUtil import arrayToBytes, jsonToArray, decodeData -from .util.arrayUtil import bytesToArray, bytesArrayToList, getNumElements from .util.domainUtil import isValidBucketName from .datanode_lib import get_obj_id, get_metadata_obj, save_metadata_obj + +from . import config from . import hsds_logger as log @@ -359,15 +362,17 @@ async def PUT_Attributes(request): log.request(request) app = request.app params = request.rel_url.query - log.debug(f"got PUT_Attributes params: {params}") + log.debug(f"got PUT_Attributes params: {dict(params)}") obj_id = get_obj_id(request) + now = getNow(app) + max_timestamp_drift = int(config.get("max_timestamp_drift", default=300)) if not request.has_body: log.error("PUT_Attribute with no body") raise HTTPBadRequest(message="body expected") body = await request.json() - log.debug(f"got body: {body}") + log.debug(f"PUT_Attributes got body: {body}") if "bucket" in params: bucket = params["bucket"] elif "bucket" in body: @@ -433,11 +438,10 @@ async def PUT_Attributes(request): raise HTTPBadRequest(reason=msg) log.debug(f"got arr: {arr}") log.debug(f"arr.shape: {arr.shape}") - data = arr.tolist() try: - json_data = bytesArrayToList(data) - log.debug(f"converted encoded data to {json_data}") - if attr_shape["class"] == "H5S_SCALAR": + json_data = bytesArrayToList(arr) + log.debug(f"converted encoded data to '{json_data}'") + if attr_shape["class"] == "H5S_SCALAR" and isinstance(json_data, list): attr_json["value"] = json_data[0] # just store the scalar else: attr_json["value"] = json_data @@ -458,11 +462,18 @@ async def PUT_Attributes(request): attributes = obj_json["attributes"] - create_time = time.time() # check for conflicts new_attributes = set() # attribute names that are new or replacements for attr_name in items: attribute = items[attr_name] + if attribute.get("created"): + create_time = attribute["created"] + log.debug(f"attribute {attr_name} has create time: {create_time}") + if abs(create_time - now) > max_timestamp_drift: + log.warn(f"attribute {attr_name} create time stale, ignoring") + create_time = now + else: + create_time = now if attr_name in attributes: log.debug(f"attribute {attr_name} exists") old_item = attributes[attr_name] @@ -510,7 +521,7 @@ async def PUT_Attributes(request): if new_attributes: # update the obj lastModified - now = time.time() + now = getNow(app) obj_json["lastModified"] = now # write back to S3, save to metadata cache await save_metadata_obj(app, obj_id, obj_json, bucket=bucket) @@ -609,7 +620,7 @@ async def DELETE_Attributes(request): if save_obj: # update the object lastModified - now = time.time() + now = getNow(app) obj_json["lastModified"] = now await save_metadata_obj(app, obj_id, obj_json, bucket=bucket) diff --git a/hsds/attr_sn.py b/hsds/attr_sn.py index b7ecdce4..44346929 100755 --- a/hsds/attr_sn.py +++ b/hsds/attr_sn.py @@ -18,21 +18,21 @@ from aiohttp.web import StreamResponse from json import JSONDecodeError +from h5json.hdf5dtype import createDataType, getItemSize +from h5json.array_util import jsonToArray, getNumElements +from h5json.array_util import bytesToArray, arrayToBytes, decodeData, encodeData +from h5json.objid import isValidUuid +from h5json.shape_util import getShapeDims + from .util.httpUtil import getAcceptType, jsonResponse, getHref, getBooleanParam from .util.globparser import globmatch -from .util.idUtil import isValidUuid, getRootObjId from .util.authUtil import getUserPasswordFromRequest, validateUserPassword from .util.domainUtil import getDomainFromRequest, isValidDomain from .util.domainUtil import getBucketForDomain, verifyRoot from .util.attrUtil import validateAttributeName, getRequestCollectionName -from .util.hdf5dtype import validateTypeItem, getBaseTypeJson -from .util.hdf5dtype import createDataType, getItemSize -from .util.arrayUtil import jsonToArray, getNumElements, bytesArrayToList -from .util.arrayUtil import bytesToArray, arrayToBytes, decodeData, encodeData -from .util.dsetUtil import getShapeDims - -from .servicenode_lib import getDomainJson, getObjectJson, validateAction -from .servicenode_lib import getAttributes, putAttributes, deleteAttributes + +from .servicenode_lib import getDomainJson, getAttributeFromRequest, getAttributesFromRequest +from .servicenode_lib import getAttributes, putAttributes, deleteAttributes, validateAction from .domain_crawl import DomainCrawler from . import hsds_logger as log from . import config @@ -295,244 +295,6 @@ async def GET_Attribute(request): return resp -async def _getTypeFromRequest(app, body, obj_id=None, bucket=None): - """ return a type json from the request body """ - if "type" not in body: - msg = "PUT attribute with no type in body" - log.warn(msg) - raise HTTPBadRequest(reason=msg) - datatype = body["type"] - - if isinstance(datatype, str) and datatype.startswith("t-"): - # Committed type - fetch type json from DN - ctype_id = datatype - log.debug(f"got ctypeid: {ctype_id}") - ctype_json = await getObjectJson(app, ctype_id, bucket=bucket) - log.debug(f"ctype {ctype_id}: {ctype_json}") - root_id = getRootObjId(obj_id) - if ctype_json["root"] != root_id: - msg = "Referenced committed datatype must belong in same domain" - log.warn(msg) - raise HTTPBadRequest(reason=msg) - datatype = ctype_json["type"] - # add the ctype_id to the type - datatype["id"] = ctype_id - elif isinstance(datatype, str): - try: - # convert predefined type string (e.g. "H5T_STD_I32LE") to - # corresponding json representation - datatype = getBaseTypeJson(datatype) - except TypeError: - msg = "PUT attribute with invalid predefined type" - log.warn(msg) - raise HTTPBadRequest(reason=msg) - - try: - validateTypeItem(datatype) - except KeyError as ke: - msg = f"KeyError creating type: {ke}" - log.warn(msg) - raise HTTPBadRequest(reason=msg) - except TypeError as te: - msg = f"TypeError creating type: {te}" - log.warn(msg) - raise HTTPBadRequest(reason=msg) - except ValueError as ve: - msg = f"ValueError creating type: {ve}" - log.warn(msg) - raise HTTPBadRequest(reason=msg) - - return datatype - - -def _getShapeFromRequest(body): - """ get shape json from request body """ - shape_json = {} - if "shape" in body: - shape_body = body["shape"] - shape_class = None - if isinstance(shape_body, dict) and "class" in shape_body: - shape_class = shape_body["class"] - elif isinstance(shape_body, str): - shape_class = shape_body - if shape_class: - if shape_class == "H5S_NULL": - shape_json["class"] = "H5S_NULL" - if isinstance(shape_body, dict) and "dims" in shape_body: - msg = "can't include dims with null shape" - log.warn(msg) - raise HTTPBadRequest(reason=msg) - if isinstance(shape_body, dict) and "value" in body: - msg = "can't have H5S_NULL shape with value" - log.warn(msg) - raise HTTPBadRequest(reason=msg) - elif shape_class == "H5S_SCALAR": - shape_json["class"] = "H5S_SCALAR" - dims = getShapeDims(shape_body) - if len(dims) != 1 or dims[0] != 1: - msg = "dimensions aren't valid for scalar attribute" - log.warn(msg) - raise HTTPBadRequest(reason=msg) - elif shape_class == "H5S_SIMPLE": - shape_json["class"] = "H5S_SIMPLE" - dims = getShapeDims(shape_body) - shape_json["dims"] = dims - else: - msg = f"Unknown shape class: {shape_class}" - log.warn(msg) - raise HTTPBadRequest(reason=msg) - else: - # no class, interpet shape value as dimensions and - # use H5S_SIMPLE as class - if isinstance(shape_body, list) and len(shape_body) == 0: - shape_json["class"] = "H5S_SCALAR" - else: - shape_json["class"] = "H5S_SIMPLE" - dims = getShapeDims(shape_body) - shape_json["dims"] = dims - else: - shape_json["class"] = "H5S_SCALAR" - - return shape_json - - -def _getValueFromRequest(body, data_type, data_shape): - """ Get attribute value from request json """ - dims = getShapeDims(data_shape) - if "value" in body: - if dims is None: - msg = "Bad Request: data can not be included with H5S_NULL space" - log.warn(msg) - raise HTTPBadRequest(reason=msg) - value = body["value"] - # validate that the value agrees with type/shape - arr_dtype = createDataType(data_type) # np datatype - if len(dims) == 0: - np_dims = [1, ] - else: - np_dims = dims - - if body.get("encoding"): - item_size = getItemSize(data_type) - if item_size == "H5T_VARIABLE": - msg = "base64 encoding is not support for variable length attributes" - log.warn(msg) - raise HTTPBadRequest(reason=msg) - try: - data = decodeData(value) - except ValueError: - msg = "unable to decode data" - log.warn(msg) - raise HTTPBadRequest(reason=msg) - - expected_numbytes = arr_dtype.itemsize * np.prod(dims) - if len(data) != expected_numbytes: - msg = f"expected: {expected_numbytes} but got: {len(data)}" - log.warn(msg) - raise HTTPBadRequest(reason=msg) - - # check to see if this works with our shape and type - try: - arr = bytesToArray(data, arr_dtype, np_dims) - except ValueError as e: - log.debug(f"data: {data}") - log.debug(f"type: {arr_dtype}") - log.debug(f"np_dims: {np_dims}") - msg = f"Bad Request: encoded input data doesn't match shape and type: {e}" - log.warn(msg) - raise HTTPBadRequest(reason=msg) - - value_json = None - # now try converting to JSON - list_data = arr.tolist() - try: - value_json = bytesArrayToList(list_data) - except ValueError as err: - msg = f"Cannot decode bytes to list: {err}, will store as encoded bytes" - log.warn(msg) - if value_json: - log.debug("will store base64 input as json") - if data_shape["class"] == "H5S_SCALAR": - # just use the scalar value - value = value_json[0] - else: - value = value_json # return this - else: - value = data # return bytes to signal that this needs to be encoded - else: - # verify that the input data matches the array shape and type - try: - jsonToArray(np_dims, arr_dtype, value) - except ValueError as e: - msg = f"Bad Request: input data doesn't match selection: {e}" - log.warn(msg) - raise HTTPBadRequest(reason=msg) - else: - value = None - - return value - - -async def _getAttributeFromRequest(app, req_json, obj_id=None, bucket=None): - """ return attribute from given request json """ - attr_item = {} - attr_type = await _getTypeFromRequest(app, req_json, obj_id=obj_id, bucket=bucket) - attr_shape = _getShapeFromRequest(req_json) - attr_item = {"type": attr_type, "shape": attr_shape} - attr_value = _getValueFromRequest(req_json, attr_type, attr_shape) - if attr_value is not None: - if isinstance(attr_value, bytes): - attr_value = encodeData(attr_value) # store as base64 - attr_item["encoding"] = "base64" - else: - # just store the JSON dict or primitive value - attr_item["value"] = attr_value - else: - attr_item["value"] = None - - return attr_item - - -async def _getAttributesFromRequest(request, req_json, obj_id=None, bucket=None): - """ read the given JSON dictinary and return dict of attribute json """ - - app = request.app - attr_items = {} - kwargs = {"obj_id": obj_id} - if bucket: - kwargs["bucket"] = bucket - if "attributes" in req_json: - attributes = req_json["attributes"] - if not isinstance(attributes, dict): - msg = f"expected list for attributes but got: {type(attributes)}" - log.warn(msg) - raise HTTPBadRequest(reason=msg) - # read each attr_item and canonicalize the shape, type, verify value - for attr_name in attributes: - attr_json = attributes[attr_name] - attr_item = await _getAttributeFromRequest(app, attr_json, **kwargs) - attr_items[attr_name] = attr_item - - elif "type" in req_json: - # single attribute create - fake an item list - attr_item = await _getAttributeFromRequest(app, req_json, **kwargs) - if "name" in req_json: - attr_name = req_json["name"] - else: - attr_name = request.match_info.get("name") - validateAttributeName(attr_name) - if not attr_name: - msg = "Missing attribute name" - log.warn(msg) - raise HTTPBadRequest(reason=msg) - - attr_items[attr_name] = attr_item - else: - log.debug(f"_getAttributes from request - no attribute defined in {req_json}") - - return attr_items - - async def PUT_Attribute(request): """HTTP method to create a new attribute""" log.request(request) @@ -555,7 +317,7 @@ async def PUT_Attribute(request): log.debug(f"Attribute name: [{attr_name}]") validateAttributeName(attr_name) - log.info(f"PUT Attributes id: {req_obj_id} name: {attr_name}") + log.info(f"PUT Attribute id: {req_obj_id} name: {attr_name}") username, pswd = getUserPasswordFromRequest(request) # write actions need auth await validateUserPassword(app, username, pswd) @@ -587,7 +349,7 @@ async def PUT_Attribute(request): # get attribute from request body kwargs = {"bucket": bucket, "obj_id": req_obj_id} - attr_body = await _getAttributeFromRequest(app, body, **kwargs) + attr_body = await getAttributeFromRequest(app, body, **kwargs) # write attribute to DN attr_json = {attr_name: attr_body} @@ -596,7 +358,7 @@ async def PUT_Attribute(request): kwargs = {"bucket": bucket} if "replace" in params and params["replace"]: # allow attribute to be overwritten - log.debug("setting replace for PUT Atttribute") + log.debug("setting replace for PUT Attribute") kwargs["replace"] = True else: log.debug("replace is not set for PUT Attribute") @@ -624,7 +386,7 @@ async def PUT_Attributes(request): await validateUserPassword(app, username, pswd) if not request.has_body: - msg = "PUT Attribute with no body" + msg = "PUT Attributes with no body" log.warn(msg) raise HTTPBadRequest(reason=msg) try: @@ -654,10 +416,10 @@ async def PUT_Attributes(request): if not req_obj_id: req_obj_id = domain_json["root"] kwargs = {"obj_id": req_obj_id, "bucket": bucket} - attr_items = await _getAttributesFromRequest(request, body, **kwargs) + attr_items = await getAttributesFromRequest(app, body, **kwargs) if attr_items: - log.debug(f"PUT Attribute {len(attr_items)} attibutes to add") + log.debug(f"PUT Attribute {len(attr_items)} attributes to add") else: log.debug("no attributes defined yet") @@ -666,6 +428,7 @@ async def PUT_Attributes(request): obj_ids = {} if "obj_ids" in body: body_ids = body["obj_ids"] + if isinstance(body_ids, list): # multi cast the attributes - each attribute in attr-items # will be written to each of the objects identified by obj_id @@ -685,7 +448,7 @@ async def PUT_Attributes(request): msg += f"{len(obj_ids)} objects" log.info(msg) elif isinstance(body_ids, dict): - # each value is body_ids is a set of attriutes to write to the object + # each value is body_ids is a set of attributes to write to the object # unlike the above case, different attributes can be written to # different objects if attr_items: @@ -701,7 +464,7 @@ async def PUT_Attributes(request): id_json = body_ids[obj_id] kwargs = {"obj_id": obj_id, "bucket": bucket} - obj_items = await _getAttributesFromRequest(request, id_json, **kwargs) + obj_items = await getAttributesFromRequest(app, id_json, **kwargs) if obj_items: obj_ids[obj_id] = obj_items @@ -1056,7 +819,7 @@ async def PUT_AttributeValue(request): log.debug("PUT AttributeValue - request_type is binary") request_type = "binary" elif "application/json" in content_type: - log.debug("PUT AttribueValue - request type is json") + log.debug("PUT AttributeValue - request type is json") else: msg = f"Unknown content_type: {content_type}" log.warn(msg) @@ -1088,11 +851,7 @@ async def PUT_AttributeValue(request): msg += f"but got {len(binary_data)}" log.warn(msg) raise HTTPBadRequest(reason=msg) - arr = np.fromstring(binary_data, dtype=np_dtype) - if attr_shape["class"] == "H5S_SCALAR": - arr = arr.reshape([]) - else: - arr = arr.reshape(np_shape) # conform to selection shape + arr = bytesToArray(binary_data, np_dtype, np_shape) log.debug(f"got array {arr} from binary data") else: try: @@ -1137,6 +896,7 @@ async def PUT_AttributeValue(request): attr_body["value"] = data.decode("ascii") attr_body["encoding"] = "base64" attr_json = {attr_name: attr_body} + log.debug(f"putting attr {attr_name} to DN: {attr_json}") kwargs = {"bucket": bucket, "replace": True} diff --git a/hsds/basenode.py b/hsds/basenode.py index f3356f34..9d46b9c9 100644 --- a/hsds/basenode.py +++ b/hsds/basenode.py @@ -25,15 +25,17 @@ from aiohttp.web_exceptions import HTTPInternalServerError from aiohttp.web_exceptions import HTTPServiceUnavailable + from . import config from .util.httpUtil import http_get, http_post, jsonResponse -from .util.idUtil import createNodeId, getNodeNumber, getNodeCount from .util.authUtil import getUserPasswordFromRequest, validateUserPassword from .util.authUtil import isAdminUser from .util.k8sClient import getDnLabelSelector, getPodIps +from .util.nodeUtil import createNodeId, getNodeNumber, getNodeCount + from . import hsds_logger as log -HSDS_VERSION = "0.9.2" +HSDS_VERSION = "1.0.0" def getVersion(): @@ -236,15 +238,23 @@ async def docker_update_dn_info(app): log.error("HEAD node seems to be down.") app["dn_urls"] = [] app["dn_ids"] = [] + app["cluster_state"] = "WAITING" except OSError: log.error("failed to register") app["dn_urls"] = [] app["dn_ids"] = [] + app["cluster_state"] = "WAITING" if rsp_json is not None: log.debug(f"register response: {rsp_json}") app["dn_urls"] = rsp_json["dn_urls"] app["dn_ids"] = rsp_json["dn_ids"] + # the head node only reports "READY" once the *target* number of + # sn/dn nodes have registered - use it (rather than just "did this + # node see a non-empty, self-consistent dn_urls list") to decide + # readiness, so a node doesn't start routing/serving requests + # against a partial dn roster while the cluster is still scaling up + app["cluster_state"] = rsp_json.get("cluster_state", "WAITING") def get_dn_id_set(app): @@ -303,6 +313,15 @@ def updateReadyState(app, old_dn_urls=None): if len(dn_urls) > 0: log.warning(f"not all dn_ids found, got: {dn_ids}") is_ready = False + elif app.get("cluster_state") != "READY": + # the head node hasn't seen the full target number of sn/dn nodes + # register yet - don't flip this node to READY (and start + # routing/serving requests against a partial dn roster) until it + # has, otherwise nodes with different partial views of dn_urls can + # compute different partitions for the same obj_id (getObjPartition + # depends on dn count) and reject each other's requests + log.debug("updateReadyState - cluster_state from head is not READY yet") + is_ready = False if app["node_type"] == "dn": # dn node @@ -544,7 +563,7 @@ async def info(request): def baseInit(node_type): - """Intitialize application and return app object""" + """Initialize application and return app object""" # setup log config log_level = config.get("log_level") @@ -562,6 +581,7 @@ def baseInit(node_type): app = Application() app["node_state"] = "INITIALIZING" + app["cluster_state"] = "WAITING" app["node_number"] = -1 app["node_type"] = node_type app["start_time"] = int(time.time()) # seconds after epoch diff --git a/hsds/chunk_crawl.py b/hsds/chunk_crawl.py index 847f0933..1538425a 100755 --- a/hsds/chunk_crawl.py +++ b/hsds/chunk_crawl.py @@ -15,8 +15,6 @@ # import asyncio -import time -import traceback import random from asyncio import CancelledError import numpy as np @@ -24,16 +22,19 @@ from aiohttp.web_exceptions import HTTPInternalServerError from aiohttp.client_exceptions import ClientError +from h5json.hdf5dtype import createDataType +from h5json.array_util import jsonToArray, getNumpyValue +from h5json.array_util import getNumElements, arrayToBytes, bytesToArray +from h5json.shape_util import getShapeDims, getRank +from h5json.dset_util import getChunkDims +from h5json.time_util import getNow + +from .util.nodeUtil import getDataNodeUrl, getNodeCount from .util.httpUtil import http_get, http_put, http_post, get_http_client from .util.httpUtil import isUnixDomainUrl -from .util.idUtil import getDataNodeUrl, getNodeCount -from .util.hdf5dtype import createDataType -from .util.dsetUtil import getSliceQueryParam, getShapeDims -from .util.dsetUtil import getSelectionShape, getChunkLayout -from .util.chunkUtil import getChunkCoverage, getDataCoverage -from .util.chunkUtil import getChunkIdForPartition, getQueryDtype -from .util.arrayUtil import jsonToArray, getNumpyValue -from .util.arrayUtil import getNumElements, arrayToBytes, bytesToArray +from .util.dsetUtil import getSliceQueryParam +from .util.chunkUtil import getChunkCoverage, getDataCoverage, toNumpyIndex +from .util.chunkUtil import getChunkIdForPartition from . import config from . import hsds_logger as log @@ -83,10 +84,9 @@ async def write_chunk_hyperslab( msg = f"write_chunk_hyperslab, chunk_id: {chunk_id}, slices: {slices}, " msg += f"bucket: {bucket}" + msg += f" dset_json: {dset_json}" log.info(msg) - if "layout" not in dset_json: - log.error(f"No layout found in dset_json: {dset_json}") - raise HTTPInternalServerError() + partition_chunk_id = getChunkIdForPartition(chunk_id, dset_json) if partition_chunk_id != chunk_id: log.debug(f"using partition_chunk_id: {partition_chunk_id}") @@ -106,7 +106,7 @@ async def write_chunk_hyperslab( log.debug(f"setting fields_param to: {fields_param}") params["fields"] = fields_param - layout = getChunkLayout(dset_json) + layout = getChunkDims(dset_json) log.debug(f"getChunkCoverage({chunk_id}, {slices}, {layout})") chunk_sel = getChunkCoverage(chunk_id, slices, layout) if chunk_sel is None: @@ -122,8 +122,8 @@ async def write_chunk_hyperslab( if np.prod(arr.shape) != 1: do_broadcast = False else: - for s in slices: - if s.step is None: + for s in slices.slices: + if not isinstance(s, slice) or s.step is None: continue if s.step > 1: do_broadcast = False @@ -134,7 +134,7 @@ async def write_chunk_hyperslab( params["element_count"] = 1 arr_chunk = arr else: - arr_chunk = arr[data_sel] + arr_chunk = arr[toNumpyIndex(data_sel)] req = getDataNodeUrl(app, chunk_id) req += "/chunks/" + chunk_id @@ -163,6 +163,7 @@ async def read_chunk_hyperslab( select_dtype=None, query=None, query_update=None, + query_indices=False, limit=0, chunk_map=None, bucket=None, @@ -219,6 +220,7 @@ async def read_chunk_hyperslab( raise HTTPInternalServerError() type_json = dset_json["type"] dset_dt = createDataType(type_json) + dset_rank = getRank(dset_json) chunk_shape = None # expected return array shape chunk_sel = None # for hyperslab @@ -236,7 +238,14 @@ async def read_chunk_hyperslab( if "data_sel" in chunk_info: data_sel = chunk_info["data_sel"] log.debug(f"read_chunk_hyperslab - data_sel: {data_sel}") - chunk_shape = getSelectionShape(chunk_sel) + chunk_shape = chunk_sel.mshape + if np_arr is not None and len(np_arr.shape) > dset_rank: + # an array/subarray dtype's own dims are absorbed into + # np_arr's shape beyond the dataset's own logical rank (e.g. + # np_arr.shape == (2, 3) for a 2-element dataset of 3-int8 + # arrays) - append them so bytesToArray()/reshape() below + # expect the correct total shape, not just the logical one + chunk_shape = tuple(chunk_shape) + np_arr.shape[dset_rank:] log.debug(f"hyperslab selection - chunk_shape: {chunk_shape}") if "points" in chunk_info: @@ -254,8 +263,15 @@ async def read_chunk_hyperslab( if query is None and query_update is None: query_dtype = None + elif query_update is not None or query_indices: + # PUT_Chunk's query-update handling, and GET_Chunk's query_indices + # mode, both return the global dataset indices of matching + # elements, as (n, rank) coordinate tuples + query_dtype = np.dtype("i8") else: - query_dtype = getQueryDtype(select_dtype) + # GET_Chunk's query handling (h5json.query_util.arrayQuery) returns + # the matching values themselves, typed as select_dtype + query_dtype = select_dtype chunk_arr = None array_data = None @@ -269,7 +285,7 @@ async def read_chunk_hyperslab( if "s3offset" in chunk_info: s3offset = chunk_info["s3offset"] if isinstance(s3offset, list): - # convert to a colon seperated string + # convert to a colon separated string s3offset = ":".join(map(str, s3offset)) else: s3offset = int(s3offset) @@ -278,7 +294,7 @@ async def read_chunk_hyperslab( if "s3size" in chunk_info: s3size = chunk_info["s3size"] if isinstance(s3size, list): - # convert to a colon seperated string + # convert to a colon separated string s3size = ":".join(map(str, s3size)) else: s3size = int(s3size) @@ -287,7 +303,7 @@ async def read_chunk_hyperslab( if "hyper_dims" in chunk_info: hyper_dims = chunk_info["hyper_dims"] if isinstance(hyper_dims, list): - # convert to colon seperated string + # convert to colon separated string hyper_dims = ":".join(map(str, hyper_dims)) params["hyper_dims"] = hyper_dims @@ -304,6 +320,8 @@ async def read_chunk_hyperslab( params["query"] = query if limit > 0: params["Limit"] = limit + if query_indices: + params["query_indices"] = "1" # bucket will be used to get dset json even when s3path is used for # the chunk data @@ -378,9 +396,15 @@ async def read_chunk_hyperslab( log.debug(f"data: {len(array_data)} bytes") if query is not None or query_update is not None: # TBD: this needs to be fixed up for variable length dtypes - nrows = len(array_data) // query_dtype.itemsize + if query_update is not None or query_indices: + # indices are returned as (n, rank) + nrows = len(array_data) // (query_dtype.itemsize * dset_rank) + rsp_shape = (nrows, dset_rank) + else: + nrows = len(array_data) // query_dtype.itemsize + rsp_shape = (nrows,) try: - chunk_arr = bytesToArray(array_data, query_dtype, (nrows,)) + chunk_arr = bytesToArray(array_data, query_dtype, rsp_shape) except ValueError as ve: log.warn(f"bytesToArray ValueError: {ve}") raise HTTPBadRequest() @@ -417,7 +441,7 @@ async def read_chunk_hyperslab( np_arr[point_index] = chunk_arr else: # hyperslab selection - np_arr[data_sel] = chunk_arr + np_arr[toNumpyIndex(data_sel)] = chunk_arr log.debug(f"read_chunk_hyperslab {chunk_id} - done") @@ -572,18 +596,29 @@ async def write_point_sel( # create a numpy array with point_data # if point data was already decoded from binary, don't decode again - if len(point_data) > 0 and isinstance(point_data[0], np.ndarray): + if len(point_data) > 0 and isinstance(point_data[0], np.void) and len(point_data[0].dtype): + # a structured (compound) scalar - may be a fields-narrowed subset + # of dset_dtype (e.g. a "fields" selection), so use its own dtype + # rather than forcing it through the full dataset dtype data_arr = point_data + value_dtype = point_data[0].dtype + elif len(point_data) > 0 and isinstance(point_data[0], np.ndarray): + # already-decoded data (e.g. vlen sub-arrays) - don't decode again, + # but the "value" wire type is still the full dataset dtype (e.g. + # the object/vlen dtype), not the sub-array's own element dtype + data_arr = point_data + value_dtype = dset_dtype else: - data_arr = jsonToArray((num_points,), dset_dtype, point_data) + value_dtype = dset_dtype + data_arr = jsonToArray((num_points,), value_dtype, point_data) # create a numpy array with the following type: - # (coord1, coord2, ...) | dset_dtype + # (coord1, coord2, ...) | value_dtype if rank == 1: coord_type_str = "uint64" else: coord_type_str = f"({rank},)uint64" - type_fields = [("coord", np.dtype(coord_type_str)), ("value", dset_dtype)] + type_fields = [("coord", np.dtype(coord_type_str)), ("value", value_dtype)] comp_type = np.dtype(type_fields) np_arr = np.zeros((num_points,), dtype=comp_type) @@ -602,6 +637,11 @@ async def write_point_sel( params["action"] = "put" params["count"] = num_points params["bucket"] = bucket + if len(value_dtype) < len(dset_dtype): + # field selection, pass in the field names so the DN narrows too + fields_param = ":".join(value_dtype.names) + log.debug(f"setting fields_param to: {fields_param}") + params["fields"] = fields_param json_rsp = await http_post(app, req, params=params, data=post_data, client=client) log.debug(f"post to {req} returned {json_rsp}") @@ -623,6 +663,7 @@ def __init__( select_dtype=None, query=None, query_update=None, + query_indices=False, limit=0, points=None, action=None, @@ -646,6 +687,7 @@ def __init__( self._points = points self._query = query self._query_update = query_update + self._query_indices = query_indices self._hits = 0 self._limit = limit self._status_map = {} # map of chunk_ids to status code @@ -675,6 +717,9 @@ def __init__( app["cc_clients"] = {} self._clients = app["cc_clients"] + def now(self): + return getNow(app=self._app) + def get_status(self): if len(self._status_map) != len(self._chunk_ids): msg = "get_status code while crawler not complete" @@ -719,7 +764,7 @@ async def work(self): log.info(f"ChunkCrawler - client_name: {client_name}") while True: try: - start = time.time() + start = self.now() chunk_id = await self._q.get() if self._limit > 0 and self._hits >= self._limit: msg = f"ChunkCrawler - maxhits exceeded, skipping fetch for chunk: {chunk_id}" @@ -744,7 +789,7 @@ async def work(self): await self.do_work(chunk_id, client=client) self._q.task_done() - elapsed = time.time() - start + elapsed = self.now() - start msg = f"ChunkCrawler - task {chunk_id} start: {start:.3f} " msg += f"elapsed: {elapsed:.3f}" log.debug(msg) @@ -774,6 +819,7 @@ async def do_work(self, chunk_id, client=None): select_dtype=self._select_dtype, query=self._query, query_update=self._query_update, + query_indices=self._query_indices, limit=self._limit, chunk_map=self._chunk_map, bucket=self._bucket, @@ -890,8 +936,6 @@ async def do_work(self, chunk_id, client=None): status_code = 500 msg = f"Unexpected exception {type(e)} for {self._action}({chunk_id}): {e} " log.error(msg) - tb = traceback.format_exc() - print("traceback:", tb) retry += 1 if status_code == 200: break diff --git a/hsds/chunk_dn.py b/hsds/chunk_dn.py index e2671b61..e9f8b209 100644 --- a/hsds/chunk_dn.py +++ b/hsds/chunk_dn.py @@ -11,33 +11,49 @@ ############################################################################## # # value operations -# handles regauests to read/write chunk data +# handles requests to read/write chunk data # +import json import numpy as np import traceback from aiohttp.web_exceptions import HTTPBadRequest, HTTPInternalServerError from aiohttp.web_exceptions import HTTPNotFound, HTTPServiceUnavailable from aiohttp.web import json_response, StreamResponse +from h5json.hdf5dtype import createDataType, getSubType +from h5json.array_util import bytesToArray, arrayToBytes, getBroadcastShape +from h5json.objid import getS3Key, isValidUuid +from h5json.shape_util import getShapeDims +from h5json.dset_util import getChunkDims +from h5json.query_util import arrayQuery + from .util.httpUtil import request_read, getContentType -from .util.arrayUtil import bytesToArray, arrayToBytes, getBroadcastShape -from .util.idUtil import getS3Key, validateInPartition, isValidUuid from .util.storUtil import isStorObj, deleteStorObj -from .util.hdf5dtype import createDataType, getSubType -from .util.dsetUtil import getSelectionList, getChunkLayout, getShapeDims -from .util.dsetUtil import getSelectionShape, getChunkInitializer -from .util.chunkUtil import getChunkIndex, getDatasetId, chunkQuery +from .util.dsetUtil import getSelectionList, getChunkInitializer +from .util.dsetUtil import getSelect +from .util.chunkUtil import getChunkIndex, getDatasetId from .util.chunkUtil import chunkWriteSelection, chunkReadSelection from .util.chunkUtil import chunkWritePoints, chunkReadPoints from .util.domainUtil import isValidBucketName -from .util.boolparser import BooleanParser +from .util.nodeUtil import validateInPartition from .datanode_lib import get_metadata_obj, get_chunk, save_chunk from . import hsds_logger as log from . import config +def _localToGlobalIndices(chunk_id, chunk_dims, local_indices): + """ convert chunk-local query match indices (from arrayQuery) into + global dataset index coordinates, using the chunk's grid position + (encoded in its id) and its per-dimension extent (chunk_dims) """ + rank = local_indices.shape[1] + chunk_index = getChunkIndex(chunk_id) + offset_list = [chunk_index[i] * chunk_dims[i] for i in range(rank)] + offset = np.array(offset_list, dtype=local_indices.dtype) + return local_indices + offset + + async def PUT_Chunk(request): """ Update the requested chunk/selection @@ -46,24 +62,30 @@ async def PUT_Chunk(request): app = request.app params = request.rel_url.query query = None - query_update = None - limit = 0 bucket = None input_arr = None element_count = None + limit = 0 if "query" in params: query = params["query"] log.info(f"PUT_Chunk query: {query}") if "Limit" in params: - limit = int(params["Limit"]) + param_limit = params["Limit"] + try: + limit = int(param_limit) + except ValueError: + msg = f"invalid Limit param: {param_limit}" + log.warn(msg) + raise HTTPBadRequest(reason=msg) + log.debug(f"PUT_Chunk limit: {limit}") chunk_id = request.match_info.get("id") if not chunk_id: msg = "Missing chunk id" log.error(msg) raise HTTPBadRequest(reason=msg) - if not isValidUuid(chunk_id, "Chunk"): + if not isValidUuid(chunk_id, obj_class="chunks"): msg = f"Invalid chunk id: {chunk_id}" log.warn(msg) raise HTTPBadRequest(reason=msg) @@ -129,8 +151,7 @@ async def PUT_Chunk(request): dset_json = await get_metadata_obj(app, dset_id, bucket=bucket) # TBD - does this work with linked datasets? - dims = getChunkLayout(dset_json) - rank = len(dims) + dims = getChunkDims(dset_json) type_json = dset_json["type"] dset_dt = createDataType(type_json) @@ -147,19 +168,14 @@ async def PUT_Chunk(request): # TBD - cancel pending read if present? # get chunk selection from query params - if "select" in params: - select = params["select"] - log.debug(f"PUT_Chunk got select param: {select}") - else: - select = None # put for entire dataspace try: - selection = getSelectionList(select, dims) + selection = getSelect(params, dims) except ValueError as ve: - log.error(f"ValueError for select: {select}: {ve}") + log.error(f"ValueError for select: {params.get('select')}: {ve}") raise HTTPInternalServerError() log.debug(f"PUT_Chunk slices: {selection}") - mshape = getSelectionShape(selection) + mshape = selection.mshape if element_count is not None: bcshape = getBroadcastShape(mshape, element_count) log.debug(f"using bcshape: {bcshape}") @@ -190,60 +206,53 @@ async def PUT_Chunk(request): raise HTTPNotFound() if query: - if not dset_dt.fields: - log.error("expected compound dtype for PUT query") - raise HTTPInternalServerError() - if rank != 1: - log.error("expected one-dimensional array for PUT query") - raise HTTPInternalServerError() - try: - parser = BooleanParser(query) - except Exception as e: + indices = arrayQuery(query, chunk_arr, selection=selection, limit=limit) + except (TypeError, ValueError) as e: msg = f"query: {query} is not valid, got exception: {e}" - log.error(msg) - raise HTTPInternalServerError() - try: - eval_str = parser.getEvalStr() - except Exception as e: - msg = f"query: {query} unable to get eval str, got exception: {e}" - log.error(msg) - raise HTTPInternalServerError() - log.debug(f"got eval str: {eval_str} for query: {query}") + log.warn(msg) + raise HTTPBadRequest(reason=msg) + + log.debug(f"PUT_Chunk - query matched {len(indices)} elements") - query_update = await request.json() - if not query_update: - log.warn("PUT_Chunk with query but no query update") - raise HTTPBadRequest() - log.debug(f"query_update: {query_update}") - # TBD - send back binary response to SN node try: - kwargs = { - "chunk_id": chunk_id, - "chunk_layout": dims, - "chunk_arr": chunk_arr, - "slices": selection, - "query": eval_str, - "query_update": query_update, - "limit": limit, - } - rsp_arr = chunkQuery(**kwargs) - log.debug(f"query_update returned: {len(rsp_arr)} rows") - except TypeError as te: - log.warn(f"chunkQuery - TypeError: {te}") - raise HTTPBadRequest() - except ValueError as ve: - log.warn(f"chunkQuery - ValueError: {ve}") - raise HTTPBadRequest() - num_hits = rsp_arr.shape[0] - if num_hits > 0: + update_value = await request.json() + except json.JSONDecodeError: + msg = "Unable to load JSON body for query update" + log.warn(msg) + raise HTTPBadRequest(reason=msg) + + rank = len(chunk_arr.shape) + fancy_index = tuple(indices[:, i] for i in range(rank)) + + if len(indices) > 0: + # query_update is only allowed when the value is one element - + # that element gets broadcast across all matching positions + if select_dt.names: + # compound type - value is a JSON object of field name to + # value; only the given fields are updated, others are + # left as-is + if not isinstance(update_value, dict): + msg = "expected a JSON object for compound type query update" + log.warn(msg) + raise HTTPBadRequest(reason=msg) + for field_name, field_value in update_value.items(): + if field_name not in select_dt.names: + msg = f"field: {field_name} not found in dataset type" + log.warn(msg) + raise HTTPBadRequest(reason=msg) + chunk_arr[field_name][fancy_index] = field_value + else: + # simple type - value is the (scalar) element itself + if isinstance(update_value, dict) and "value" in update_value: + update_value = update_value["value"] + chunk_arr[fancy_index] = update_value is_dirty = True - # save chunk - save_chunk(app, chunk_id, dset_json, chunk_arr, bucket=bucket) - status_code = 201 - # stream back response array - read_resp = arrayToBytes(rsp_arr) + # return the global dataset indices of the matching elements + global_indices = _localToGlobalIndices(chunk_id, dims, indices) + + read_resp = arrayToBytes(global_indices) try: resp = StreamResponse() resp.headers["Content-Type"] = "application/octet-stream" @@ -255,7 +264,12 @@ async def PUT_Chunk(request): raise HTTPInternalServerError() finally: await resp.write_eof() - return + + if is_dirty or config.get("write_zero_chunks", default=False): + save_chunk(app, chunk_id, dset_json, chunk_arr, bucket=bucket) + + log.response(request, resp=resp) + return resp else: # regular chunk update # check that the content_length is what we expect @@ -289,16 +303,21 @@ async def PUT_Chunk(request): raise HTTPBadRequest(reason="unable to decode bytestring") + # bytesToArray() already absorbs an array/subarray dtype's own + # shape (select_dt.shape) into input_arr's shape via frombuffer, so + # it must be appended to bcshape/mshape too, or the reshape below + # would incorrectly try to drop those elements + subarray_shape = select_dt.shape if select_dt.shape else () if bcshape: - input_arr = input_arr.reshape(bcshape) + input_arr = input_arr.reshape(tuple(bcshape) + subarray_shape) log.debug(f"broadcasting {bcshape} to mshape {mshape}") arr_tmp = np.zeros(mshape, dtype=select_dt) arr_tmp[...] = input_arr input_arr = arr_tmp else: - input_arr = input_arr.reshape(mshape) + input_arr = input_arr.reshape(tuple(mshape) + subarray_shape) - kwargs = {"chunk_arr": chunk_arr, "slices": selection, "data": input_arr} + kwargs = {"chunk_arr": chunk_arr, "selection": selection, "data": input_arr} is_dirty = chunkWriteSelection(**kwargs) # chunk update successful @@ -328,6 +347,7 @@ async def GET_Chunk(request): dims = None query = None limit = 0 + query_indices = False app = request.app params = request.rel_url.query @@ -337,7 +357,7 @@ async def GET_Chunk(request): msg = "Missing chunk id" log.error(msg) raise HTTPBadRequest(reason=msg) - if not isValidUuid(chunk_id, "Chunk"): + if not isValidUuid(chunk_id, obj_class="chunks"): msg = f"Invalid chunk id: {chunk_id}" log.warn(msg) raise HTTPBadRequest(reason=msg) @@ -369,7 +389,7 @@ async def GET_Chunk(request): param_s3offset = params["s3offset"] try: if param_s3offset.find(":") > 0: - # colon seperated index values, convert to list + # colon separated index values, convert to list s3offset = list(map(int, param_s3offset.split(":"))) else: s3offset = int(param_s3offset) @@ -406,6 +426,10 @@ async def GET_Chunk(request): query = params["query"] log.debug(f"got query: {query}") + if "query_indices" in params and params["query_indices"]: + query_indices = True + log.debug("GET_Chunk - returning query match indices, not values") + if "Limit" in params: param_limit = params["Limit"] log.debug(f"limit: {limit}") @@ -431,25 +455,16 @@ async def GET_Chunk(request): dset_id = getDatasetId(chunk_id) dset_json = await get_metadata_obj(app, dset_id, bucket=bucket) - shape_dims = getShapeDims(dset_json["shape"]) + shape_dims = getShapeDims(dset_json) log.debug(f"shape_dims: {shape_dims}") - dims = getChunkLayout(dset_json) + dims = getChunkDims(dset_json) log.debug(f"GET_Chunk - got dims: {dims}") # get chunk selection from query params - if "select" in params: - select = params["select"] - else: - select = None # get slices for entire datashape - if select is not None: - log.debug(f"GET_Chunk - using select string: {select}") - else: - log.debug("GET_Chunk - no selection string") - try: - selection = getSelectionList(select, dims) + selection = getSelect(params, dims) except ValueError as ve: - log.error(f"ValueError for select: {select}: {ve}") + log.error(f"ValueError for select: {params.get('select')}: {ve}") raise HTTPInternalServerError() log.debug(f"GET_Chunk - got selection: {selection}") @@ -497,59 +512,37 @@ async def GET_Chunk(request): select_dt = chunk_arr.dtype if query: - # if there's a where clause, just use the expression - # part with BooleanParser - # TBD: Remove when BooleanParser knows how to use where keyword - if query.startswith("where"): - query_expr = None - else: - n = query.find(" where ") - if n > 0: - query_expr = query[:n] - else: - query_expr = query - if query_expr: - try: - parser = BooleanParser(query_expr) - except Exception as e: - msg = f"query: {query} is not valid, got exception: {e}" - log.error(msg) - raise HTTPInternalServerError() - try: - eval_str = parser.getEvalStr() - except Exception as e: - msg = f"query: {query} unable to get eval str, got exception: {e}" - log.error(msg) - raise HTTPInternalServerError() - log.debug(f"got eval str: {eval_str} for query: {query}") - - # run given query try: - kwargs = { - "chunk_id": chunk_id, - "chunk_layout": dims, - "chunk_arr": chunk_arr, - "slices": selection, - "query": query, - "limit": limit, - "select_dt": select_dt, - } - output_arr = chunkQuery(**kwargs) - except TypeError as te: - log.warn(f"chunkQuery - TypeError: {te}") - raise HTTPBadRequest() - except ValueError as ve: - log.warn(f"chunkQuery - ValueError: {ve}") - raise HTTPBadRequest() - if output_arr is None or output_arr.shape[0] == 0: - # no matches to query - msg = f"chunk {chunk_id} no results for query: {query}" - log.debug(msg) - raise HTTPNotFound() - log.debug(f"test - got output_arr: {output_arr}") + indices = arrayQuery(query, chunk_arr, selection=selection, limit=limit) + except (TypeError, ValueError) as e: + msg = f"query: {query} is not valid, got exception: {e}" + log.warn(msg) + raise HTTPBadRequest(reason=msg) + log.debug(f"GET_Chunk - query matched {len(indices)} elements") + + if query_indices: + # return the global dataset indices of the matching elements, + # rather than their values + output_arr = _localToGlobalIndices(chunk_id, dims, indices) + else: + # gather the matching values into a 1-d array + rank = len(chunk_arr.shape) + fancy_index = tuple(indices[:, i] for i in range(rank)) + output_arr = chunk_arr[fancy_index] + + if len(select_dt) < len(chunk_arr.dtype): + # do a field selection + arr = np.zeros(output_arr.shape, dtype=select_dt) + fields = select_dt.names + if len(fields) > 1: + for field in fields: + arr[field] = output_arr[field] + else: + arr[...] = output_arr[fields[0]] + output_arr = arr else: # read selected data from chunk - output_arr = chunkReadSelection(chunk_arr, slices=selection, select_dt=select_dt) + output_arr = chunkReadSelection(chunk_arr, selection=selection, select_dt=select_dt) # write response if output_arr is not None: @@ -653,7 +646,7 @@ async def POST_Chunk(request): chunk_index = getChunkIndex(chunk_id) log.debug(f"chunk_index: {chunk_index}") - if not isValidUuid(chunk_id, "Chunk"): + if not isValidUuid(chunk_id, obj_class="chunks"): msg = f"Invalid chunk id: {chunk_id}" log.warn(msg) raise HTTPBadRequest(reason=msg) @@ -680,7 +673,7 @@ async def POST_Chunk(request): dset_json = await get_metadata_obj(app, dset_id, bucket=bucket) log.debug(f"get_metadata_obj for {dset_id} returned {dset_json}") - dims = getChunkLayout(dset_json) + dims = getChunkDims(dset_json) rank = len(dims) type_json = dset_json["type"] @@ -789,7 +782,7 @@ async def POST_Chunk(request): raise HTTPInternalServerError() log.debug(f"GET_Chunk - got selection: {selection}") # read selected data from chunk - output_arr = chunkReadSelection(chunk_arr, slices=selection, select_dt=select_dt) + output_arr = chunkReadSelection(chunk_arr, selection=selection, select_dt=select_dt) else: # read points @@ -840,7 +833,7 @@ async def DELETE_Chunk(request): raise HTTPBadRequest(reason=msg) log.info(f"DELETE chunk: {chunk_id}") - if not isValidUuid(chunk_id, "Chunk"): + if not isValidUuid(chunk_id, obj_class="chunks"): msg = f"Invalid chunk id: {chunk_id}" log.warn(msg) raise HTTPBadRequest(reason=msg) diff --git a/hsds/chunk_sn.py b/hsds/chunk_sn.py index 68575007..82be77f4 100755 --- a/hsds/chunk_sn.py +++ b/hsds/chunk_sn.py @@ -19,29 +19,31 @@ import numpy as np from json import JSONDecodeError -from asyncio import IncompleteReadError from aiohttp.web_exceptions import HTTPException, HTTPBadRequest from aiohttp.web_exceptions import HTTPRequestEntityTooLarge from aiohttp.web_exceptions import HTTPConflict, HTTPInternalServerError from aiohttp.web import StreamResponse +from h5json.hdf5dtype import getItemSize, getDtypeItemSize, getSubType, createDataType +from h5json.array_util import bytesArrayToList, jsonToArray, getNumElements, arrayToBytes +from h5json.array_util import bytesToArray, squeezeArray, getBroadcastShape +from h5json.objid import isValidUuid +from h5json.shape_util import isNullSpace, isScalar, getShapeDims, getMaxDims, getRank +from h5json.dset_util import getChunkDims, isExtensible +from h5json import selections + from .util.httpUtil import getHref, getAcceptType, getContentType -from .util.httpUtil import request_read, jsonResponse, isAWSLambda -from .util.idUtil import isValidUuid +from .util.httpUtil import request_read, jsonResponse from .util.domainUtil import getDomainFromRequest, isValidDomain from .util.domainUtil import getBucketForDomain -from .util.hdf5dtype import getItemSize, getDtypeItemSize, getSubType, createDataType -from .util.dsetUtil import isNullSpace, isScalarSpace, get_slices, getShapeDims -from .util.dsetUtil import isExtensible, getSelectionPagination -from .util.dsetUtil import getSelectionShape, getDsetMaxDims, getChunkLayout -from .util.chunkUtil import getNumChunks, getChunkIds, getChunkId -from .util.arrayUtil import bytesArrayToList, jsonToArray -from .util.arrayUtil import getNumElements, arrayToBytes, bytesToArray -from .util.arrayUtil import squeezeArray, getBroadcastShape +from .util.dsetUtil import getSelectionPagination, get_slices +from .util.dsetUtil import isSelect, getSelectParam, getSelect +from .util.dsetUtil import parseRegionRefParam, extractJsonArrayElement +from .util.dsetUtil import regionRefSelectionToTargetSelection, unwrapSingleElement from .util.authUtil import getUserPasswordFromRequest, validateUserPassword -from .servicenode_lib import getDsetJson, validateAction -from .dset_lib import getSelectionData, getParser, extendShape -from .chunk_crawl import ChunkCrawler +from .servicenode_lib import getDsetJson, validateAction, getAttributes +from .dset_lib import getSelectionData, validateQuery, extendShape, doPointWrite, doHyperslabWrite +from .dset_lib import doQueryUpdate from . import config from . import hsds_logger as log @@ -56,7 +58,7 @@ def get_hrefs(request, dset_json): hrefs = [] dset_id = dset_json["id"] dset_uri = f"/datasets/{dset_id}" - self_uri = f"{dset_uri}/value" + self_uri = request.path # e.g. .../value or .../query, whichever was requested hrefs.append({"rel": "self", "href": getHref(request, self_uri)}) root_uri = "/groups/" + dset_json["root"] hrefs.append({"rel": "root", "href": getHref(request, root_uri)}) @@ -69,8 +71,6 @@ def use_http_streaming(request, rank): """ return boolean indicating whether http streaming should be used """ if rank == 0: return False - if isAWSLambda(request): - return False if not config.get("http_streaming", default=True): return False return True @@ -149,25 +149,23 @@ def _getAppendRows(params, dset_json, body=None): if append_rows: log.info(f"append_rows: {append_rows}") - datashape = dset_json["shape"] - dims = getShapeDims(datashape) + dims = getShapeDims(dset_json) rank = len(dims) if rank == 0: msg = "append can't be used in scalar or null space datasets" log.warn(msg) raise HTTPBadRequest(reason=msg) # select can't be used with append - if _isSelect(params, body=body): + if isSelect(params, body=body): msg = "select query parameter can not be used with append" log.warn(msg) raise HTTPBadRequest(reason=msg) # shape must be extensible - datashape = dset_json["shape"] - dims = getShapeDims(datashape) - rank = len(dims) - maxdims = getDsetMaxDims(dset_json) - if not isExtensible(dims, maxdims): + dims = getShapeDims(dset_json) + rank = getRank(dset_json) + maxdims = getMaxDims(dset_json) + if not isExtensible(dset_json): msg = "Dataset shape must be extensible for packet updates" log.warn(msg) raise HTTPBadRequest(reason=msg) @@ -177,7 +175,7 @@ def _getAppendRows(params, dset_json, body=None): log.warn(msg) raise HTTPBadRequest(reason=msg) - if maxdims[append_dim] != 0: + if maxdims[append_dim] not in (0, "H5S_UNLIMITED"): if dims[append_dim] + append_rows > maxdims[append_dim]: log.warn("unable to append to dataspace") raise HTTPConflict() @@ -185,58 +183,114 @@ def _getAppendRows(params, dset_json, body=None): return append_rows -def _isSelect(params, body=None): - """ return True if select param or select is set in request body - """ - if "select" in params and params["select"]: - return True - - if isinstance(body, dict): - if "select" in body and body["select"]: - return True - for key in ("start", "stop", "step"): - if key in body and body[key]: - return True - return False - - def _getSelect(params, dset_json, body=None): - """ return selection region if any as a list - of slices. """ - slices = None - log.debug(f"_getSelect params: {params} body: {body}") + """ return the requested selection region, if any, as a + selections.Selection. """ + if _isAppend(params, body=body) and isSelect(params, body=body): + msg = "append can't be used with selection" + log.warn(msg) + raise HTTPBadRequest(reason=msg) + + log.debug(f"_getSelect params: {dict(params)} body: {body}") try: - if body and isinstance(body, dict): - if "select" in body and body["select"]: - select = body.get("select") - slices = get_slices(select, dset_json) - elif "start" in body and "stop" in body: - slices = get_slices(body, dset_json) - if "select" in params and params["select"]: - select = params.get("select") - if slices: - msg = "select defined in both request body and query parameters" - raise ValueError(msg) - slices = get_slices(select, dset_json) + select = getSelectParam(params, body=body) + selection = get_slices(select, dset_json) except ValueError as ve: log.warn(f"Invalid selection: {ve}") raise HTTPBadRequest(reason="Invalid selection") - if _isAppend(params, body=body) and slices: - msg = "append can't be used with selection" + log.debug(f"_getSelect returning: {selection}") + return selection + + +def _validateRegionRefType(type_json): + """ raise HTTPBadRequest unless type_json is exactly a region reference + type (H5T_REFERENCE / H5T_STD_REF_DSETREG) - vlen/compound-wrapped + region references are not supported as a regionref source. """ + msg = "regionref path does not refer to a H5T_STD_REF_DSETREG value" + if not isinstance(type_json, dict): + log.warn(msg) + raise HTTPBadRequest(reason=msg) + if type_json.get("class") != "H5T_REFERENCE": + log.warn(msg) + raise HTTPBadRequest(reason=msg) + if type_json.get("base") != "H5T_STD_REF_DSETREG": log.warn(msg) raise HTTPBadRequest(reason=msg) - if not slices: - # just return the entire dataspace - datashape = dset_json["shape"] - dims = getShapeDims(datashape) - slices = [] - for dim in dims: - s = slice(0, dim, 1) - slices.append(s) - log.debug(f"_getSelect returning: {slices}") - return slices + +async def _resolveRegionRef(app, domain, username, bucket, params, target_dims): + """ Resolve the 'regionref' query param to a selections.Selection over + target_dims (the URL's dataset - never the id embedded in the region + reference itself). Returns None if the resolved region reference is + null/unbound (caller should respond with 204 No Content in that case). + """ + collection, obj_id, attr_name = parseRegionRefParam(params.get("regionref")) + + # verify the source object belongs to this domain and is readable + await validateAction(app, domain, obj_id, username, "read") + + ref_dset_json = None + if attr_name is not None: + attributes = await getAttributes( + app, obj_id, attr_names=[attr_name], bucket=bucket, include_data=True + ) + if not attributes: + msg = f"regionref attribute not found: {attr_name}" + log.warn(msg) + raise HTTPBadRequest(reason=msg) + attr_json = attributes[0] + src_type = attr_json["type"] + src_shape = attr_json["shape"] + src_value = attr_json["value"] + else: + ref_dset_json = await getDsetJson(app, obj_id, bucket=bucket) + src_type = ref_dset_json["type"] + src_shape = ref_dset_json["shape"] + src_value = None + + _validateRegionRefType(src_type) + + if src_shape.get("class") == "H5S_NULL": + msg = "regionref source can not be a null-space object" + log.warn(msg) + raise HTTPBadRequest(reason=msg) + + src_dims = getShapeDims(src_shape) + + if not src_dims: + # scalar source - a select param, if given, isn't meaningful; ignore it + if attr_name is not None: + single_json = src_value + else: + scalar_sel = get_slices(None, ref_dset_json) + kwargs = {"slices": scalar_sel, "bucket": bucket} + sub_arr = await getSelectionData(app, obj_id, ref_dset_json, **kwargs) + single_json = unwrapSingleElement(bytesArrayToList(sub_arr)) + else: + try: + sel = getSelect(params, src_dims) + except ValueError as ve: + msg = f"Invalid regionref select: {ve}" + log.warn(msg) + raise HTTPBadRequest(reason=msg) + if sel.nselect != 1: + msg = "regionref selection must resolve to exactly one element, " + msg += f"got {sel.nselect}" + log.warn(msg) + raise HTTPBadRequest(reason=msg) + + if attr_name is not None: + arr_dtype = createDataType(src_type) + single_json = extractJsonArrayElement(tuple(src_dims), arr_dtype, src_value, sel) + else: + sub_arr = await getSelectionData(app, obj_id, ref_dset_json, slices=sel, bucket=bucket) + single_json = unwrapSingleElement(bytesArrayToList(sub_arr)) + + if single_json is None: + return None + + return regionRefSelectionToTargetSelection(single_json, target_dims) def _getSelectDtype(params, dset_dtype, body=None): @@ -316,7 +370,7 @@ def _getPoints(body, rank=1): return points -def _getQuery(params, dtype, rank=1, body=None): +def _getQuery(params, dtype, body=None): """ get query parameter and validate if set """ kw = "query" @@ -332,18 +386,9 @@ def _getQuery(params, dtype, rank=1, body=None): msg = "Query string can not be used with append parameter" log.warn(msg) raise HTTPBadRequest(reason=msg) - # validate the query string - if rank > 1: - msg = "Query string is not supported for multidimensional datasets" - log.warn(msg) - raise HTTPBadRequest(reason=msg) - - if len(dtype) == 0: - msg = "Query string is not supported for primitive type datasets" - log.warn(msg) # following will throw HTTPBadRequest if query is malformed - getParser(query, dtype) + validateQuery(query, dtype) return query @@ -385,7 +430,7 @@ async def _getRequestData(request, http_streaming=True): log.debug(f"getRequestData - got json: {body}") if "value" in body: input_data = body["value"] - log.debug("input_data: {input_data}") + log.debug(f"input_data: {input_data}") elif "value_base64" in body: base64_data = body["value_base64"] base64_data = base64_data.encode("ascii") @@ -426,12 +471,13 @@ async def _getRequestData(request, http_streaming=True): async def arrayResponse(arr, request, dset_json): - """ return the array as binary or json response based on accept type """ + """ return query match indices (from GET_Query, or PUT_Value's + query-update mode) as a binary or json response based on accept type """ response_type = getAcceptType(request) if response_type == "binary": output_data = arr.tobytes() - msg = f"PUT_Value query - returning {len(output_data)} bytes binary data" + msg = f"arrayResponse - returning {len(output_data)} bytes binary indices" log.debug(msg) # write response @@ -448,204 +494,22 @@ async def arrayResponse(arr, request, dset_json): except Exception as e: log.error(f"Exception during binary data write: {e}") else: - log.debug("PUT Value query - returning JSON data") + log.debug("arrayResponse - returning JSON indices") rsp_json = {} data = arr.tolist() - log.debug(f"got rsp data {len(data)} points") + log.debug(f"got rsp data {len(data)} indices") try: - json_query_data = bytesArrayToList(data) + indices_data = bytesArrayToList(data) except ValueError as err: msg = f"Cannot decode provided bytes to list: {err}" raise HTTPBadRequest(reason=msg) - rsp_json["value"] = json_query_data + rsp_json["indices"] = indices_data rsp_json["hrefs"] = get_hrefs(request, dset_json) resp = await jsonResponse(request, rsp_json) return resp -async def _doPointWrite(app, - request, - points=None, - data=None, - dset_json=None, - bucket=None - ): - """ write the given points to the dataset """ - - num_points = len(points) - log.debug(f"doPointWrite - num_points: {num_points}") - dset_id = dset_json["id"] - layout = getChunkLayout(dset_json) - datashape = dset_json["shape"] - dims = getShapeDims(datashape) - rank = len(dims) - - chunk_dict = {} # chunk ids to list of points in chunk - - for pt_indx in range(num_points): - if rank == 1: - point = int(points[pt_indx]) - else: - point_tuple = points[pt_indx] - point = [] - for i in range(len(point_tuple)): - point.append(int(point_tuple[i])) - if rank == 1: - if point < 0 or point >= dims[0]: - msg = f"PUT Value point: {point} is not within the " - msg += "bounds of the dataset" - log.warn(msg) - raise HTTPBadRequest(reason=msg) - else: - if len(point) != rank: - msg = "PUT Value point value did not match dataset rank" - log.warn(msg) - raise HTTPBadRequest(reason=msg) - for i in range(rank): - if point[i] < 0 or point[i] >= dims[i]: - msg = f"PUT Value point: {point} is not within the " - msg += "bounds of the dataset" - log.warn(msg) - raise HTTPBadRequest(reason=msg) - chunk_id = getChunkId(dset_id, point, layout) - # get the pt_indx element from the input data - value = data[pt_indx] - if chunk_id not in chunk_dict: - point_list = [point, ] - point_data = [value, ] - chunk_dict[chunk_id] = {"indices": point_list, "points": point_data} - else: - item = chunk_dict[chunk_id] - point_list = item["indices"] - point_list.append(point) - point_data = item["points"] - point_data.append(value) - - num_chunks = len(chunk_dict) - log.debug(f"num_chunks: {num_chunks}") - max_chunks = int(config.get("max_chunks_per_request", default=1000)) - if num_chunks > max_chunks: - msg = f"PUT value request with more than {max_chunks} chunks" - log.warn(msg) - - chunk_ids = list(chunk_dict.keys()) - chunk_ids.sort() - - crawler = ChunkCrawler( - app, - chunk_ids, - dset_json=dset_json, - bucket=bucket, - points=chunk_dict, - action="write_point_sel", - ) - await crawler.crawl() - - crawler_status = crawler.get_status() - - if crawler_status not in (200, 201): - msg = f"doPointWritte raising HTTPInternalServerError for status: {crawler_status}" - log.error(msg) - raise HTTPInternalServerError() - else: - log.info("doPointWrite success") - - -async def _doHyperslabWrite(app, - request, - page_number=0, - page=None, - data=None, - dset_json=None, - select_dtype=None, - bucket=None - ): - """ write the given page selection to the dataset """ - dset_id = dset_json["id"] - log.info(f"_doHyperslabWrite on {dset_id} - page: {page_number}") - type_json = dset_json["type"] - - if select_dtype is not None: - item_size = getDtypeItemSize(select_dtype) - else: - item_size = getItemSize(type_json) - if item_size == "H5T_VARIABLE" and data is None: - msg = "unexpected call to _doHyperslabWrite for variable length data" - log.error(msg) - raise HTTPInternalServerError() - - layout = getChunkLayout(dset_json) - - num_chunks = getNumChunks(page, layout) - log.debug(f"num_chunks: {num_chunks}") - max_chunks = int(config.get("max_chunks_per_request", default=1000)) - if num_chunks > max_chunks: - msg = f"PUT value chunk count: {num_chunks} exceeds max_chunks: {max_chunks}" - log.warn(msg) - select_shape = getSelectionShape(page) - log.debug(f"got select_shape: {select_shape} for page: {page_number}") - - if data is None: - num_bytes = math.prod(select_shape) * item_size - log.debug(f"reading {num_bytes} from request stream") - # read page of data from input stream - try: - page_bytes = await request_read(request, count=num_bytes) - except HTTPRequestEntityTooLarge as tle: - msg = "Got HTTPRequestEntityTooLarge exception during " - msg += f"binary read: {tle}) for page: {page_number}" - log.warn(msg) - raise # re-throw - except IncompleteReadError as ire: - msg = "Got asyncio.IncompleteReadError during binary " - msg += f"read: {ire} for page: {page_number}" - log.warn(msg) - raise HTTPBadRequest(reason=msg) - log.debug(f"read {len(page_bytes)} for page: {page_number}") - try: - arr = bytesToArray(page_bytes, select_dtype, select_shape) - except ValueError as ve: - msg = f"bytesToArray value error for page: {page_number}: {ve}" - log.warn(msg) - raise HTTPBadRequest(reason=msg) - else: - arr = data # use array provided to function - - try: - chunk_ids = getChunkIds(dset_id, page, layout) - except ValueError: - log.warn("getChunkIds failed") - raise HTTPInternalServerError() - if len(chunk_ids) < 10: - log.debug(f"chunk_ids: {chunk_ids}") - else: - log.debug(f"chunk_ids: {chunk_ids[:10]} ...") - if len(chunk_ids) > max_chunks: - msg = f"got {len(chunk_ids)} for page: {page_number}. max_chunks: {max_chunks}" - log.warn(msg) - - crawler = ChunkCrawler( - app, - chunk_ids, - dset_json=dset_json, - bucket=bucket, - slices=page, - arr=arr, - action="write_chunk_hyperslab", - ) - await crawler.crawl() - - crawler_status = crawler.get_status() - - if crawler_status not in (200, 201): - msg = f"crawler failed for page: {page_number} with status: {crawler_status}" - log.error(msg) - raise HTTPInternalServerError() - else: - log.info("crawler write_chunk_hyperslab successful") - - async def PUT_Value(request): """ Handler for PUT //value request @@ -682,7 +546,7 @@ async def PUT_Value(request): msg = "Missing dataset id" log.warn(msg) raise HTTPBadRequest(reason=msg) - if not isValidUuid(dset_id, "Dataset"): + if not isValidUuid(dset_id, obj_class="datasets"): msg = f"Invalid dataset id: {dset_id}" log.warn(msg) raise HTTPBadRequest(reason=msg) @@ -700,15 +564,15 @@ async def PUT_Value(request): # get state for dataset from DN - will need this to validate # some of the query parameters dset_json = await getDsetJson(app, dset_id, bucket=bucket) + log.debug(f"got dset_json: {dset_json}") - datashape = dset_json["shape"] if isNullSpace(dset_json): msg = "Null space datasets can not be used as target for PUT value" log.warn(msg) raise HTTPBadRequest(reason=msg) - dims = getShapeDims(datashape) - rank = len(dims) + dims = getShapeDims(dset_json) + rank = getRank(dset_json) type_json = dset_json["type"] dset_dtype = createDataType(type_json) @@ -741,9 +605,41 @@ async def PUT_Value(request): raise HTTPBadRequest(reason=msg) # if there's no selection parameter, this will return entire dataspace - slices = _getSelect(params, dset_json, body=body) + if params.get("regionref"): + if append_rows: + msg = "regionref cannot be combined with append" + log.warn(msg) + raise HTTPBadRequest(reason=msg) + if points is not None: + msg = "regionref cannot be combined with points" + log.warn(msg) + raise HTTPBadRequest(reason=msg) + if params.get("query") or (isinstance(body, dict) and body.get("query")): + msg = "regionref cannot be combined with query" + log.warn(msg) + raise HTTPBadRequest(reason=msg) + selection = await _resolveRegionRef(app, domain, username, bucket, params, dims) + if selection is None: + msg = "regionref resolved to a null region reference" + log.warn(msg) + raise HTTPBadRequest(reason=msg) + if selection.select_type == selections.H5S_SEL_POINTS: + # a points-type region ref can't go through the hyperslab write + # path (chunkWriteSelection expects real slices) - convert to + # the same points array _getPoints() would build from a body + coords = list(zip(*selection.slices)) + if rank == 1: + points = np.array([c[0] for c in coords], dtype=np.uint64) + else: + points = np.array(coords, dtype=np.uint64) + elif selection.select_type not in (selections.H5S_SEL_HYPERSLABS, selections.H5S_SEL_ALL): + msg = "regionref selection type is not supported for PUT Value" + log.warn(msg) + raise HTTPBadRequest(reason=msg) + else: + selection = _getSelect(params, dset_json, body=body) - query = _getQuery(params, dset_dtype, rank=rank, body=body) + query = _getQuery(params, dset_dtype, body=body) element_count = _getElementCount(params, body=body) @@ -758,11 +654,11 @@ async def PUT_Value(request): log.debug(f"got query: {query}") limit = _getLimit(params, body=body) - arr_rsp = await getSelectionData( + arr_rsp = await doQueryUpdate( app, dset_id, dset_json, - slices=slices, + slices=selection, query=query, bucket=bucket, limit=limit, @@ -792,14 +688,23 @@ async def PUT_Value(request): log.warn(msg) raise HTTPBadRequest(reason=msg) - slices = await extendShape(app, dset_json, append_rows, axis=append_dim, bucket=bucket) - np_shape = getSelectionShape(slices) + selection = await extendShape(app, dset_json, append_rows, axis=append_dim, bucket=bucket) + # extend dims based on slices that exceed shape + dims = list(dims) + for i in range(rank): + s = selection.slices[i] + if s.stop > dims[i]: + dims[i] = s.stop + log.debug(f"updated dims: {dims}") + dims = tuple(dims) + selection = selections.select(dims, selection.slices) + np_shape = selection.mshape log.debug(f"np_shape based on append_rows: {np_shape}") elif points is None: # The selection parameters will determine expected put value shape - log.debug(f"PUT Value selection: {slices}") + log.debug(f"PUT Value selection: {selection}") # not point selection, get hyperslab selection shape - np_shape = getSelectionShape(slices) + np_shape = selection.mshape else: # point update np_shape = [len(points),] @@ -841,19 +746,20 @@ async def PUT_Value(request): log.warn(f"bytesToArray value error: {ve}") raise HTTPBadRequest() else: - # fixed item size - if len(input_data) % item_size != 0: - msg = f"Expected request size to be a multiple of {item_size}, " + # fixed item size - check against number of bytes + # (use select_item_size/select_dtype, not the full dataset's + # item_size/dset_dtype, so a "fields" selection is accounted for) + if len(input_data) % select_item_size != 0: + msg = f"Expected request size to be a multiple of {select_item_size}, " msg += f"but {len(input_data)} bytes received" log.warn(msg) raise HTTPBadRequest(reason=msg) - if len(input_data) // item_size != num_elements: - msg = f"expected {item_size * num_elements} bytes but got {len(input_data)}" + if len(input_data) // select_item_size != num_elements: + msg = f"expected {select_item_size * num_elements} bytes but got {len(input_data)}" log.warn(msg) raise HTTPBadRequest(reason=msg) - - arr = np.fromstring(input_data, dtype=dset_dtype) + arr = np.frombuffer(input_data, dtype=select_dtype) log.debug(f"read fixed type array: {arr}") if bc_shape: @@ -869,11 +775,19 @@ async def PUT_Value(request): arr = arr_tmp if element_count != 1: + # np.frombuffer() above already absorbed an array/subarray + # dtype's own shape (select_dtype.shape) into arr's shape, so + # it must be appended to np_shape too, or this reshape would + # incorrectly try to drop those elements + if not select_dtype.shape: + target_shape = np_shape + else: + target_shape = tuple(np_shape) + select_dtype.shape try: - arr = arr.reshape(np_shape) # conform to selection shape + arr = arr.reshape(target_shape) # conform to selection shape except ValueError: msg = "Bad Request: binary input data doesn't match selection " - msg += f"reshaping {arr.shape} to {np_shape}" + msg += f"reshaping {arr.shape} to {target_shape}" log.warn(msg) raise HTTPBadRequest(reason=msg) @@ -913,17 +827,16 @@ async def PUT_Value(request): else: log.debug("will use streaming for request data") - slices = tuple(slices) # no more edits to slices if points is None: # do a hyperslab write if arr is not None: # make a one page list to handle the write in one chunk crawler run # (larger write request should user binary streaming) - pages = (slices,) - log.debug(f"non-streaming data, setting page list to: {slices}") + pages = (selection,) + log.debug(f"non-streaming data, setting page list to: {selection}") else: max_request_size = int(config.get("max_request_size")) - pages = getSelectionPagination(slices, dims, select_item_size, max_request_size) + pages = getSelectionPagination(selection, dims, select_item_size, max_request_size) log.debug(f"getSelectionPagination returned: {len(pages)} pages") for page_number in range(len(pages)): @@ -940,13 +853,13 @@ async def PUT_Value(request): else: kwargs["data"] = None # do write for one page selection - await _doHyperslabWrite(app, request, **kwargs) + await doHyperslabWrite(app, request, **kwargs) else: # # Do point put # kwargs = {"points": points, "data": arr, "dset_json": dset_json, "bucket": bucket} - await _doPointWrite(app, request, **kwargs) + await doPointWrite(app, request, **kwargs) # write successful @@ -968,7 +881,7 @@ async def GET_Value(request): msg = "Missing dataset id" log.warn(msg) raise HTTPBadRequest(reason=msg) - if not isValidUuid(dset_id, "Dataset"): + if not isValidUuid(dset_id, obj_class="datasets"): msg = f"Invalid dataset id: {dset_id}" log.warn(msg) raise HTTPBadRequest(reason=msg) @@ -999,23 +912,32 @@ async def GET_Value(request): log.warn(msg) raise HTTPBadRequest(reason=msg) - datashape = dset_json["shape"] - dims = getShapeDims(datashape) + dims = getShapeDims(dset_json) log.debug(f"dset shape: {dims}") - rank = len(dims) + rank = getRank(dset_json) - layout = getChunkLayout(dset_json) + layout = getChunkDims(dset_json) log.debug(f"chunk layout: {layout}") await validateAction(app, domain, dset_id, username, "read") # Get query parameter for selection - slices = _getSelect(params, dset_json) + if params.get("regionref"): + if params.get("query"): + msg = "regionref cannot be combined with query" + log.warn(msg) + raise HTTPBadRequest(reason=msg) + selection = await _resolveRegionRef(app, domain, username, bucket, params, dims) + if selection is None: + # null/unbound region reference + return await jsonResponse(request, {}, status=204) + else: + selection = _getSelect(params, dset_json) # dtype for selection, or just dset_dtype if no fields are given select_dtype = _getSelectDtype(params, dset_dtype) - log.debug(f"GET Value selection: {slices}") + log.debug(f"GET Value selection: {selection}") if len(dset_dtype) < 10: log.debug(f"dset_dtype: {dset_dtype}, select_dtype: {select_dtype}") @@ -1027,7 +949,7 @@ async def GET_Value(request): ignore_nan = False log.debug(f"ignore nan: {ignore_nan}") - query = _getQuery(params, dset_dtype, rank=rank) + query = _getQuery(params, dset_dtype) response_type = getAcceptType(request) @@ -1044,7 +966,7 @@ async def GET_Value(request): log.debug(f"item size based on dtype: {item_size}") # get the shape of the response array - np_shape = getSelectionShape(slices) + np_shape = selection.mshape log.debug(f"selection shape: {np_shape}") # check that the array size is reasonable @@ -1055,10 +977,6 @@ async def GET_Value(request): request_size *= item_size log.debug(f"request_size: {request_size}") max_request_size = int(config.get("max_request_size")) - if isAWSLambda(request): - # reduce max size to account for hex_encoding and other JSON content - max_request_size -= 1000 - max_request_size /= 2 if request_size >= max_request_size and not stream_pagination: msg = "GET value request too large" log.warn(msg) @@ -1089,13 +1007,12 @@ async def GET_Value(request): arr = None # will be set based on returned data if stream_pagination: - # example # get binary data a page at a time and write back to response if item_size == "H5T_VARIABLE": page_item_size = VARIABLE_AVG_ITEM_SIZE # random guess of avg item_size else: page_item_size = item_size - pages = getSelectionPagination(slices, dims, page_item_size, max_request_size) + pages = getSelectionPagination(selection, dims, page_item_size, max_request_size) log.debug(f"getSelectionPagination returned: {len(pages)} pages") bytes_streamed = 0 try: @@ -1163,7 +1080,7 @@ async def GET_Value(request): app, dset_id, dset_json, - slices=slices, + slices=selection, select_dtype=select_dtype, query=query, bucket=bucket, @@ -1185,8 +1102,6 @@ async def GET_Value(request): elif response_type == "binary": if resp_json["status"] != 200: # write json with status_code - # resp_json = resp_json.encode('utf-8') - # await resp.write(resp_json) log.warn(f"GET Value - got error status: {resp_json['status']}") else: log.debug("preparing binary response") @@ -1201,15 +1116,13 @@ async def GET_Value(request): if "reduce_dim" in params and params["reduce_dim"]: arr = squeezeArray(arr) - data = arr.tolist() try: - json_data = bytesArrayToList(data) + json_data = bytesArrayToList(arr) except ValueError as err: msg = f"Cannot decode bytes to list: {err}" raise HTTPBadRequest(reason=msg) - datashape = dset_json["shape"] - if datashape["class"] == "H5S_SCALAR": + if isScalar(dset_json): # convert array response to value resp_json["value"] = json_data[0] else: @@ -1224,15 +1137,98 @@ async def GET_Value(request): await resp.write_eof() except Exception as e: log.error(f"{type(e)} Exception during data write: {e}") - import traceback - - tb = traceback.format_exc() - print("traceback:", tb) raise HTTPInternalServerError() return resp +async def GET_Query(request): + """ + Handler for GET //query request - like GET_Value with a + query param, except the response is the global dataset indices of the + elements that match the query, not their values (query is required). + """ + log.request(request) + app = request.app + params = request.rel_url.query + + dset_id = request.match_info.get("id") + if not dset_id: + msg = "Missing dataset id" + log.warn(msg) + raise HTTPBadRequest(reason=msg) + if not isValidUuid(dset_id, obj_class="datasets"): + msg = f"Invalid dataset id: {dset_id}" + log.warn(msg) + raise HTTPBadRequest(reason=msg) + + username, pswd = getUserPasswordFromRequest(request) + if username is None and app["allow_noauth"]: + username = "default" + else: + await validateUserPassword(app, username, pswd) + + domain = getDomainFromRequest(request) + if not isValidDomain(domain): + msg = f"Invalid domain: {domain}" + log.warn(msg) + raise HTTPBadRequest(reason=msg) + bucket = getBucketForDomain(domain) + + dset_json = await getDsetJson(app, dset_id, bucket=bucket) + type_json = dset_json["type"] + dset_dtype = createDataType(type_json) + + if isNullSpace(dset_json): + msg = "Null space datasets can not be used as target for GET query" + log.warn(msg) + raise HTTPBadRequest(reason=msg) + + await validateAction(app, domain, dset_id, username, "read") + + # Get query parameter for selection + selection = _getSelect(params, dset_json) + + query = _getQuery(params, dset_dtype) + if not query: + msg = "query parameter is required for GET_Query" + log.warn(msg) + raise HTTPBadRequest(reason=msg) + + limit = _getLimit(params) + + # check that the (unfiltered) select region size is reasonable - same + # caveat as GET_Value's query mode: based on the select region, not + # the (typically much smaller) query-matched result + item_size = getDtypeItemSize(dset_dtype) + np_shape = selection.mshape + request_size = math.prod(np_shape) + if item_size == "H5T_VARIABLE": + request_size *= VARIABLE_AVG_ITEM_SIZE + else: + request_size *= item_size + max_request_size = int(config.get("max_request_size")) + if request_size >= max_request_size: + msg = "GET query request too large" + log.warn(msg) + raise HTTPRequestEntityTooLarge(max_request_size, request_size) + + arr_rsp = await getSelectionData( + app, + dset_id, + dset_json, + slices=selection, + query=query, + bucket=bucket, + limit=limit, + query_indices=True, + ) + + resp = await arrayResponse(arr_rsp, request, dset_json) + log.response(request, resp=resp) + return resp + + async def POST_Value(request): """ Handler for POST //value request - point selection or hyperslab read @@ -1247,7 +1243,7 @@ async def POST_Value(request): msg = "Missing dataset id" log.warn(msg) raise HTTPBadRequest(reason=msg) - if not isValidUuid(dset_id, "Dataset"): + if not isValidUuid(dset_id, obj_class="datasets"): msg = f"Invalid dataset id: {dset_id}" log.warn(msg) raise HTTPBadRequest(reason=msg) @@ -1289,12 +1285,11 @@ async def POST_Value(request): msg = "POST value not supported for datasets with NULL shape" log.warn(msg) raise HTTPBadRequest(reason=msg) - if isScalarSpace(dset_json): + if isScalar(dset_json): msg = "POST value not supported for datasets with SCALAR shape" log.warn(msg) raise HTTPBadRequest(reason=msg) - datashape = dset_json["shape"] - dims = getShapeDims(datashape) + dims = getShapeDims(dset_json) rank = len(dims) type_json = dset_json["type"] @@ -1303,7 +1298,7 @@ async def POST_Value(request): log.debug(f"item size: {item_size}") # read body data - slices = None # this will be set for hyperslab selection + selection = None # this will be set for hyperslab selection points = None # this will be set for point selection point_dt = np.dtype("u8") # use unsigned long for point index @@ -1315,12 +1310,12 @@ async def POST_Value(request): log.warn(msg) raise HTTPBadRequest(reason=msg) - if _isSelect(params, body=body) and "points" in body: + if isSelect(params, body=body) and "points" in body: msg = "Unexpected points and select key in request body" log.warn(msg) raise HTTPBadRequest(reason=msg) - slices = _getSelect(params, dset_json, body=body) + selection = _getSelect(params, dset_json, body=body) select_dtype = _getSelectDtype(params, dset_dtype, body=body) if request_type == "json": @@ -1333,7 +1328,7 @@ async def POST_Value(request): points = np.asarray(points_list, dtype=point_dt) log.debug(f"get {len(points)} points from json request") - elif not _isSelect(params, body=body): + elif not isSelect(params, body=body): msg = "Expected points or select key in request body" log.warn(msg) raise HTTPBadRequest(reason=msg) @@ -1351,7 +1346,7 @@ async def POST_Value(request): log.warn(msg) raise HTTPBadRequest(reason=msg) num_points = request.content_length // point_dt.itemsize - points = np.fromstring(binary_data, dtype=point_dt) + points = np.frombuffer(binary_data, dtype=point_dt) # reshape the data based on the rank (num_points x rank) if rank > 1: if len(points) % rank != 0: @@ -1366,9 +1361,9 @@ async def POST_Value(request): log.debug(f"got {len(points)} num_points") # get the shape of the response array - if _isSelect(params, body=body): + if isSelect(params, body=body): # hyperslab post - np_shape = getSelectionShape(slices) + np_shape = selection.mshape else: # point selection np_shape = [len(points), ] @@ -1435,7 +1430,7 @@ async def POST_Value(request): kwargs = {"bucket": bucket} if points is None: - kwargs["slices"] = slices + kwargs["slices"] = selection else: kwargs["points"] = points kwargs["select_dtype"] = select_dtype @@ -1457,10 +1452,9 @@ async def POST_Value(request): else: log.debug("POST Value - returning JSON data") resp_json = {} - data = arr_rsp.tolist() - log.debug(f"got rsp data {len(data)} points") + log.debug(f"got rsp data shape: {arr_rsp.shape}") try: - json_data = bytesArrayToList(data) + json_data = bytesArrayToList(arr_rsp) except ValueError as err: msg = f"Cannot decode bytes to list: {err}" raise HTTPBadRequest(reason=msg) @@ -1474,10 +1468,6 @@ async def POST_Value(request): await resp.write(resp_body) except Exception as e: log.error(f"{type(e)} Exception during response write") - import traceback - - tb = traceback.format_exc() - print("traceback:", tb) # finalize response await resp.write_eof() diff --git a/hsds/chunklocator.py b/hsds/chunklocator.py index 6727de9e..76b0767e 100644 --- a/hsds/chunklocator.py +++ b/hsds/chunklocator.py @@ -1,17 +1,20 @@ import sys -import time import h5py import s3fs import numpy as np from . import config from . import hsds_logger as log -from .util.arrayUtil import bytesArrayToList, getNumElements -from .util.dsetUtil import getSelectionList, getSelectionShape + +from h5json.time_util import getNow +from h5json.array_util import bytesArrayToList, getNumElements + +from .util.dsetUtil import getSelectionList def get_cmd_options(): """ read command line options and return as dict """ - required = ("fileuri", "h5path", "select") + required = ("fileuri", "h5path") + optional = ("select",) cmd_options = {} for option in required: val = config.getCmdLineArg(option) @@ -20,6 +23,8 @@ def get_cmd_options(): log.error(msg) sys.exit(-1) cmd_options[option] = val + for option in optional: + cmd_options[option] = config.getCmdLineArg(option) return cmd_options @@ -132,18 +137,20 @@ def chunk_callback(chunk_info, args): log.debug(f"using chunktable_dims: {chunktable_dims}") if select: - slices = getSelectionList(select, chunktable_dims) + selection = getSelectionList(select, chunktable_dims) + arr_shape = selection.mshape + slices = selection.slices else: slices = [] for i in range(rank): slices.append(slice(0, chunktable_dims[i])) + arr_shape = tuple(chunktable_dims[i] for i in range(rank)) log.debug(f"got slices: {slices}") - arr_shape = getSelectionShape(slices) log.debug(f"arr_shape: {arr_shape}") dtype = get_chunktable_dtype() - # initilize chunk table array + # initialize chunk table array chunkinfo_arr = np.zeros(arr_shape, dtype=dtype) dsid = dset.id @@ -189,9 +196,12 @@ def main(): prefix = config.get("log_prefix") log_timestamps = config.get("log_timestamps", default=False) log.setLogConfig(log_level, prefix=prefix, timestamps=log_timestamps) - start_time = time.time() + start_time = getNow() log.info(f"chunklocator start: {start_time:.2f}") + # expected usage example: + # hsds-chunklocator --h5path=/dset --fileuri=/hdf5/hdf5test/small1dchunk.h5 --select [0:200] + cmd_options = get_cmd_options() h5path = cmd_options["h5path"] log.info(f"h5path: {h5path}") @@ -229,6 +239,6 @@ def main(): sys.exit(1) log.info('done') - stop_time = time.time() + stop_time = getNow() log.info(f"chunklocator stop: {stop_time:.2f}") log.info(f"chunklocator elapsed: {(stop_time - start_time):.2f}") diff --git a/hsds/config.py b/hsds/config.py index a6f86934..2e69600e 100755 --- a/hsds/config.py +++ b/hsds/config.py @@ -157,7 +157,6 @@ def _load_cfg(): msg = "Error applying command line override value for " msg += f"key: {x}: {ve}" eprint(msg) - # raise KeyError(msg) cfgval = override # replace the yml value if _has_unit(cfgval): diff --git a/hsds/ctype_dn.py b/hsds/ctype_dn.py index f06b98b3..021f454b 100755 --- a/hsds/ctype_dn.py +++ b/hsds/ctype_dn.py @@ -18,11 +18,13 @@ from aiohttp.web_exceptions import HTTPInternalServerError from aiohttp.web import json_response -from .util.idUtil import isValidUuid, validateUuid +from h5json.objid import isValidUuid, validateUuid +from h5json.time_util import getNow + from .datanode_lib import get_obj_id, get_metadata_obj, save_metadata_obj from .datanode_lib import delete_metadata_obj, check_metadata_obj from .util.domainUtil import isValidBucketName -from .util.timeUtil import getNow + from . import hsds_logger as log @@ -33,7 +35,7 @@ async def GET_Datatype(request): params = request.rel_url.query ctype_id = get_obj_id(request) - if not isValidUuid(ctype_id, obj_class="type"): + if not isValidUuid(ctype_id, obj_class="datatypes"): log.error(f"Unexpected type_id: {ctype_id}") raise HTTPInternalServerError() @@ -90,15 +92,21 @@ async def POST_Datatype(request): raise HTTPBadRequest(reason=msg) ctype_id = get_obj_id(request, body=body) - if not isValidUuid(ctype_id, obj_class="datatype"): - log.error("Unexpected type_id: {ctype_id}") + if not isValidUuid(ctype_id, obj_class="datatypes"): + log.error(f"Unexpected type_id: {ctype_id}") raise HTTPInternalServerError() + deleted_ids = app["deleted_ids"] + if ctype_id in deleted_ids: + log.warn(f"POST Dataset has id: {ctype_id} that has previously been deleted") + deleted_ids.remove(ctype_id) + # verify the id doesn't already exist obj_found = await check_metadata_obj(app, ctype_id, bucket=bucket) if obj_found: - log.error(f"Post with existing type_id: {ctype_id}") - raise HTTPInternalServerError() + msg = f"Post with existing type_id: {ctype_id}" + log.warn(msg) + raise HTTPBadRequest(reason=msg) root_id = None @@ -120,10 +128,17 @@ async def POST_Datatype(request): raise HTTPInternalServerError() type_json = body["type"] + if "attributes" in body: + # initialize attributes + attrs = body["attributes"] + log.debug(f"POST datatype with attributes: {attrs}") + else: + attrs = {} + # ok - all set, create committed type obj now = getNow(app) - log.info(f"POST_datatype, typejson: {type_json}") + log.info(f"POST_datatype, type_json: {type_json}") ctype_json = { "id": ctype_id, @@ -131,7 +146,7 @@ async def POST_Datatype(request): "created": now, "lastModified": now, "type": type_json, - "attributes": {}, + "attributes": attrs, } kwargs = {"bucket": bucket, "notify": True, "flush": True} @@ -143,7 +158,7 @@ async def POST_Datatype(request): resp_json["created"] = ctype_json["created"] resp_json["lastModified"] = ctype_json["lastModified"] resp_json["type"] = type_json - resp_json["attributeCount"] = 0 + resp_json["attributeCount"] = len(attrs) resp = json_response(resp_json, status=201) log.response(request, resp=resp) diff --git a/hsds/ctype_sn.py b/hsds/ctype_sn.py index 84cdd17f..dfa96f98 100755 --- a/hsds/ctype_sn.py +++ b/hsds/ctype_sn.py @@ -16,18 +16,20 @@ from aiohttp.web_exceptions import HTTPBadRequest, HTTPGone from json import JSONDecodeError + +from h5json.objid import isValidUuid + from .util.httpUtil import getHref, respJsonAssemble, getBooleanParam from .util.httpUtil import jsonResponse -from .util.idUtil import isValidUuid -from .util.linkUtil import validateLinkName from .util.authUtil import getUserPasswordFromRequest, aclCheck from .util.authUtil import validateUserPassword from .util.domainUtil import getDomainFromRequest, getPathForDomain, isValidDomain from .util.domainUtil import getBucketForDomain, verifyRoot -from .util.hdf5dtype import validateTypeItem, getBaseTypeJson from .servicenode_lib import getDomainJson, getObjectJson, validateAction -from .servicenode_lib import getObjectIdByPath, getPathForObjectId -from .servicenode_lib import createObject, createObjectByPath, deleteObject +from .servicenode_lib import getObjectIdByPath, getPathForObjectId, deleteObject +from .servicenode_lib import getCreateArgs, createDatatypeObj +from .post_crawl import createDatatypeObjs +from .domain_crawl import DomainCrawler from . import hsds_logger as log @@ -49,7 +51,7 @@ async def GET_Datatype(request): include_attrs = True if ctype_id: - if not isValidUuid(ctype_id, "Type"): + if not isValidUuid(ctype_id, obj_class="datatypes"): msg = f"Invalid type id: {ctype_id}" log.warn(msg) raise HTTPBadRequest(reason=msg) @@ -60,7 +62,7 @@ async def GET_Datatype(request): group_id = None if "grpid" in params: group_id = params["grpid"] - if not isValidUuid(group_id, "Group"): + if not isValidUuid(group_id, obj_class="groups"): msg = f"Invalid parent group id: {group_id}" log.warn(msg) raise HTTPBadRequest(reason=msg) @@ -101,7 +103,7 @@ async def GET_Datatype(request): # throws 404 if not found kwargs = {"bucket": bucket, "domain": domain} ctype_id, domain, _ = await getObjectIdByPath(app, group_id, h5path, **kwargs) - if not isValidUuid(ctype_id, "Datatype"): + if not isValidUuid(ctype_id, obj_class="datatypes"): msg = f"No datatype exist with the path: {h5path}" log.warn(msg) raise HTTPGone() @@ -153,7 +155,7 @@ async def POST_Datatype(request): await validateUserPassword(app, username, pswd) if not request.has_body: - msg = "POST Datatype with no body" + msg = "POST datatype with no body" log.warn(msg) raise HTTPBadRequest(reason=msg) @@ -163,35 +165,6 @@ async def POST_Datatype(request): msg = "Unable to load JSON body" log.warn(msg) raise HTTPBadRequest(reason=msg) - if "type" not in body: - msg = "POST Datatype has no type key in body" - log.warn(msg) - raise HTTPBadRequest(reason=msg) - datatype = body["type"] - if isinstance(datatype, str): - try: - # convert predefined type string (e.g. "H5T_STD_I32LE") to - # corresponding json representation - datatype = getBaseTypeJson(datatype) - log.debug(f"got datatype: {datatype}") - except TypeError: - msg = "POST Dataset with invalid predefined type" - log.warn(msg) - raise HTTPBadRequest(reason=msg) - try: - validateTypeItem(datatype) - except KeyError as ke: - msg = f"KeyError creating type: {ke}" - log.warn(msg) - raise HTTPBadRequest(reason=msg) - except TypeError as te: - msg = f"TypeError creating type: {te}" - log.warn(msg) - raise HTTPBadRequest(reason=msg) - except ValueError as ve: - msg = f"ValueError creating type: {ve}" - log.warn(msg) - raise HTTPBadRequest(reason=msg) domain = getDomainFromRequest(request) if not isValidDomain(domain): @@ -207,56 +180,92 @@ async def POST_Datatype(request): verifyRoot(domain_json) root_id = domain_json["root"] - parent_id = None - link_title = None - h5path = None - if "link" in body: - if "h5path" in body: - msg = "link can't be used with h5path" - log.warn(msg) - raise HTTPBadRequest(reason=msg) - link_body = body["link"] - if "id" in link_body: - parent_id = link_body["id"] - if "name" in link_body: - link_title = link_body["name"] - try: - # will throw exception if there's a slash in the name - validateLinkName(link_title) - except ValueError: - msg = f"invalid link title: {link_title}" - log.warn(msg) - raise HTTPBadRequest(reason=msg) + # allow parent group creation or not + implicit = getBooleanParam(params, "implicit") - if parent_id and link_title: - log.debug(f"parent id: {parent_id}, link_title: {link_title}") - h5path = link_title # just use the link name as the h5path + post_rsp = None - if "h5path" in body: - h5path = body["h5path"] - if "parent_id" not in body: - parent_id = root_id + if isinstance(body, list): + count = len(body) + log.debug(f"multiple ctype create: {count} items") + if count == 0: + # equivalent to no body + msg = "POST Datatype with no body" + log.warn(msg) + raise HTTPBadRequest(reason=msg) + elif count == 1: + # just create one object in typical way + kwargs = getCreateArgs(body[0], + root_id=root_id, + bucket=bucket, + implicit=implicit) else: - parent_id = body["parent_id"] - - # setup args to createObject - kwargs = {"bucket": bucket, "obj_type": datatype} - # TBD: creation props for datatype obj? - if parent_id: - kwargs["parent_id"] = parent_id - kwargs["h5path"] = h5path - # allow parent group creation or not - implicit = getBooleanParam(params, "implicit") - if implicit: - kwargs["implicit"] = True - ctype_json = await createObjectByPath(app, **kwargs) + # create multiple ctype objects + kwarg_list = [] # list of kwargs for each object + + for item in body: + log.debug(f"item: {item}") + if not isinstance(item, dict): + msg = f"Post_Datatype - invalid item type: {type(item)}" + log.warn(msg) + raise HTTPBadRequest(reason=msg) + kwargs = getCreateArgs(item, root_id=root_id, bucket=bucket) + kwargs["ignore_link"] = True # will create parent links later + kwarg_list.append(kwargs) + kwargs = {"bucket": bucket, "root_id": root_id} + log.debug(f"createDatatypeObjects, items: {kwarg_list}") + post_rsp = await createDatatypeObjs(app, kwarg_list, **kwargs) else: - # create an anonymous datatype - kwargs["root_id"] = root_id - ctype_json = await createObject(app, **kwargs) + # single object create + kwargs = getCreateArgs(body, root_id=root_id, bucket=bucket, implicit=implicit) + log.debug(f"kwargs for datatype create: {kwargs}") + + if post_rsp is None: + # Handle cases other than multi ctype create here + post_rsp = await createDatatypeObj(app, **kwargs) + + log.debug(f"returning resp: {post_rsp}") + + if "objects" in post_rsp: + # add any links in multi request + objects = post_rsp["objects"] + obj_count = len(objects) + log.debug(f"Post datatype multi create: {obj_count} objects") + if len(body) != obj_count: + msg = f"Expected {obj_count} objects but got {len(body)}" + log.warn(msg) + raise HTTPBadRequest(reason=msg) + parent_ids = {} + for index in range(obj_count): + item = body[index] + if "link" in item: + link_item = item["link"] + parent_id = link_item.get("id") + title = link_item.get("name") + if parent_id and title: + # add a hard link + object = objects[index] + obj_id = object["id"] + if parent_id not in parent_ids: + parent_ids[parent_id] = {} + links = parent_ids[parent_id] + links[title] = {"id": obj_id} + if parent_ids: + log.debug(f"POST datatype multi - adding links: {parent_ids}") + kwargs = {"action": "put_link", "bucket": bucket} + kwargs["replace"] = True + + crawler = DomainCrawler(app, parent_ids, **kwargs) + + # will raise exception on not found, server busy, etc. + await crawler.crawl() + + status = crawler.get_status() + + log.info(f"DomainCrawler done for put_links action, status: {status}") # datatype creation successful - resp = await jsonResponse(request, ctype_json, status=201) + resp = await jsonResponse(request, post_rsp, status=201) log.response(request, resp=resp) return resp @@ -271,7 +280,7 @@ async def DELETE_Datatype(request): msg = "Missing committed type id" log.warn(msg) raise HTTPBadRequest(reason=msg) - if not isValidUuid(ctype_id, "Type"): + if not isValidUuid(ctype_id, obj_class="datatypes"): msg = f"Invalid committed type id: {ctype_id}" log.warn(msg) raise HTTPBadRequest(reason=msg) diff --git a/hsds/datanode.py b/hsds/datanode.py index b7c00b9d..8bba6cee 100644 --- a/hsds/datanode.py +++ b/hsds/datanode.py @@ -14,17 +14,17 @@ # import asyncio -import traceback from aiohttp.web import run_app +from h5json.objid import isValidUuid, isSchema2Id, getCollectionForId +from h5json.objid import isRootObjId +from h5json.time_util import getNow + from . import config from .util.lruCache import LruCache -from .util.idUtil import isValidUuid, isSchema2Id, getCollectionForId -from .util.idUtil import isRootObjId from .util.httpUtil import isUnixDomainUrl, bindToSocket, getPortFromUrl from .util.httpUtil import jsonResponse, release_http_client from .util.storUtil import setBloscThreads, getBloscThreads -from .util.timeUtil import getNow from .basenode import healthCheck, baseInit from . import hsds_logger as log from .domain_dn import GET_Domain, PUT_Domain, DELETE_Domain, PUT_ACL @@ -44,7 +44,7 @@ async def init(): - """Intitialize application and return app object""" + """Initialize application and return app object""" app = baseInit("dn") # @@ -150,20 +150,14 @@ async def bucketScan(app): msg = f"bucketScan - HTTPBadRequest error scanning {root_id}: " msg += f"{bre}" log.error(msg) - tb = traceback.format_exc() - print("traceback:", tb) except HTTPInternalServerError as ise: msg = "bucketScan - HTTPInternalServer error scanning " msg += f"{root_id}: {ise}" log.error(msg) - tb = traceback.format_exc() - print("traceback:", tb) except Exception as e: msg = "bucketScan - Unexpected exception scanning " msg += f"{root_id}: {e}" log.error(msg) - tb = traceback.format_exc() - print("traceback:", tb) last_action = getNow(app) diff --git a/hsds/datanode_lib.py b/hsds/datanode_lib.py index 08ecc52a..4189dc3e 100644 --- a/hsds/datanode_lib.py +++ b/hsds/datanode_lib.py @@ -19,22 +19,30 @@ from aiohttp.web_exceptions import HTTPGone, HTTPInternalServerError from aiohttp.web_exceptions import HTTPNotFound, HTTPForbidden from aiohttp.web_exceptions import HTTPServiceUnavailable, HTTPBadRequest -from .util.idUtil import validateInPartition, getS3Key, isValidUuid -from .util.idUtil import isValidChunkId, getDataNodeUrl, isSchema2Id -from .util.idUtil import getRootObjId, isRootObjId + +from h5json.hdf5dtype import createDataType, isVlen +from h5json.array_util import arrayToBytes, bytesToArray, jsonToArray +from h5json.filters import getFilters, getCompressionFilter, getShuffleFilter +from h5json.objid import getS3Key, isValidUuid +from h5json.objid import isValidChunkId, isSchema2Id +from h5json.objid import getRootObjId, isRootObjId +from h5json.shape_util import getShapeDims +from h5json.dset_util import getChunkDims, getDatasetLayoutClass +from h5json.time_util import getNow +from h5json import selections + +from .util.nodeUtil import getDataNodeUrl from .util.storUtil import getStorJSONObj, putStorJSONObj, putStorBytes from .util.storUtil import getStorBytes, isStorObj, deleteStorObj, getHyperChunks from .util.storUtil import getBucketFromStorURI, getKeyFromStorURI, getURIFromKey from .util.domainUtil import isValidDomain, getBucketForDomain from .util.attrUtil import getRequestCollectionName from .util.httpUtil import http_post -from .util.dsetUtil import getChunkLayout, getFilterOps, getShapeDims -from .util.dsetUtil import getChunkInitializer, getSliceQueryParam, getFilters +from .util.dsetUtil import getChunkInitializer, getSliceQueryParam from .util.chunkUtil import getDatasetId, getChunkSelection, getChunkIndex -from .util.arrayUtil import arrayToBytes, bytesToArray, jsonToArray -from .util.hdf5dtype import createDataType +from .util.nodeUtil import validateInPartition from .util.rangegetUtil import ChunkLocation, chunkMunge, getHyperChunkIndex, getHyperChunkFactors -from .util.timeUtil import getNow + from . import config from . import hsds_logger as log from .dset_lib import getFillValue @@ -115,7 +123,7 @@ async def check_metadata_obj(app, obj_id, bucket=None): if obj_id in meta_cache: found = True else: - # Not in chache, check s3 obj exists + # Not in cache, check s3 obj exists s3_key = getS3Key(obj_id) log.debug(f"check_metadata_obj({s3_key})") # does key exist? @@ -231,7 +239,7 @@ async def write_s3_obj(app, obj_id, bucket=None): dirty_count = chunk_cache.dirtyCount msg = f"write_s3_obj: {obj_id} updated - " msg += f"Chunk cache utilization: {cache_utilization} " - msg += "per, dirty_count: {dirty_count}" + msg += f"per, dirty_count: {dirty_count}" log.debug(msg) else: # meta data update @@ -554,6 +562,54 @@ async def delete_metadata_obj(app, obj_id, notify=True, root_id=None, bucket=Non log.debug(f"delete_metadata_obj for {obj_id} done") +def getFilterOps(app, dset_id, filters, dtype=None, chunk_shape=None): + """Get list of filter operations to be used for this dataset""" + filter_map = app["filter_map"] + + if dset_id in filter_map: + return filter_map[dset_id] + + compressionFilter = getCompressionFilter(filters) + + filter_ops = {} + + shuffleFilter = getShuffleFilter(filters) + + if shuffleFilter and not isVlen(dtype): + shuffle_name = shuffleFilter["name"] + if shuffle_name == "shuffle": + filter_ops["shuffle"] = 1 # use regular shuffle + elif shuffle_name == "bitshuffle": + filter_ops["shuffle"] = 2 # use bitshuffle + else: + filter_ops["shuffle"] = 0 # no shuffle + else: + filter_ops["shuffle"] = 0 # no shuffle + + if compressionFilter: + if compressionFilter["class"] == "H5Z_FILTER_DEFLATE": + filter_ops["compressor"] = "zlib" # blosc compressor + else: + if "name" in compressionFilter: + filter_ops["compressor"] = compressionFilter["name"] + else: + filter_ops["compressor"] = "lz4" # default to lz4 + if "level" not in compressionFilter: + filter_ops["level"] = 5 # medium level + else: + filter_ops["level"] = int(compressionFilter["level"]) + + if filter_ops: + # save the chunk shape and dtype + filter_ops["chunk_shape"] = chunk_shape + filter_ops["dtype"] = dtype + filter_map[dset_id] = filter_ops # save + + return filter_ops + else: + return None + + def arange_chunk_init( app, initializer, @@ -580,13 +636,12 @@ def arange_chunk_init( raise None type_class = type_json.get("class") if type_class not in ("H5T_INTEGER", "H5T_FLOAT"): - msg = "arange initializer: unsupported type class: {type_class}" + msg = f"arange initializer: unsupported type class: {type_class}" log.warn(msg) raise None - try: - chunk_layout = getChunkLayout(dset_json) - except HTTPInternalServerError: + chunk_layout = getChunkDims(dset_json) + if chunk_layout is None: msg = "non-chunked dataset" log.warning(msg) raise None @@ -596,7 +651,7 @@ def arange_chunk_init( log.debug(msg) if len(chunk_index) != 1: - msg = "expected chunk_index to be one-element list, but got: {chunk_index}" + msg = f"expected chunk_index to be one-element list, but got: {chunk_index}" log.error(msg) raise HTTPInternalServerError() @@ -710,15 +765,15 @@ async def run_chunk_initializer( dims = getShapeDims(datashape) log.debug(f"dataset shape: {dims}") # get the chunk layout for this dataset - layout = getChunkLayout(dset_json) + layout = getChunkDims(dset_json) log.debug(f"chunk layout: {layout}") rank = len(dims) slices = [] for dim in range(rank): slices.append(slice(0, dims[dim], 1)) - slices = tuple(slices) - chunk_selection = getChunkSelection(chunk_id, slices, layout) + selection = selections.select(tuple(dims), tuple(slices)) + chunk_selection = getChunkSelection(chunk_id, selection, layout) log.debug(f"got chunk_selection: {chunk_selection}") select = getSliceQueryParam(chunk_selection) select_arg = f"--select={select}" @@ -1004,12 +1059,11 @@ async def get_chunk( log.debug(msg) chunk_arr = None - dims = getChunkLayout(dset_json) + dims = getChunkDims(dset_json) type_json = dset_json["type"] dt = createDataType(type_json) - layout_json = dset_json["layout"] - layout_class = layout_json.get("class") - chunk_dims = getChunkLayout(dset_json) + layout_class = getDatasetLayoutClass(dset_json) + chunk_dims = getChunkDims(dset_json) fill_value = getFillValue(dset_json) # note - officially we should follow the order in which the filters are @@ -1034,14 +1088,13 @@ async def get_chunk( except ValueError as ve: log.error(f"Invalid URI path: {s3path} exception: {ve}") raise - # raise HTTPInternalServerError() msg = f"Using s3path bucket: {bucket} and s3key: {s3key} " msg += f"offset: {s3offset} length: {s3size}" log.debug(msg) else: s3key = getS3Key(chunk_id) - log.debug(f"getChunk chunkid: {chunk_id} bucket: {bucket}") + log.debug(f"getChunk chunkid: {chunk_id} bucket: {bucket} using key: {s3key}") if chunk_id in chunk_cache: log.debug(f"getChunk chunkid: {chunk_id} found in cache") chunk_arr = chunk_cache[chunk_id] @@ -1121,7 +1174,7 @@ async def get_chunk( # no room in the cache, just skip caching msg = "getChunk, cache utilization: " msg += f"{chunk_cache.cacheUtilizationPercent}, " - msg += "skip cache for chunk_id {chunk_id}" + msg += f"skip cache for chunk_id {chunk_id}" log.warn(msg) if chunk_arr is None and chunk_init: @@ -1163,7 +1216,7 @@ def save_chunk(app, chunk_id, dset_json, chunk_arr, bucket=None): dset_id = dset_json["id"] dtype = createDataType(dset_json["type"]) - chunk_shape = getChunkLayout(dset_json) + chunk_shape = getChunkDims(dset_json) # will store filter options into app['filter_map'] filters = getFilters(dset_json) diff --git a/hsds/domain_crawl.py b/hsds/domain_crawl.py index b8e0ba39..d9285d45 100644 --- a/hsds/domain_crawl.py +++ b/hsds/domain_crawl.py @@ -18,8 +18,11 @@ from aiohttp.web_exceptions import HTTPServiceUnavailable, HTTPConflict, HTTPBadRequest from aiohttp.web_exceptions import HTTPInternalServerError, HTTPNotFound, HTTPGone -from .util.httpUtil import isOK -from .util.idUtil import getCollectionForId, getDataNodeUrl +from h5json.objid import getCollectionForId +from h5json.array_util import arrayToBytes + +from .util.nodeUtil import getDataNodeUrl +from .util.httpUtil import isOK, http_put from .util.globparser import globmatch from .servicenode_lib import getObjectJson, getAttributes, putAttributes, getLinks, putLinks from . import hsds_logger as log @@ -231,7 +234,7 @@ async def put_attributes(self, obj_id, attr_items): try: status = await putAttributes(self._app, obj_id, attr_items, **kwargs) except HTTPConflict: - log.warn("DomainCrawler - got HTTPConflict from http_put") + log.warn("DomainCrawler - got HTTPConflict from putAttributers") status = 409 except HTTPServiceUnavailable: status = 503 @@ -246,7 +249,7 @@ async def put_attributes(self, obj_id, attr_items): async def get_obj_json(self, obj_id): """ get the given obj_json for the obj_id. for each group found, search the links if follow_links is set """ - log.debug(f"get_obj_json: {obj_id}") + log.debug(f"DomainCrawler get_obj_json: {obj_id}") collection = getCollectionForId(obj_id) kwargs = {"bucket": self._bucket, "include_attrs": self._include_attrs} @@ -406,7 +409,7 @@ async def get_links(self, grp_id, titles=None): async def put_links(self, grp_id, link_items): # write the given links for the obj_id - log.debug(f"put_links for {grp_id}, {len(link_items)} links") + log.debug(f"DomainCrawler put_links for {grp_id}, {len(link_items)} links") req = getDataNodeUrl(self._app, grp_id) req += f"/groups/{grp_id}/links" kwargs = {"bucket": self._bucket} @@ -417,8 +420,10 @@ async def put_links(self, grp_id, link_items): log.warn("DomainCrawler - got HTTPConflict from http_put") status = 409 except HTTPServiceUnavailable: + log.warn("DomainCrawler - got HTTPServiceUnavailable exception") status = 503 except HTTPInternalServerError: + log.warn("DomainCrawler - got 500 error from DN") status = 500 except Exception as e: log.error(f"unexpected exception {e}") @@ -426,6 +431,38 @@ async def put_links(self, grp_id, link_items): log.debug(f"DomainCrawler fetch for {grp_id} - returning status: {status}") self._obj_dict[grp_id] = {"status": status} + async def put_data(self, chunk_id, arr): + # write a one-chunk dataset value + log.debug(f"DomainCrawler put_data for {chunk_id}, arr.shape: {arr.shape}") + req = getDataNodeUrl(self._app, chunk_id) + req += "/chunks/" + chunk_id + params = {"bucket": self._bucket} + + data = arrayToBytes(arr) + + log.debug(f"DomainCrawler - put_data req: {req}, {len(data)} bytes") + try: + rsp = await http_put(self._app, req, data=data, params=params) + log.debug(f"http_put return: {rsp}") + status = 200 + except HTTPConflict: + log.warn("DomainCrawler - got HTTPConflict from http_put") + status = 409 + except HTTPServiceUnavailable: + log.warn("DomainCrawler - got HTTPServiceUnavailable exception") + status = 503 + except HTTPInternalServerError: + log.warn("DomainCrawler - got 500 error from DN") + status = 500 + except Exception as e: + log.error(f"unexpected exception {e}") + status = 500 + finally: + log.debug("DomainCrawler put_data end try") + + log.debug(f"DomainCrawler put_data for {chunk_id} - returning status: {status}") + self._obj_dict[chunk_id] = {"status": status} + def get_status(self): """ return the highest status of any of the returned objects """ status = None @@ -464,7 +501,7 @@ async def crawl(self): pass # ok elif status == 400: log.warn("DomainCrawler - BadRequest") - raise HTTPBadRequest(reason="unkown") + raise HTTPBadRequest(reason="unknown") elif status == 404: log.warn("DomainCrawler - not found") raise HTTPNotFound() @@ -526,7 +563,7 @@ async def fetch(self, obj_id): await self.put_attributes(obj_id, attr_items) elif self._action == "get_link": - log.debug("DomainCrawlwer - get links") + log.debug("DomainCrawler - get links") log.debug(f"self._objs: {self._objs}, type: {type(self._objs)}") if self._objs is None or obj_id not in self._objs: @@ -546,7 +583,7 @@ async def fetch(self, obj_id): log.debug(f"DomainCrawler - get link titles: {link_titles}") await self.get_links(obj_id, link_titles) elif self._action == "put_link": - log.debug("DomainCrawlwer - put links") + log.debug("DomainCrawler - put links") # write links if self._objs and obj_id not in self._objs: log.error(f"couldn't find {obj_id} in self._objs") @@ -555,11 +592,23 @@ async def fetch(self, obj_id): log.debug(f"got {len(link_items)} link items for {obj_id}") await self.put_links(obj_id, link_items) + elif self._action == "put_data": + log.debug("DomainCrawler - put data") + # write one chunk per dataset + if self._objs and obj_id not in self._objs: + log.error(f"couldn't find {obj_id} in self._objs") + return + data = self._objs[obj_id] + if data is None: + log.error(f"no data found for {obj_id}") + return + + await self.put_data(obj_id, data) else: msg = f"DomainCrawler: unexpected action: {self._action}" log.error(msg) msg = f"DomainCrawler - fetch complete obj_id: {obj_id}, " - msg += f"{len(self._obj_dict)} objects found" + msg += f"{len(self._obj_dict)} objects processed" log.debug(msg) log.debug(f"obj_dict: {len(self._obj_dict)} items") diff --git a/hsds/domain_dn.py b/hsds/domain_dn.py index 83932e5d..c7ce200e 100755 --- a/hsds/domain_dn.py +++ b/hsds/domain_dn.py @@ -16,10 +16,11 @@ from aiohttp.web_exceptions import HTTPConflict, HTTPInternalServerError from aiohttp.web import json_response +from h5json.time_util import getNow + from .util.authUtil import getAclKeys from .util.domainUtil import isValidDomain, getBucketForDomain -from .util.idUtil import validateInPartition -from .util.timeUtil import getNow +from .util.nodeUtil import validateInPartition from .datanode_lib import get_metadata_obj, save_metadata_obj from .datanode_lib import delete_metadata_obj, check_metadata_obj from . import hsds_logger as log @@ -134,8 +135,6 @@ async def PUT_Domain(request): now = getNow(app) domain_json["created"] = now domain_json["lastModified"] = now - # if "class" in body_json: - # domain_json["class"] = body_json["class"] # write the domain json to S3 immediately so it will show up in a get # domains S3 scan diff --git a/hsds/domain_sn.py b/hsds/domain_sn.py index 56d3611a..5a6db9ef 100755 --- a/hsds/domain_sn.py +++ b/hsds/domain_sn.py @@ -17,25 +17,32 @@ import json import os.path as op +import numpy as np + from aiohttp.web_exceptions import HTTPBadRequest, HTTPForbidden, HTTPNotFound -from aiohttp.web_exceptions import HTTPInternalServerError +from aiohttp.web_exceptions import HTTPInternalServerError, HTTPGone from aiohttp.web_exceptions import HTTPConflict, HTTPServiceUnavailable from aiohttp.web import json_response +from h5json.objid import createObjId, getCollectionForId +from h5json.objid import isValidUuid, isRootObjId, isSchema2Id +from h5json.time_util import getNow +from h5json.hdf5dtype import createDataType +from h5json.array_util import getNumpyValue +from h5json.query_util import arrayQuery + + +from .util.nodeUtil import getNodeCount, getDataNodeUrl from .util.httpUtil import getObjectClass, http_post, http_put, http_delete from .util.httpUtil import getHref, respJsonAssemble from .util.httpUtil import jsonResponse -from .util.idUtil import getDataNodeUrl, createObjId, getCollectionForId -from .util.idUtil import isValidUuid, isSchema2Id, getNodeCount from .util.authUtil import getUserPasswordFromRequest, aclCheck, isAdminUser from .util.authUtil import validateUserPassword, getAclKeys from .util.domainUtil import getParentDomain, getDomainFromRequest from .util.domainUtil import isValidDomain, getBucketForDomain, isValidBucketName from .util.domainUtil import getPathForDomain, getLimits from .util.storUtil import getStorKeys, getCompressors -from .util.boolparser import BooleanParser from .util.globparser import globmatch -from .util.timeUtil import getNow from .servicenode_lib import getDomainJson, getObjectJson, getObjectIdByPath from .servicenode_lib import getRootInfo, checkBucketAccess, doFlush, getDomainResponse from .basenode import getVersion @@ -97,7 +104,7 @@ async def get_collections(app, root_id, bucket=None, max_objects_limit=None): async def getDomainObjects(app, root_id, include_attrs=False, bucket=None): - """Iterate through all objects in heirarchy and add to obj_dict + """Iterate through all objects in hierarchy and add to obj_dict keyed by obj id """ @@ -148,6 +155,54 @@ def getIdList(objs, marker=None, limit=None): return ret_ids +_QUERYABLE_ATTR_TYPES = ("H5T_INTEGER", "H5T_FLOAT", "H5T_STRING") + + +def _isMissingAttributeError(exc): + """ True if the given h5json.query_util.arrayQuery exception is just + because a referenced attribute name isn't defined for the dtype it + was run against - as opposed to a genuine syntax error. Not treated + as fatal for domain queries, since different domains can have + different attributes: a query naming an attribute that doesn't exist + on a given domain (or on any domain at all) should just not match + that domain, not fail the whole request. """ + msg = str(exc) + return "not found in dtype" in msg or "is not valid for non-compound dtype" in msg + + +def _getQueryableFields(attributes): + """ Given a group's "attributes" dict (as returned by the crawler), + return (field_defs, values) for just the scalar primitive attributes, + suitable for building a single-row structured numpy array to + evaluate a domain query against - the same query syntax/engine used + by GET .../value (h5json.query_util.arrayQuery). """ + field_defs = [] + values = [] + for attr_name in attributes: + attr_json = attributes[attr_name] + attr_type = attr_json.get("type", {}) + if attr_type.get("class") not in _QUERYABLE_ATTR_TYPES: + continue + attr_shape = attr_json.get("shape", {}) + if attr_shape.get("class") != "H5S_SCALAR": + continue + field_defs.append({"name": attr_name, "type": attr_type}) + values.append(attr_json.get("value")) + return field_defs, values + + +def _domainRowArray(field_defs, values): + """ Build a single-row structured numpy array from the given field + definitions and values (as returned by _getQueryableFields). """ + row_type = {"class": "H5T_COMPOUND", "fields": field_defs} + row_dtype = createDataType(row_type) + row_arr = np.zeros((1,), dtype=row_dtype) + for field_def, value in zip(field_defs, values): + field_name = field_def["name"] + row_arr[field_name][0] = getNumpyValue(value, dt=row_dtype[field_name]) + return row_arr + + async def get_domains(request): """This method is called by GET_Domains and GET_Domain""" app = request.app @@ -302,75 +357,78 @@ async def get_domains(request): if query: log.info(f"get_domains - proccessing query: {query}") - try: - parser = BooleanParser(query) - except IndexError as ie: - log.warn(f"get_domains - domain query syntax error: {ie}") - raise HTTPBadRequest(reason="Invalid query expression") - attr_names = parser.getVariables() - log.info(f"get_domains - query variables: {attr_names}") - # remove any domains from dict for which the attribute query is false + # remove any domains from dict for which the attribute query is + # false (or doesn't apply - e.g. folders, or missing attributes) domain_keys = list(crawler._domain_dict.keys()) log.debug(f"get_domains - querying through {len(domain_keys)}") + # Validate the query once upfront, against the union of every + # queryable attribute seen across all candidate domains. This + # surfaces genuine syntax errors as 400 without rejecting the + # whole request just because some (or all) domains don't happen + # to have a given attribute - that's a per-domain non-match + # below, not a request error. + union_fields = {} + for domain in domain_keys: + domain_json = crawler._domain_dict[domain] + if "root" not in domain_json: + continue + root_id = domain_json["root"] + if root_id not in crawler._group_dict: + continue + root_json = crawler._group_dict[root_id] + field_defs, _ = _getQueryableFields(root_json.get("attributes", {})) + for field_def in field_defs: + union_fields.setdefault(field_def["name"], field_def["type"]) + + if union_fields: + union_field_defs = [{"name": n, "type": t} for n, t in union_fields.items()] + try: + union_type = {"class": "H5T_COMPOUND", "fields": union_field_defs} + dummy_arr = np.zeros((1,), dtype=createDataType(union_type)) + arrayQuery(query, dummy_arr) + except (TypeError, ValueError) as e: + if not _isMissingAttributeError(e): + msg = f"get_domains - invalid query: {query}: {e}" + log.warn(msg) + raise HTTPBadRequest(reason="Invalid query expression") + for domain in domain_keys: log.debug(f"get_domains - query search for: {domain}") domain_json = crawler._domain_dict[domain] if "root" not in domain_json: msg = f"get_domains - skipping folder: {domain} for " msg += "attribute query search" - log.debug() - del domain_keys[domain] + log.debug(msg) + del crawler._domain_dict[domain] continue root_id = domain_json["root"] if root_id not in crawler._group_dict: log.warn(f"Expected to find {root_id} in crawler group dict") + del crawler._domain_dict[domain] continue root_json = crawler._group_dict[root_id] - attributes = root_json["attributes"] - variable_dict = {} - for attr_name in attr_names: - if attr_name not in attributes: - log.debug(f"{attr_name} not found") - del crawler._domain_dict[domain] - continue - attr_json = attributes[attr_name] - log.debug(f"{attr_name}: {attr_json}") - attr_type = attr_json["type"] - attr_type_class = attr_type["class"] - primative_types = ("H5T_INTEGER", "H5T_FLOAT", "H5T_STRING") - if attr_type_class not in primative_types: - msg = "unable to query non-primitive attribute class: " - msg += f"{attr_type_class}" - log.debug(msg) - del crawler._domain_dict[domain] - continue - attr_shape = attr_json["shape"] - attr_shape_class = attr_shape["class"] - if attr_shape_class == "H5S_SCALAR": - variable_dict[attr_name] = attr_json["value"] - else: - msg = "get_domains - unable to query non-scalar " - msg += "attributes" - log.debug(msg) - del crawler._domain_dict[domain] - continue - # evaluate the boolean expression - if len(variable_dict) == len(attr_names): - # we have all the variables, evaluate - parser_value = False - try: - parser_value = parser.evaluate(variable_dict) - except TypeError as te: - msg = f"get_domains - evaluate {query} for {domain} but " - msg += f"got error: {te}" - log.warn(msg) - if parser_value: - log.info(f"get_domains - {domain} passed query test") - else: - log.debug(f"get_domains - {domain} failed query test") - del crawler._domain_dict[domain] + field_defs, values = _getQueryableFields(root_json.get("attributes", {})) + + if not field_defs: + log.debug(f"get_domains - no queryable attributes for {domain}") + del crawler._domain_dict[domain] + continue + + try: + matches = arrayQuery(query, _domainRowArray(field_defs, values)) + except (TypeError, ValueError) as e: + msg = f"get_domains - query: {query} for {domain}: {e}" + log.debug(msg) + del crawler._domain_dict[domain] + continue + + if len(matches) > 0: + log.info(f"get_domains - {domain} passed query test") + else: + log.debug(f"get_domains - {domain} failed query test") + del crawler._domain_dict[domain] for domain in domainNames: if domain in crawler._domain_dict: @@ -444,6 +502,13 @@ async def GET_Domain(request): log.warn(f"Invalid domain: {domain}") raise HTTPBadRequest(reason="Invalid domain name") + if not domain: + # no domain param/header given - use GET /domains to list + # top-level domains instead + msg = "No domain provided" + log.warn(msg) + raise HTTPBadRequest(reason=msg) + bucket = getBucketForDomain(domain) log.debug(f"GET_Domain domain: {domain} bucket: {bucket}") @@ -459,15 +524,10 @@ async def GET_Domain(request): if "verbose" in params and params["verbose"]: verbose = True - if not domain: - log.info("no domain passed in, returning all top-level domains") - # no domain passed in, return top-level domains for this request - domains = await get_domains(request) - rsp_json = {"domains": domains} - rsp_json["hrefs"] = [] - resp = await jsonResponse(request, rsp_json) - log.response(request, resp=resp) - return resp + getobjs = False + # include domain objects if requested + if params.get("getobjs"): + getobjs = True log.info(f"get domain: {domain}") @@ -540,23 +600,9 @@ async def GET_Domain(request): return resp # return just the keys as per the REST API - kwargs = {"verbose": verbose, "bucket": bucket} + kwargs = {"verbose": verbose, "getobjs": getobjs, "bucket": bucket} rsp_json = await getDomainResponse(app, domain_json, **kwargs) - # include domain objects if requested - if params.get("getobjs") and "root" in domain_json: - - log.debug("getting all domain objects") - root_id = domain_json["root"] - kwargs = {"include_attrs": include_attrs, "bucket": bucket} - domain_objs = await getDomainObjects(app, root_id, **kwargs) - if domain_objs: - rsp_json["domain_objs"] = domain_objs - - # include domain class if present - # if "class" in domain_json: - # rsp_json["class"] = domain_json["class"] - # include dn_ids if requested if "getdnids" in params and params["getdnids"]: rsp_json["dn_ids"] = app["dn_ids"] @@ -752,7 +798,7 @@ async def PUT_Domain(request): username, pswd = getUserPasswordFromRequest(request) await validateUserPassword(app, username, pswd) - # inital perms for owner and default + # initial perms for owner and default owner_perm = { "create": True, "read": True, @@ -853,17 +899,22 @@ async def PUT_Domain(request): log.info(f"rescan for domain: {domain}") domain_json = await getDomainJson(app, domain, reload=True) log.debug(f"got domain_json: {domain_json}") - if "root" in domain_json: - # nothing to update for folders + if "root" not in domain_json: + # rescan only makes sense for domains with a root group - + # folder domains have nothing to scan + msg = f"rescan not supported for folder domain: {domain}" + log.warn(msg) + raise HTTPBadRequest(reason=msg) + else: root_id = domain_json["root"] - if not isValidUuid(root_id): + if not isValidUuid(root_id, obj_class="groups"): msg = f"domain: {domain} with invalid root id: {root_id}" log.error(msg) raise HTTPInternalServerError() if not isSchema2Id(root_id): msg = "rescan not supported for v1 ids" log.info(msg) - raise HTTPBadRequest(reashon=msg) + raise HTTPBadRequest(reason=msg) aclCheck(app, domain_json, "update", username) log.debug(f"notify_root: {root_id}") notify_req = getDataNodeUrl(app, root_id) + "/roots/" + root_id @@ -901,15 +952,12 @@ async def PUT_Domain(request): linked_domain = None linked_bucket = None root_id = None - # domain_class = None if body and "folder" in body: if body["folder"]: is_folder = True if body and "owner" in body: owner = body["owner"] - # if body and "class" in body: - # domain_class = body["class"] if body and "linked_domain" in body: if is_folder: msg = "Folder domains can not be used for links" @@ -983,8 +1031,33 @@ async def PUT_Domain(request): if not is_folder and not linked_json: # create a root group for the new domain - root_id = createObjId("roots") - log.debug(f"new root group id: {root_id}") + if body and "root_id" in body: + root_id = body["root_id"] + if not isRootObjId(root_id): + msg = f"invalid client provided root id: {root_id}" + log.warn(msg) + raise HTTPBadRequest(reason=msg) + # verify that the group object doesn't already exist + log.debug(f"attempting to fetch root id: {root_id}") + kwargs = { + "refresh": True, + "include_links": False, + "include_attrs": False, + "bucket": bucket, + } + try: + await getObjectJson(app, root_id, **kwargs) + msg = "client specified root_id already exists" + log.warn(msg) + raise HTTPConflict() + except HTTPNotFound: + log.debug(f"root_id: {root_id} not found (expected)") + except HTTPGone: + log.debug(f"root_id: {root_id} has been removed (expected)") + log.debug(f"using client supplied root_id: {root_id}") + else: + root_id = createObjId("groups") + log.debug(f"new root group id: {root_id}") group_json = {"id": root_id, "root": root_id, "domain": domain} log.debug(f"create group for domain, body: {group_json}") @@ -1049,9 +1122,6 @@ async def PUT_Domain(request): if root_id: body["root"] = root_id - # if domain_class: - # body["class"] = domain_class - log.debug(f"creating domain: {domain} with body: {body}") domain_json = await http_put(app, req, data=body) @@ -1381,6 +1451,13 @@ async def PUT_ACL(request): if bucket: checkBucketAccess(app, bucket, action="write") + # use reload to get authoritative domain json + domain_json = await getDomainJson(app, domain, reload=True) + + # validate that the requesting user has permission to update ACLs + # in this domain - throws exception if not authorized + aclCheck(app, domain_json, "updateACL", username) + # don't use app["domain_cache"] if a direct domain request is made # as opposed to an implicit request as with other operations, query # the domain from the authoritative source (the dn node) diff --git a/hsds/dset_dn.py b/hsds/dset_dn.py index 34a8ff6f..1be2daf2 100755 --- a/hsds/dset_dn.py +++ b/hsds/dset_dn.py @@ -17,10 +17,10 @@ from aiohttp.web_exceptions import HTTPInternalServerError from aiohttp.web import json_response +from h5json.objid import isValidUuid, validateUuid +from h5json.time_util import getNow -from .util.idUtil import isValidUuid, validateUuid from .util.domainUtil import isValidBucketName -from .util.timeUtil import getNow from .datanode_lib import get_obj_id, check_metadata_obj, get_metadata_obj from .datanode_lib import save_metadata_obj, delete_metadata_obj from . import hsds_logger as log @@ -33,7 +33,7 @@ async def GET_Dataset(request): params = request.rel_url.query dset_id = get_obj_id(request) - if not isValidUuid(dset_id, obj_class="dataset"): + if not isValidUuid(dset_id, obj_class="datasets"): log.error(f"Unexpected dataset_id: {dset_id}") raise HTTPInternalServerError() if "bucket" in params: @@ -57,9 +57,12 @@ async def GET_Dataset(request): resp_json["shape"] = dset_json["shape"] resp_json["attributeCount"] = len(dset_json["attributes"]) if "creationProperties" in dset_json: - resp_json["creationProperties"] = dset_json["creationProperties"] + cpl = dset_json["creationProperties"] + else: + cpl = {} if "layout" in dset_json: - resp_json["layout"] = dset_json["layout"] + cpl["layout"] = dset_json["layout"] + resp_json["creationProperties"] = cpl if "include_attrs" in params and params["include_attrs"]: resp_json["attributes"] = dset_json["attributes"] @@ -94,15 +97,21 @@ async def POST_Dataset(request): raise HTTPBadRequest(reason=msg) dset_id = get_obj_id(request, body=body) - if not isValidUuid(dset_id, obj_class="dataset"): + if not isValidUuid(dset_id, obj_class="datasets"): log.error(f"Unexpected dataset_id: {dset_id}") raise HTTPInternalServerError() + deleted_ids = app["deleted_ids"] + if dset_id in deleted_ids: + log.warn(f"POST Dataset has id: {dset_id} that has previously been deleted") + deleted_ids.remove(dset_id) + # verify the id doesn't already exist obj_found = await check_metadata_obj(app, dset_id, bucket=bucket) if obj_found: - log.error("Post with existing dset_id: {}".format(dset_id)) - raise HTTPInternalServerError() + msg = f"Post with existing dset_id: {dset_id}" + log.warn(msg) + raise HTTPBadRequest(reason=msg) if "root" not in body: msg = "POST_Dataset with no root" @@ -127,14 +136,20 @@ async def POST_Dataset(request): raise HTTPInternalServerError() shape_json = body["shape"] - layout = None if "layout" in body: - layout = body["layout"] # client specified chunk layout + log.error("unexpected key for POST Dataset: 'layout'") - # ok - all set, create committed type obj + # ok - all set, create dataset obj now = getNow(app) - log.debug(f"POST_dataset typejson: {type_json}, shapejson: {shape_json}") + if "attributes" in body: + # initialize attributes + attrs = body["attributes"] + log.debug(f"POST Dataset with attributes: {attrs}") + else: + attrs = {} + + log.debug(f"POST_dataset type_json: {type_json}, shape_json: {shape_json}") dset_json = { "id": dset_id, @@ -143,13 +158,14 @@ async def POST_Dataset(request): "lastModified": now, "type": type_json, "shape": shape_json, - "attributes": {}, + "attributes": attrs, } if "creationProperties" in body: - dset_json["creationProperties"] = body["creationProperties"] - if layout is not None: - dset_json["layout"] = layout + cpl = body["creationProperties"] + else: + cpl = {} + dset_json["creationProperties"] = cpl kwargs = {"bucket": bucket, "notify": True, "flush": True} await save_metadata_obj(app, dset_id, dset_json, **kwargs) @@ -161,7 +177,8 @@ async def POST_Dataset(request): resp_json["type"] = type_json resp_json["shape"] = shape_json resp_json["lastModified"] = dset_json["lastModified"] - resp_json["attributeCount"] = 0 + resp_json["attributeCount"] = len(attrs) + resp_json["creationProperties"] = cpl resp = json_response(resp_json, status=201) log.response(request, resp=resp) @@ -176,7 +193,7 @@ async def DELETE_Dataset(request): dset_id = request.match_info.get("id") log.info(f"DELETE dataset: {dset_id}") - if not isValidUuid(dset_id, obj_class="dataset"): + if not isValidUuid(dset_id, obj_class="datasets"): log.error(f"Unexpected dataset id: {dset_id}") raise HTTPInternalServerError() @@ -220,7 +237,7 @@ async def PUT_DatasetShape(request): params = request.rel_url.query dset_id = request.match_info.get("id") - if not isValidUuid(dset_id, obj_class="dataset"): + if not isValidUuid(dset_id, obj_class="datasets"): log.error(f"Unexpected dset_id: {dset_id}") raise HTTPInternalServerError() @@ -273,7 +290,7 @@ async def PUT_DatasetShape(request): if i == extend_dim: lb = dims[i] ub = lb + extension - if maxdims[extend_dim] != 0 and ub > maxdims[extend_dim]: + if maxdims[extend_dim] not in (0, "H5S_UNLIMITED") and ub > maxdims[extend_dim]: msg = "maximum extent exceeded" log.warn(msg) raise HTTPConflict() @@ -299,6 +316,7 @@ async def PUT_DatasetShape(request): # write back to S3, save to metadata cache log.info(f"Updated dimensions: {dims}") + resp_json["dims"] = dims await save_metadata_obj(app, dset_id, dset_json, bucket=bucket) resp = json_response(resp_json, status=201) diff --git a/hsds/dset_lib.py b/hsds/dset_lib.py index 1fe89b3e..c8d8a011 100755 --- a/hsds/dset_lib.py +++ b/hsds/dset_lib.py @@ -11,22 +11,31 @@ ############################################################################## import asyncio +from asyncio import IncompleteReadError + import math import numpy as np from aiohttp.client_exceptions import ClientError -from aiohttp.web_exceptions import HTTPBadRequest, HTTPConflict, HTTPInternalServerError -from .util.arrayUtil import getNumpyValue -from .util.boolparser import BooleanParser -from .util.dsetUtil import isNullSpace, getDatasetLayout, getDatasetLayoutClass, get_slices -from .util.dsetUtil import getChunkLayout, getSelectionShape, getShapeDims +from aiohttp.web_exceptions import HTTPBadRequest, HTTPConflict +from aiohttp.web_exceptions import HTTPInternalServerError, HTTPRequestEntityTooLarge + +from h5json.hdf5dtype import createDataType, getItemSize, getDtypeItemSize +from h5json.array_util import getNumpyValue, bytesToArray +from h5json.objid import isSchema2Id, getS3Key, getObjId +from h5json.shape_util import isNullSpace, getShapeDims, getRank, getMaxDims +from h5json.shape_util import isExtensible, getShapeClass +from h5json.dset_util import getChunkDims, getDatasetLayout, getDatasetLayoutClass +from h5json.query_util import arrayQuery +from h5json import selections + +from .util.nodeUtil import getDataNodeUrl +from .util.dsetUtil import get_slices from .util.chunkUtil import getChunkCoordinate, getChunkIndex, getChunkSuffix from .util.chunkUtil import getNumChunks, getChunkIds, getChunkId from .util.chunkUtil import getChunkCoverage, getDataCoverage -from .util.chunkUtil import getQueryDtype, get_chunktable_dims -from .util.hdf5dtype import createDataType, getItemSize -from .util.httpUtil import http_delete, http_put -from .util.idUtil import getDataNodeUrl, isSchema2Id, getS3Key, getObjId +from .util.chunkUtil import get_chunktable_dims +from .util.httpUtil import http_delete, http_put, request_read from .util.rangegetUtil import getHyperChunkFactors from .util.storUtil import getStorKeys @@ -115,26 +124,19 @@ async def getChunkLocations(app, dset_id, dset_json, chunkinfo_map, chunk_ids, b log.debug(msg) return - chunk_dims = None - if "layout" in dset_json: - dset_layout = dset_json["layout"] - log.debug(f"dset_json layout: {dset_layout}") - if "dims" in dset_layout: - chunk_dims = dset_layout["dims"] + chunk_dims = getChunkDims(dset_json) if chunk_dims is None: msg = "no chunk dimensions set in dataset layout" log.error(msg) raise HTTPInternalServerError() - datashape = dset_json["shape"] datatype = dset_json["type"] if isNullSpace(dset_json): log.error("H5S_NULL shape class used with reference chunk layout") raise HTTPInternalServerError() - dims = getShapeDims(datashape) - rank = len(dims) - # chunk_ids = list(chunkinfo_map.keys()) - # chunk_ids.sort() + dims = getShapeDims(dset_json) + rank = getRank(dset_json) + num_chunks = len(chunk_ids) msg = f"getChunkLocations for dset: {dset_id} bucket: {bucket} " msg += f"rank: {rank} num chunk_ids: {num_chunks}" @@ -227,19 +229,19 @@ def getChunkItem(chunkid): # get state for dataset from DN. chunktable_json = await getDsetJson(app, chunktable_id, bucket=bucket) # log.debug(f"chunktable_json: {chunktable_json}") - chunktable_dims = getShapeDims(chunktable_json["shape"]) - chunktable_layout = chunktable_json["layout"] - if chunktable_layout.get("class") == "H5D_CHUNKED_REF_INDIRECT": - # We don't support recursive chunked_ref_indirect classes - msg = "chunktable layout: H5D_CHUNKED_REF_INDIRECT is invalid" - log.warn(msg) - raise HTTPBadRequest(reason=msg) + chunktable_dims = getShapeDims(chunktable_json) if len(chunktable_dims) != rank: msg = "Rank of chunktable should be same as the dataset" log.warn(msg) raise HTTPBadRequest(reason=msg) + if getDatasetLayoutClass(chunktable_json) == "H5D_CHUNKED_REF_INDIRECT": + # We don't support recursive chunked_ref_indirect classes + msg = "chunktable layout: H5D_CHUNKED_REF_INDIRECT is invalid" + log.warn(msg) + raise HTTPBadRequest(reason=msg) + # convert the list of chunk_ids into a set of points to query in # the chunk table log.debug(f"datashape: {dims}") @@ -364,7 +366,7 @@ def get_chunk_selections(chunk_map, chunk_ids, slices, dset_json): log.debug("no slices set, returning") return # nothing to do log.debug(f"slices: {slices}") - layout = getChunkLayout(dset_json) + layout = getChunkDims(dset_json) for chunk_id in chunk_ids: if chunk_id in chunk_map: item = chunk_map[chunk_id] @@ -381,48 +383,20 @@ def get_chunk_selections(chunk_map, chunk_ids, slices, dset_json): item["data_sel"] = data_sel -def getParser(query, dtype): - """ get query BooleanParser. If query contains variables that - arent' part of the data type, throw a HTTPBadRequest exception. """ - - # separate out the where clause if any - if query.startswith("where"): - where_in = query - expr = None - else: - n = query.find(" where ") - if n > 0: - where_in = query[(n + 1):] - expr = query[:n] - else: - where_in = None - expr = query - - if where_in: - log.debug(f"got where in clause: {where_in}") - # TBD: do full syntax check on this - - if not expr: - # just a where clause - return None - +def validateQuery(query, dtype): + """ Validate that the given query string is syntactically valid and + only references fields present in dtype, by running it against a + trivial array with h5json.query_util.arrayQuery (the same engine + used to actually execute the query at the DN). Raises HTTPBadRequest + if the query is malformed or references an invalid field. """ + dummy_arr = np.zeros((1,), dtype=dtype) try: - parser = BooleanParser(expr) - except Exception: - msg = f"query: {expr} is not valid" + arrayQuery(query, dummy_arr) + except (TypeError, ValueError) as e: + msg = f"query: {query} is not valid: {e}" log.warn(msg) raise HTTPBadRequest(reason=msg) - field_names = set(dtype.names) - variables = parser.getVariables() - for variable in variables: - if variable not in field_names: - msg = f"query variable {variable} not valid" - log.warn(msg) - raise HTTPBadRequest(reason=msg) - - return parser - async def getSelectionData( app, @@ -433,8 +407,10 @@ async def getSelectionData( points=None, query=None, query_update=None, + query_indices=False, bucket=None, - limit=0 + limit=0, + only_chunk_ids=None, ): """Read selected slices and return numpy array""" log.debug("getSelectionData") @@ -442,20 +418,25 @@ async def getSelectionData( log.error("getSelectionData - expected either slices or points to be set") raise HTTPInternalServerError() - layout = getChunkLayout(dset_json) + layout = getChunkDims(dset_json) chunkinfo = {} if slices is not None: - num_chunks = getNumChunks(slices, layout) - log.debug(f"num_chunks: {num_chunks}") + if only_chunk_ids is not None: + # caller already knows exactly which chunk(s) to target - + # used by doQueryUpdate to process one chunk at a time + chunk_ids = only_chunk_ids + else: + num_chunks = getNumChunks(slices, layout) + log.debug(f"num_chunks: {num_chunks}") - max_chunks = int(config.get("max_chunks_per_request", default=1000)) - if num_chunks > max_chunks: - msg = f"num_chunks over {max_chunks} limit, but will attempt to fetch with crawler" - log.warn(msg) + max_chunks = int(config.get("max_chunks_per_request", default=1000)) + if num_chunks > max_chunks: + msg = f"num_chunks over {max_chunks} limit, but will attempt to fetch with crawler" + log.warn(msg) - chunk_ids = getChunkIds(dset_id, slices, layout) + chunk_ids = getChunkIds(dset_id, slices, layout) else: # points - already checked it is not None num_points = len(points) @@ -505,6 +486,7 @@ async def getSelectionData( points=points, query=query, query_update=query_update, + query_indices=query_indices, limit=limit, chunk_map=chunkinfo, bucket=bucket, @@ -513,6 +495,108 @@ async def getSelectionData( return arr +def _globalIndexToChunkId(dset_id, coord, layout): + """ convert a (rank,) global index coordinate array into the id of + the chunk that contains it """ + rank = len(layout) + if rank == 1: + point = int(coord[0]) + else: + point = tuple(int(c) for c in coord) + return getChunkId(dset_id, point, layout) + + +async def doQueryUpdate( + app, + dset_id, + dset_json, + slices=None, + query=None, + query_update=None, + bucket=None, + limit=0, +): + """ Perform a query-based conditional update (PUT .../value with a + query param), correctly handling `limit` when the matching elements + span more than one chunk. + + Each chunk's own query-update is only aware of its own matches, so + naively sending the same `limit` to every relevant chunk (as a plain + getSelectionData call would) can update more than `limit` elements in + total. Instead: + 1. Find the (up to `limit`) global indices that would be updated, + without modifying anything yet (a read-only query_indices pass). + 2. If there turn out to be fewer matches than `limit` (the limit + isn't actually binding), or no limit was given at all, process + every relevant chunk in the normal (parallel, unlimited) way. + 3. Otherwise, process chunks one at a time, in dataset order, + decrementing the remaining budget by the number of elements + *actually* updated in each chunk (not the number predicted in + step 1) - the data may have changed between the read-only pass + and the update, so this stays correct under concurrent writes. + Stops as soon as the budget is exhausted (often just the first + chunk). + + Returns the array of global indices that were actually updated. """ + if not limit or limit <= 0: + # no limit - nothing to coordinate, process all chunks at once + return await getSelectionData( + app, dset_id, dset_json, slices=slices, query=query, + query_update=query_update, bucket=bucket, limit=0, + ) + + target_indices = await getSelectionData( + app, dset_id, dset_json, slices=slices, query=query, + query_indices=True, bucket=bucket, limit=limit, + ) + + if len(target_indices) == 0: + # no matches at all + return target_indices + + if len(target_indices) < limit: + # fewer matches than the limit overall - limit isn't actually + # binding, so there's no over-application risk in processing + # every relevant chunk in one (parallel) pass + return await getSelectionData( + app, dset_id, dset_json, slices=slices, query=query, + query_update=query_update, bucket=bucket, limit=0, + ) + + # limit may be binding and could span multiple chunks - process + # chunks one at a time, in order, so `limit` is respected globally + layout = getChunkDims(dset_json) + ordered_chunk_ids = [] + seen_chunk_ids = set() + for coord in target_indices: + chunk_id = _globalIndexToChunkId(dset_id, coord, layout) + if chunk_id not in seen_chunk_ids: + seen_chunk_ids.add(chunk_id) + ordered_chunk_ids.append(chunk_id) + + remaining = limit + results = [] + for chunk_id in ordered_chunk_ids: + if remaining <= 0: + break + chunk_result = await getSelectionData( + app, dset_id, dset_json, slices=slices, query=query, + query_update=query_update, bucket=bucket, limit=remaining, + only_chunk_ids=[chunk_id], + ) + if len(chunk_result) > 0: + results.append(chunk_result) + # use the actual count returned (not the step-1 prediction), + # since the data may have been modified concurrently + remaining -= len(chunk_result) + + if not results: + return target_indices[:0] + if len(results) == 1: + return results[0] + return np.concatenate(results, axis=0) + + async def doReadSelection( app, chunk_ids, @@ -522,6 +606,7 @@ async def doReadSelection( points=None, query=None, query_update=None, + query_indices=False, chunk_map=None, bucket=None, limit=0, @@ -546,14 +631,18 @@ async def doReadSelection( select_dtype = dset_dtype if query is None: query_dtype = None + elif query_update is not None or query_indices: + # PUT_Chunk's query-update handling, and GET_Chunk's query_indices + # mode, both return the global dataset indices of matching + # elements, as (n, rank) coordinate tuples + log.debug(f"query: {query} limit: {limit} query_update: {query_update}") + query_dtype = np.dtype("i8") + query_rank = getRank(dset_json) else: + # GET_Chunk's query handling (h5json.query_util.arrayQuery) returns + # the matching values themselves, typed as select_dtype log.debug(f"query: {query} limit: {limit}") - query_dtype = getQueryDtype(select_dtype) - if query_dtype: - if len(query_dtype) < 10: - log.debug(f"query_dtype: {query_dtype}") - else: - log.debug(f"query_dtype {len(query_dtype)}") + query_dtype = select_dtype # create array to hold response data arr = None @@ -566,7 +655,7 @@ async def doReadSelection( np_shape = None elif slices is not None: log.debug(f"get np_shape for slices: {slices}") - np_shape = getSelectionShape(slices) + np_shape = slices.mshape else: log.error("doReadSelection - expected points or slices to be set") raise HTTPInternalServerError() @@ -605,6 +694,7 @@ async def doReadSelection( slices=slices, query=query, query_update=query_update, + query_indices=query_indices, limit=limit, arr=arr, select_dtype=select_dtype, @@ -629,7 +719,10 @@ async def doReadSelection( nrows = limit else: nrows = crawler._hits - arr = np.empty((nrows,), dtype=query_dtype) + if query_update is not None or query_indices: + arr = np.empty((nrows, query_rank), dtype=query_dtype) + else: + arr = np.empty((nrows,), dtype=query_dtype) start = 0 for chunkid in chunk_ids: if chunkid not in chunk_map: @@ -762,9 +855,8 @@ async def getAllocatedChunkIds(app, dset_id, bucket=None): async def extendShape(app, dset_json, nelements, axis=0, bucket=None): """ extend the shape of the dataset by nelements along given axis """ dset_id = dset_json["id"] - datashape = dset_json["shape"] - dims = getShapeDims(datashape) - rank = len(dims) + dims = getShapeDims(dset_json) + rank = getRank(dset_json) log.info(f"extendShape of {dset_id} dims: {dims} by {nelements} on axis: {axis}") # do some sanity checks here if rank == 0: @@ -782,17 +874,23 @@ async def extendShape(app, dset_json, nelements, axis=0, bucket=None): if bucket: params["bucket"] = bucket selection = None + new_dims = None try: shape_rsp = await http_put(app, req, data=body, params=params) log.info(f"got shape put rsp: {shape_rsp}") if "selection" in shape_rsp: selection = shape_rsp["selection"] + if "dims" in shape_rsp: + new_dims = tuple(shape_rsp["dims"]) except HTTPConflict: log.warn("got 409 extending dataspace for PUT value") raise - if not selection: + if selection is None: log.error("expected to get selection in PUT shape response") raise HTTPInternalServerError() + if new_dims is None: + log.error("expected dims in PUT shape response") + raise HTTPInternalServerError() # selection should be in the format [:,n:m,:]. # extract n and m and use it to update the slice for the @@ -827,7 +925,9 @@ async def extendShape(app, dset_json, nelements, axis=0, bucket=None): slices.append(s) log.debug(f"extendShape returning slices: {slices}") - return slices + sel = selections.select(new_dims, tuple(slices)) + log.debug(f"returning selection: {sel}") + return sel async def reduceShape(app, dset_json, shape_update, bucket=None): @@ -839,11 +939,10 @@ async def reduceShape(app, dset_json, shape_update, bucket=None): log.info(f"reduceShape for {dset_id} to {shape_update}") # get the current shape dims - shape_orig = dset_json["shape"] - if shape_orig["class"] != "H5S_SIMPLE": + if getShapeClass(dset_json) != "H5S_SIMPLE": raise ValueError("reduceShape can only be called on simple datasets") - dims = shape_orig["dims"] - rank = len(dims) + dims = getShapeDims(dset_json) + rank = getRank(dset_json) # get the fill value arr = getFillValue(dset_json) @@ -855,7 +954,12 @@ async def reduceShape(app, dset_json, shape_update, bucket=None): arr = np.zeros([1], dtype=dt, order="C") # and the chunk layout - layout = tuple(getChunkLayout(dset_json)) + layout = getChunkDims(dset_json) + + if not layout: + msg = f"no layout found for {dset_id}" + log.error(msg) + raise HTTPInternalServerError() log.debug(f"got layout: {layout}") # get all chunk ids for chunks that have been allocated @@ -917,12 +1021,14 @@ async def reduceShape(app, dset_json, shape_update, bucket=None): log.debug(f"update {update_element_count} elements for dim {n}") + selection = selections.select(tuple(dims), tuple(slices)) + crawler = ChunkCrawler( app, update_ids, dset_json=dset_json, bucket=bucket, - slices=slices, + slices=selection, arr=arr, action="write_chunk_hyperslab", ) @@ -954,23 +1060,17 @@ async def updateShape(app, dset_json, shape_update, bucket=None): """ Update the current dataset shape """ dset_id = dset_json["id"] - shape_orig = dset_json["shape"] - log.info(f"updateShape dset: {dset_id}: {shape_update}") + log.info(f"updateShape dset: {dset_id}") # verify that the extend request is valid - if shape_orig["class"] != "H5S_SIMPLE": - msg = "Unable to extend shape of datasets who are not H5S_SIMPLE" - log.warn(msg) - raise HTTPBadRequest(reason=msg) - - if "maxdims" not in shape_orig: + if not isExtensible(dset_json): msg = "Dataset is not extensible" log.warn(msg) raise HTTPBadRequest(reason=msg) - dims = shape_orig["dims"] - rank = len(dims) - maxdims = shape_orig["maxdims"] + dims = getShapeDims(dset_json) + rank = getRank(dset_json) + maxdims = getMaxDims(dset_json) log.debug(f"dims: {dims}, maxdims: {maxdims}") if shape_update and len(shape_update) != rank: msg = "Extent of update shape request does not match dataset sahpe" @@ -992,7 +1092,7 @@ async def updateShape(app, dset_json, shape_update, bucket=None): raise HTTPBadRequest(reason=msg) decreasing_dims.append(i) elif shape_update[i] > dims[i]: - if maxdims[i] != 0 and shape_update[i] > maxdims[i]: + if maxdims[i] not in (0, "H5S_UNLIMITED") and shape_update[i] > maxdims[i]: msg = "Extension dimension can not be extended past max extent" log.warn(msg) raise HTTPConflict() @@ -1053,3 +1153,184 @@ async def deleteAllChunks(app, dset_id, bucket=None): await removeChunks(app, chunk_ids, bucket=bucket) else: log.info(f"deleteAllChunks for {dset_id} - no chunks need deletion") + + +async def doPointWrite(app, + request, + points=None, + data=None, + dset_json=None, + bucket=None + ): + """ write the given points to the dataset """ + + num_points = len(points) + log.debug(f"doPointWrite - num_points: {num_points}") + dset_id = dset_json["id"] + layout = getChunkDims(dset_json) + dims = getShapeDims(dset_json) + rank = getRank(dset_json) + + chunk_dict = {} # chunk ids to list of points in chunk + + for pt_indx in range(num_points): + if rank == 1: + point = int(points[pt_indx]) + else: + point_tuple = points[pt_indx] + point = [] + for i in range(len(point_tuple)): + point.append(int(point_tuple[i])) + if rank == 1: + if point < 0 or point >= dims[0]: + msg = f"PUT Value point: {point} is not within the " + msg += "bounds of the dataset" + log.warn(msg) + raise HTTPBadRequest(reason=msg) + else: + if len(point) != rank: + msg = "PUT Value point value did not match dataset rank" + log.warn(msg) + raise HTTPBadRequest(reason=msg) + for i in range(rank): + if point[i] < 0 or point[i] >= dims[i]: + msg = f"PUT Value point: {point} is not within the " + msg += "bounds of the dataset" + log.warn(msg) + raise HTTPBadRequest(reason=msg) + chunk_id = getChunkId(dset_id, point, layout) + # get the pt_indx element from the input data + value = data[pt_indx] + if chunk_id not in chunk_dict: + point_list = [point, ] + point_data = [value, ] + chunk_dict[chunk_id] = {"indices": point_list, "points": point_data} + else: + item = chunk_dict[chunk_id] + point_list = item["indices"] + point_list.append(point) + point_data = item["points"] + point_data.append(value) + + num_chunks = len(chunk_dict) + log.debug(f"num_chunks: {num_chunks}") + max_chunks = int(config.get("max_chunks_per_request", default=1000)) + if num_chunks > max_chunks: + msg = f"PUT value request with more than {max_chunks} chunks" + log.warn(msg) + + chunk_ids = list(chunk_dict.keys()) + chunk_ids.sort() + + crawler = ChunkCrawler( + app, + chunk_ids, + dset_json=dset_json, + bucket=bucket, + points=chunk_dict, + action="write_point_sel", + ) + await crawler.crawl() + + crawler_status = crawler.get_status() + + if crawler_status not in (200, 201): + msg = f"doPointWrite raising HTTPInternalServerError for status: {crawler_status}" + log.error(msg) + raise HTTPInternalServerError() + else: + log.info("doPointWrite success") + + +async def doHyperslabWrite(app, + request, + page_number=0, + page=None, + data=None, + dset_json=None, + select_dtype=None, + bucket=None + ): + """ write the given page selection to the dataset """ + dset_id = dset_json["id"] + log.info(f"doHyperslabWrite on {dset_id} - page: {page_number} dset_json: {dset_json}") + type_json = dset_json["type"] + + if select_dtype is not None: + item_size = getDtypeItemSize(select_dtype) + else: + item_size = getItemSize(type_json) + if item_size == "H5T_VARIABLE" and data is None: + msg = "unexpected call to doHyperslabWrite for variable length data" + log.error(msg) + raise HTTPInternalServerError() + + layout = getChunkDims(dset_json) + log.debug(f"getNumChunks(page={page}, layout={layout})") + num_chunks = getNumChunks(page, layout) + log.debug(f"num_chunks: {num_chunks}") + max_chunks = int(config.get("max_chunks_per_request", default=1000)) + if num_chunks > max_chunks: + msg = f"PUT value chunk count: {num_chunks} exceeds max_chunks: {max_chunks}" + log.warn(msg) + select_shape = page.mshape + log.debug(f"got select_shape: {select_shape} for page: {page_number}") + + if data is None: + num_bytes = math.prod(select_shape) * item_size + log.debug(f"reading {num_bytes} from request stream") + # read page of data from input stream + try: + page_bytes = await request_read(request, count=num_bytes) + except HTTPRequestEntityTooLarge as tle: + msg = "Got HTTPRequestEntityTooLarge exception during " + msg += f"binary read: {tle}) for page: {page_number}" + log.warn(msg) + raise # re-throw + except IncompleteReadError as ire: + msg = "Got asyncio.IncompleteReadError during binary " + msg += f"read: {ire} for page: {page_number}" + log.warn(msg) + raise HTTPBadRequest(reason=msg) + log.debug(f"read {len(page_bytes)} for page: {page_number}") + try: + arr = bytesToArray(page_bytes, select_dtype, select_shape) + except ValueError as ve: + msg = f"bytesToArray value error for page: {page_number}: {ve}" + log.warn(msg) + raise HTTPBadRequest(reason=msg) + else: + arr = data # use array provided to function + + try: + chunk_ids = getChunkIds(dset_id, page, layout) + except ValueError: + log.warn("getChunkIds failed") + raise HTTPInternalServerError() + if len(chunk_ids) < 10: + log.debug(f"chunk_ids: {chunk_ids}") + else: + log.debug(f"chunk_ids: {chunk_ids[:10]} ...") + if len(chunk_ids) > max_chunks: + msg = f"got {len(chunk_ids)} for page: {page_number}. max_chunks: {max_chunks}" + log.warn(msg) + + crawler = ChunkCrawler( + app, + chunk_ids, + dset_json=dset_json, + bucket=bucket, + slices=page, + arr=arr, + action="write_chunk_hyperslab", + ) + await crawler.crawl() + + crawler_status = crawler.get_status() + + if crawler_status not in (200, 201): + msg = f"crawler failed for page: {page_number} with status: {crawler_status}" + log.error(msg) + raise HTTPInternalServerError() + else: + log.info("crawler write_chunk_hyperslab successful") diff --git a/hsds/dset_sn.py b/hsds/dset_sn.py index 38e1156a..4494d387 100755 --- a/hsds/dset_sn.py +++ b/hsds/dset_sn.py @@ -14,226 +14,33 @@ # handles dataset requests # -import math from json import JSONDecodeError -from aiohttp.web_exceptions import HTTPBadRequest, HTTPNotFound +from aiohttp.web_exceptions import HTTPBadRequest, HTTPNotFound, HTTPInternalServerError + +from h5json.hdf5dtype import createDataType +from h5json.array_util import getNumElements, jsonToArray +from h5json.objid import isValidUuid, isSchema2Id +from h5json.shape_util import getShapeDims, isNullSpace, isScalar +from h5json.dset_util import getChunkDims, getDatasetLayoutClass +from h5json import selections from .util.httpUtil import getHref, respJsonAssemble from .util.httpUtil import jsonResponse, getBooleanParam -from .util.idUtil import isValidUuid, isSchema2Id -from .util.dsetUtil import getPreviewQuery, getFilterItem, getShapeDims -from .util.arrayUtil import getNumElements, getNumpyValue -from .util.chunkUtil import getChunkSize, guessChunk, expandChunk, shrinkChunk -from .util.chunkUtil import getContiguousLayout +from .util.chunkUtil import getChunkIds +from .util.dsetUtil import getPreviewQuery from .util.authUtil import getUserPasswordFromRequest, aclCheck from .util.authUtil import validateUserPassword from .util.domainUtil import getDomainFromRequest, getPathForDomain, isValidDomain from .util.domainUtil import getBucketForDomain, verifyRoot -from .util.storUtil import getSupportedFilters -from .util.hdf5dtype import validateTypeItem, createDataType, getBaseTypeJson -from .util.hdf5dtype import getItemSize -from .util.linkUtil import validateLinkName from .servicenode_lib import getDomainJson, getObjectJson, getDsetJson, getPathForObjectId from .servicenode_lib import getObjectIdByPath, validateAction, getRootInfo -from .servicenode_lib import createObject, createObjectByPath, deleteObject +from .servicenode_lib import getDatasetCreateArgs, createDataset, deleteObject from .dset_lib import updateShape, deleteAllChunks -from . import config +from .post_crawl import createDatasets +from .domain_crawl import DomainCrawler from . import hsds_logger as log -async def validateChunkLayout(app, shape_json, item_size, layout, bucket=None): - """ - Use chunk layout given in the creationPropertiesList (if defined and - layout is valid). - Return chunk_layout_json - """ - - rank = 0 - space_dims = None - chunk_dims = None - max_dims = None - - if "dims" in shape_json: - space_dims = shape_json["dims"] - rank = len(space_dims) - - if "maxdims" in shape_json: - max_dims = shape_json["maxdims"] - if "dims" in layout: - chunk_dims = layout["dims"] - - if chunk_dims: - # validate that the chunk_dims are valid and correlates with the - # dataset shape - if isinstance(chunk_dims, int): - chunk_dims = [ - chunk_dims, - ] # promote to array - if len(chunk_dims) != rank: - msg = "Layout rank does not match shape rank" - log.warn(msg) - raise HTTPBadRequest(reason=msg) - for i in range(rank): - dim_extent = space_dims[i] - chunk_extent = chunk_dims[i] - if not isinstance(chunk_extent, int): - msg = "Layout dims must be integer or integer array" - log.warn(msg) - raise HTTPBadRequest(reason=msg) - if chunk_extent <= 0: - msg = "Invalid layout value" - log.warn(msg) - raise HTTPBadRequest(reason=msg) - if max_dims is None: - if chunk_extent > dim_extent: - msg = "Invalid layout value" - log.warn(msg) - raise HTTPBadRequest(reason=msg) - elif max_dims[i] != 0: - if chunk_extent > max_dims[i]: - msg = "Invalid layout value for extensible dimension" - log.warn(msg) - raise HTTPBadRequest(reason=msg) - else: - pass # allow any positive value for unlimited dimensions - - if "class" not in layout: - msg = "class key not found in layout for creation property list" - log.warn(msg) - raise HTTPBadRequest(reason=msg) - - layout_class = layout["class"] - - if layout_class == "H5D_CONTIGUOUS_REF": - # reference to a dataset in a traditional HDF5 files with - # contigious storage - if item_size == "H5T_VARIABLE": - # can't be used with variable types... - msg = "Datsets with variable types cannot be used with " - msg += "reference layouts" - log.warn(msg) - raise HTTPBadRequest(reason=msg) - if "file_uri" not in layout: - # needed for H5D_CONTIGUOUS_REF - msg = "'file_uri' key must be provided for " - msg += "H5D_CONTIGUOUS_REF layout" - log.warn(msg) - raise HTTPBadRequest(reason=msg) - if "offset" not in layout: - # needed for H5D_CONTIGUOUS_REF - msg = "'offset' key must be provided for " - msg += "H5D_CONTIGUOUS_REF layout" - log.warn(msg) - raise HTTPBadRequest(reason=msg) - if "size" not in layout: - # needed for H5D_CONTIGUOUS_REF - msg = "'size' key must be provided for " - msg += "H5D_CONTIGUOUS_REF layout" - log.warn(msg) - raise HTTPBadRequest(reason=msg) - if "dims" in layout: - # used defined chunk layout not allowed for H5D_CONTIGUOUS_REF - msg = "'dims' key can not be provided for " - msg += "H5D_CONTIGUOUS_REF layout" - log.warn(msg) - raise HTTPBadRequest(reason=msg) - elif layout_class == "H5D_CHUNKED_REF": - # reference to a dataset in a traditional HDF5 files with - # chunked storage - if item_size == "H5T_VARIABLE": - # can't be used with variable types.. - msg = "Datsets with variable types cannot be used with " - msg += "reference layouts" - log.warn(msg) - raise HTTPBadRequest(reason=msg) - if "file_uri" not in layout: - # needed for H5D_CHUNKED_REF - msg = "'file_uri' key must be provided for " - msg += "H5D_CHUNKED_REF layout" - log.warn(msg) - raise HTTPBadRequest(reason=msg) - if "dims" not in layout: - # needed for H5D_CHUNKED_REF - msg = "'dimns' key must be provided for " - msg += "H5D_CHUNKED_REF layout" - log.warn(msg) - raise HTTPBadRequest(reason=msg) - if "chunks" not in layout: - msg = "'chunks' key must be provided for " - msg += "H5D_CHUNKED_REF layout" - log.warn(msg) - raise HTTPBadRequest(reason=msg) - elif layout_class == "H5D_CHUNKED_REF_INDIRECT": - # reference to a dataset in a traditional HDF5 files with chunked - # storage using an auxillary dataset - if item_size == "H5T_VARIABLE": - # can't be used with variable types.. - msg = "Datsets with variable types cannot be used with " - msg += "reference layouts" - log.warn(msg) - raise HTTPBadRequest(reason=msg) - if "dims" not in layout: - # needed for H5D_CHUNKED_REF_INDIRECT - msg = "'dimns' key must be provided for " - msg += "H5D_CHUNKED_REF_INDIRECT layout" - log.warn(msg) - raise HTTPBadRequest(reason=msg) - if "chunk_table" not in layout: - msg = "'chunk_table' key must be provided for " - msg += "H5D_CHUNKED_REF_INDIRECT layout" - log.warn(msg) - raise HTTPBadRequest(reason=msg) - chunktable_id = layout["chunk_table"] - if not isValidUuid(chunktable_id, "Dataset"): - msg = f"Invalid chunk table id: {chunktable_id}" - log.warn(msg) - raise HTTPBadRequest(reason=msg) - # verify the chunk table exists and is of reasonable shape - try: - chunktable_json = await getDsetJson(app, chunktable_id, bucket=bucket) - except HTTPNotFound: - msg = f"chunk table id: {chunktable_id} not found" - log.warn(msg) - raise - chunktable_shape = chunktable_json["shape"] - if chunktable_shape["class"] == "H5S_NULL": - msg = "Null space datasets can not be used as chunk tables" - log.warn(msg) - raise HTTPBadRequest(reason=msg) - - chunktable_dims = getShapeDims(chunktable_shape) - if len(chunktable_dims) != len(space_dims): - msg = "Chunk table rank must be same as dataspace rank" - log.warn(msg) - raise HTTPBadRequest(reason=msg) - elif layout_class == "H5D_CHUNKED": - if "dims" not in layout: - msg = "dims key not found in layout for creation property list" - log.warn(msg) - raise HTTPBadRequest(reason=msg) - if shape_json["class"] != "H5S_SIMPLE": - msg = "Bad Request: chunked layout not valid with shape class: " - msg += f"{shape_json['class']}" - log.warn(msg) - raise HTTPBadRequest(reason=msg) - elif layout_class == "H5D_CONTIGUOUS": - if "dims" in layout: - msg = "dims key found in layout for creation property list " - msg += "for H5D_CONTIGUOUS storage class" - log.warn(msg) - raise HTTPBadRequest(reason=msg) - elif layout_class == "H5D_COMPACT": - if "dims" in layout: - msg = "dims key found in layout for creation property list " - msg += "for H5D_COMPACT storage class" - log.warn(msg) - raise HTTPBadRequest(reason=msg) - else: - msg = f"Unexpected layout: {layout_class}" - log.warn(msg) - raise HTTPBadRequest(reason=msg) - - async def getDatasetDetails(app, dset_id, root_id, bucket=None): """Get extra information about the given dataset""" # Gather additional info on the domain @@ -282,7 +89,7 @@ async def GET_Dataset(request): include_attrs = True if dset_id: - if not isValidUuid(dset_id, "Dataset"): + if not isValidUuid(dset_id, obj_class="datasets"): msg = f"Invalid dataset id: {dset_id}" log.warn(msg) raise HTTPBadRequest(reason=msg) @@ -293,7 +100,7 @@ async def GET_Dataset(request): group_id = None if "grpid" in params: group_id = params["grpid"] - if not isValidUuid(group_id, "Group"): + if not isValidUuid(group_id, obj_class="groups"): msg = f"Invalid parent group id: {group_id}" log.warn(msg) raise HTTPBadRequest(reason=msg) @@ -337,7 +144,7 @@ async def GET_Dataset(request): # throws 404 if not found kwargs = {"bucket": bucket, "domain": domain} dset_id, domain, _ = await getObjectIdByPath(app, group_id, h5path, **kwargs) - if not isValidUuid(dset_id, "Dataset"): + if not isValidUuid(dset_id, obj_class="datasets"): msg = f"No dataset exist with the path: {h5path}" log.warn(msg) raise HTTPNotFound() @@ -378,11 +185,13 @@ async def GET_Dataset(request): # provide a value link if the dataset is relatively small, # otherwise create a preview link that shows a limited number of # data values - dset_shape = dset_json["shape"] - if dset_shape["class"] != "H5S_NULL": - count = 1 - if dset_shape["class"] == "H5S_SIMPLE": - dims = dset_shape["dims"] + if isNullSpace(dset_json): + pass # no value link for null space datasets + else: + if isScalar(dset_json): + count = 1 + else: + dims = getShapeDims(dset_json) count = getNumElements(dims) if count <= 100: # small number of values, provide link to entire dataset @@ -390,7 +199,7 @@ async def GET_Dataset(request): hrefs.append({"rel": "data", "href": href}) else: # large number of values, create preview link - previewQuery = getPreviewQuery(dset_shape["dims"]) + previewQuery = getPreviewQuery(getShapeDims(dset_json)) kwargs = {"query": previewQuery} href = getHref(request, dset_uri + "/value", **kwargs) hrefs.append({"rel": "preview", "href": href}) @@ -425,7 +234,7 @@ async def GET_DatasetType(request): msg = "Missing dataset id" log.warn(msg) raise HTTPBadRequest(reason=msg) - if not isValidUuid(dset_id, "Dataset"): + if not isValidUuid(dset_id, obj_class="datasets"): msg = f"Invalid dataset id: {dset_id}" log.warn(msg) raise HTTPBadRequest(reason=msg) @@ -477,7 +286,7 @@ async def GET_DatasetShape(request): msg = "Missing dataset id" log.warn(msg) raise HTTPBadRequest(reason=msg) - if not isValidUuid(dset_id, "Dataset"): + if not isValidUuid(dset_id, obj_class="datasets"): msg = f"Invalid dataset id: {dset_id}" log.warn(msg) raise HTTPBadRequest(reason=msg) @@ -534,7 +343,7 @@ async def PUT_DatasetShape(request): msg = "Missing dataset id" log.warn(msg) raise HTTPBadRequest(reason=msg) - if not isValidUuid(dset_id, "Dataset"): + if not isValidUuid(dset_id, obj_class="datasets"): msg = f"Invalid dataset id: {dset_id}" log.warn(msg) raise HTTPBadRequest(reason=msg) @@ -670,7 +479,7 @@ async def POST_Dataset(request): log.warn(msg) raise HTTPBadRequest(reason=msg) - log.debug(f"got body: {body}") + log.debug(f"POST_Dataset got body: {body}") # get domain, check authorization domain = getDomainFromRequest(request) if not isValidDomain(domain): @@ -688,472 +497,218 @@ async def POST_Dataset(request): verifyRoot(domain_json) - # - # validate type input - # - if "type" not in body: - msg = "POST Dataset has no type key in body" - log.warn(msg) - raise HTTPBadRequest(reason=msg) + # allow parent group creation or not + implicit = getBooleanParam(params, "implicit") - datatype = body["type"] - log.debug(f"got datatype: {datatype}") - if isinstance(datatype, str) and datatype.startswith("t-"): - # Committed type - fetch type json from DN - ctype_id = datatype - log.debug(f"got ctypeid: {ctype_id}") - ctype_json = await getObjectJson(app, ctype_id, bucket=bucket) - log.debug(f"ctype: {ctype_json}") - if ctype_json["root"] != root_id: - msg = "Referenced committed datatype must belong in same domain" - log.warn(msg) - raise HTTPBadRequest(reason=msg) - datatype = ctype_json["type"] - # add the ctype_id to type type - datatype["id"] = ctype_id - elif isinstance(datatype, str): - try: - # convert predefined type string (e.g. "H5T_STD_I32LE") to - # corresponding json representation - datatype = getBaseTypeJson(datatype) - log.debug(f"got datatype: {datatype}") - except TypeError: - msg = "POST Dataset with invalid predefined type" - log.warn(msg) - raise HTTPBadRequest(reason=msg) + post_rsp = None - try: - validateTypeItem(datatype) - except KeyError as ke: - msg = f"KeyError creating type: {ke}" - log.warn(msg) - raise HTTPBadRequest(reason=msg) - except TypeError as te: - msg = f"TypeError creating type: {te}" - log.warn(msg) - raise HTTPBadRequest(reason=msg) - except ValueError as ve: - msg = f"ValueError creating type: {ve}" - log.warn(msg) - raise HTTPBadRequest(reason=msg) + datatype_json = None + init_values = [] # value initializer for each object - item_size = getItemSize(datatype) + def _updateInitValuesList(kwargs): + # remove value key from kwargs and append + # to init_values list + if "value" in kwargs: + init_values.append(kwargs["value"]) + log.debug(f"init value appended: {kwargs['value']}") + del kwargs["value"] + else: + # add a placeholder + init_values.append(None) # - # Validate shape input + # handle case of committed type input # - dims = None - shape_json = {} - rank = 0 - chunk_size = None - - if "shape" not in body: - shape_json["class"] = "H5S_SCALAR" - else: - shape = body["shape"] - log.debug(f"got shape: {shape}") - if isinstance(shape, int): - shape_json["class"] = "H5S_SIMPLE" - dims = [shape, ] - shape_json["dims"] = dims - rank = 1 - elif isinstance(shape, str): - # only valid string value is H5S_NULL or H5S_SCALAR - if shape == "H5S_NULL": - shape_json["class"] = "H5S_NULL" - elif shape == "H5S_SCALAR": - shape_json["class"] = "H5S_SCALAR" - else: - msg = "POST Datset with invalid shape value" + if isinstance(body, dict) and "type" in body: + + body_type = body["type"] + log.debug(f"got datatype: {body_type}") + if isinstance(body_type, str) and body_type.startswith("t-"): + ctype_id = body_type + # Committed type - fetch type json from DN + log.debug(f"got ctype_id: {ctype_id}") + ctype_json = await getObjectJson(app, ctype_id, bucket=bucket) + log.debug(f"ctype: {ctype_json}") + if ctype_json["root"] != root_id: + msg = "Referenced committed datatype must belong in same domain" log.warn(msg) raise HTTPBadRequest(reason=msg) - elif isinstance(shape, list): - if len(shape) == 0: - shape_json["class"] = "H5S_SCALAR" - else: - shape_json["class"] = "H5S_SIMPLE" - shape_json["dims"] = shape - dims = shape - rank = len(dims) + datatype_json = ctype_json["type"] + # add the ctype_id to type type + datatype_json["id"] = ctype_id else: - msg = "Bad Request: shape is invalid" - log.warn(msg) - raise HTTPBadRequest(reason=msg) - - if dims is not None: - for i in range(rank): - extent = dims[i] - if not isinstance(extent, int): - msg = "Invalid shape type" - log.warn(msg) - raise HTTPBadRequest(reason=msg) - if extent < 0: - msg = "shape dimension is negative" - log.warn(msg) - raise HTTPBadRequest(reason=msg) - - maxdims = None - if "maxdims" in body: - if dims is None: - msg = "Maxdims cannot be supplied if space is NULL" - log.warn(msg) - raise HTTPBadRequest(reason=msg) - - maxdims = body["maxdims"] - if isinstance(maxdims, int): - dim1 = maxdims - maxdims = [dim1] - elif isinstance(maxdims, list): - pass # can use as is + pass # we'll fetch type in getDatasetCreateArgs + + if isinstance(body, list): + count = len(body) + log.debug(f"multiple dataset create: {count} items") + if count == 0: + # equivalent to no body + msg = "POST Dataset with no body" + log.warn(msg) + raise HTTPBadRequest(reason=msg) + elif count == 1: + # just create one object in typical way + kwargs = getDatasetCreateArgs(body[0], + root_id=root_id, + type=datatype_json, + bucket=bucket, + implicit=implicit) + _updateInitValuesList(kwargs) else: - msg = "Bad Request: maxdims is invalid" - log.warn(msg) - raise HTTPBadRequest(reason=msg) - if len(dims) != len(maxdims): - msg = "Maxdims rank doesn't match Shape" - log.warn(msg) - raise HTTPBadRequest(reason=msg) + # create multiple dataset objects + kwarg_list = [] # list of kwargs for each object - if maxdims is not None: - for extent in maxdims: - if not isinstance(extent, int): - msg = "Invalid maxdims type" - log.warn(msg) - raise HTTPBadRequest(reason=msg) - if extent < 0: - msg = "maxdims dimension is negative" - log.warn(msg) - raise HTTPBadRequest(reason=msg) - if len(maxdims) != len(dims): - msg = "Bad Request: maxdims array length must equal " - msg += "shape array length" - log.warn(msg) - raise HTTPBadRequest(reason=msg) - shape_json["maxdims"] = [] - for i in range(rank): - maxextent = maxdims[i] - if not isinstance(maxextent, int): - msg = "Bad Request: maxdims must be integer type" - log.warn(msg) - raise HTTPBadRequest(reason=msg) - elif maxextent == 0: - # unlimited dimension - shape_json["maxdims"].append(0) - elif maxextent < dims[i]: - msg = "Bad Request: maxdims extent can't be smaller " - msg += "than shape extent" - log.warn(msg) - raise HTTPBadRequest(reason=msg) - else: - shape_json["maxdims"].append(maxextent) - - layout_props = None - min_chunk_size = int(config.get("min_chunk_size")) - max_chunk_size = int(config.get("max_chunk_size")) - if "creationProperties" in body: - creationProperties = body["creationProperties"] - log.debug(f"got creationProperties: {creationProperties}") - if "layout" in creationProperties: - layout_props = creationProperties["layout"] - await validateChunkLayout(app, shape_json, item_size, layout_props, bucket=bucket) - else: - creationProperties = {} - - # TBD: check for invalid layout class... - if layout_props: - if layout_props["class"] == "H5D_CONTIGUOUS": - # treat contiguous as chunked - layout_class = "H5D_CHUNKED" - else: - layout_class = layout_props["class"] - elif shape_json["class"] != "H5S_NULL": - layout_class = "H5D_CHUNKED" - else: - layout_class = None - - if layout_class == "H5D_COMPACT": - layout = {"class": "H5D_COMPACT"} - elif layout_class: - # initialize to H5D_CHUNKED - layout = {"class": "H5D_CHUNKED"} - else: - # null space - no layout - layout = None - - if layout_props and "dims" in layout_props: - chunk_dims = layout_props["dims"] - else: - chunk_dims = None - - if layout_class == "H5D_CONTIGUOUS_REF": - kwargs = {"chunk_min": min_chunk_size, "chunk_max": max_chunk_size} - chunk_dims = getContiguousLayout(shape_json, item_size, **kwargs) - layout["dims"] = chunk_dims - log.debug(f"autoContiguous layout: {layout}") - - if layout_class == "H5D_CHUNKED" and chunk_dims is None: - # do autochunking - chunk_dims = guessChunk(shape_json, item_size) - log.debug(f"initial autochunk layout: {chunk_dims}") - - if layout_class == "H5D_CHUNKED": - chunk_size = getChunkSize(chunk_dims, item_size) - - msg = f"chunk_size: {chunk_size}, min: {min_chunk_size}, " - msg += f"max: {max_chunk_size}" - log.debug(msg) - # adjust the chunk shape if chunk size is too small or too big - adjusted_chunk_dims = None - if chunk_size < min_chunk_size: - msg = f"chunk size: {chunk_size} less than min size: " - msg += f"{min_chunk_size}, expanding" - log.debug(msg) - kwargs = {"chunk_min": min_chunk_size, "layout_class": layout_class} - adjusted_chunk_dims = expandChunk(chunk_dims, item_size, shape_json, **kwargs) - elif chunk_size > max_chunk_size: - msg = f"chunk size: {chunk_size} greater than max size: " - msg += f"{max_chunk_size}, shrinking" - log.debug(msg) - kwargs = {"chunk_max": max_chunk_size} - adjusted_chunk_dims = shrinkChunk(chunk_dims, item_size, **kwargs) - if adjusted_chunk_dims: - msg = f"requested chunk_dimensions: {chunk_dims} modified " - msg += f"dimensions: {adjusted_chunk_dims}" - log.debug(msg) - layout["dims"] = adjusted_chunk_dims - else: - layout["dims"] = chunk_dims # don't need to adjust chunk size - - # set partition_count if needed: - max_chunks_per_folder = int(config.get("max_chunks_per_folder")) - set_partition = False - if max_chunks_per_folder > 0: - if "dims" in shape_json and "dims" in layout: - set_partition = True - - if set_partition: - chunk_dims = layout["dims"] - shape_dims = shape_json["dims"] - if "maxdims" in shape_json: - max_dims = shape_json["maxdims"] - else: - max_dims = None - num_chunks = 1 - rank = len(shape_dims) - unlimited_count = 0 - if max_dims: - for i in range(rank): - if max_dims[i] == 0: - unlimited_count += 1 - msg = f"number of unlimited dimensions: {unlimited_count}" - log.debug(msg) - - for i in range(rank): - max_dim = 1 - if max_dims: - max_dim = max_dims[i] - if max_dim == 0: - # don't really know what the ultimate extent - # could be, but assume 10^6 for total number of - # elements and square-shaped array... - MAX_ELEMENT_GUESS = 10.0 ** 6 - exp = 1 / unlimited_count - max_dim = int(math.pow(MAX_ELEMENT_GUESS, exp)) - else: - max_dim = shape_dims[i] - num_chunks *= math.ceil(max_dim / chunk_dims[i]) - - if num_chunks > max_chunks_per_folder: - partition_count = math.ceil(num_chunks / max_chunks_per_folder) - msg = f"set partition count to: {partition_count}, " - msg += f"num_chunks: {num_chunks}" - log.info(msg) - layout["partition_count"] = partition_count - else: - msg = "do not need chunk partitions, num_chunks: " - msg += f"{num_chunks} max_chunks_per_folder: " - msg += f"{max_chunks_per_folder}" - log.info(msg) - - if layout_class in ("H5D_CHUNKED_REF", "H5D_CHUNKED_REF_INDIRECT"): - chunk_size = getChunkSize(chunk_dims, item_size) - - msg = f"chunk_size: {chunk_size}, min: {min_chunk_size}, " - msg += f"max: {max_chunk_size}" - log.debug(msg) - # nothing to do about inefficiently small chunks, but large chunks - # can be subdivided - if chunk_size < min_chunk_size: - msg = f"chunk size: {chunk_size} less than min size: " - msg += f"{min_chunk_size} for {layout_class} dataset" - log.warn(msg) - elif chunk_size > max_chunk_size: - msg = f"chunk size: {chunk_size} greater than max size: " - msg += f"{max_chunk_size}, for {layout_class} dataset" - log.warn(msg) - layout["dims"] = chunk_dims - - if creationProperties: - # TBD - validate all creationProperties - if "fillValue" in creationProperties: - # validate fill value compatible with type - dt = createDataType(datatype) - fill_value = creationProperties["fillValue"] - if "fillValue_encoding" in creationProperties: - fill_value_encoding = creationProperties["fillValue_encoding"] - - if fill_value_encoding not in ("None", "base64"): - msg = f"unexpected value for fill_value_encoding: {fill_value_encoding}" + for item in body: + log.debug(f"item: {item}") + if not isinstance(item, dict): + msg = f"Post_Dataset - invalid item type: {type(item)}" log.warn(msg) raise HTTPBadRequest(reason=msg) - else: - # should see a string in this case - if not isinstance(fill_value, str): - msg = f"unexpected fill value: {fill_value} " - msg += f"for encoding: {fill_value_encoding}" - log.warn(msg) - raise HTTPBadRequest(reason=msg) - else: - fill_value_encoding = None - - try: - getNumpyValue(fill_value, dt=dt, encoding=fill_value_encoding) - except ValueError: - msg = f"invalid fill value: {fill_value}" - log.warn(msg) - raise HTTPBadRequest(reason=msg) - - if "filters" in creationProperties: - # convert to standard representation - # refer to https://hdf5-json.readthedocs.io/en/latest/bnf/\ - # filters.html#grammar-token-filter_list - f_in = creationProperties["filters"] - supported_filters = getSupportedFilters(include_compressors=True) - log.debug(f"supported_compressors: {supported_filters}") - - log.debug(f"filters provided in creationProperties: {f_in}") - - if not isinstance(f_in, list): - msg = "Expected filters in creationProperties to be a list" - log.warn(msg) - raise HTTPBadRequest(reason=msg) - - if f_in and chunk_size is None: - # filters can only be used with chunked datasets - msg = "Filters can only be used with chunked datasets" - log.warning(msg) - raise HTTPBadRequest(reason=msg) - - f_out = [] - for filter in f_in: - if isinstance(filter, int) or isinstance(filter, str): - item = getFilterItem(filter) - if not item: - msg = f"filter {filter} not recognized" - log.warn(msg) - raise HTTPBadRequest(reason=msg) - - if item["name"] not in supported_filters: - msg = f"filter {filter} is not supported" - log.warn(msg) - raise HTTPBadRequest(reason=msg) - f_out.append(item) - elif isinstance(filter, dict): - if "class" not in filter: - msg = "expected 'class' key for filter property" - log.warn(msg) - raise HTTPBadRequest(reason=msg) - if filter["class"] != "H5Z_FILTER_USER": - item = getFilterItem(filter["class"]) - elif "id" in filter: - item = getFilterItem(filter["id"]) - elif "name" in filter: - item = getFilterItem(filter["name"]) - else: - item = None - if not item: - msg = f"filter {filter['class']} not recognized" - log.warn(msg) - raise HTTPBadRequest(reason=msg) - if "id" not in filter: - filter["id"] = item["id"] - elif item["id"] != filter["id"]: - msg = f"Expected {filter['class']} to have id: " - msg += f"{item['id']} but got {filter['id']}" - log.warn(msg) - raise HTTPBadRequest(reason=msg) - if "name" not in filter: - filter["name"] = item["name"] - if filter["name"] not in supported_filters: - msg = f"filter {filter} is not supported" - log.warn(msg) - raise HTTPBadRequest(reason=msg) - - f_out.append(filter) - else: - msg = f"Unexpected type for filter: {filter}" - log.warn(msg) - raise HTTPBadRequest(reason=msg) - # replace filters with our starndardized list - log.debug(f"setting filters to: {f_out}") - creationProperties["filters"] = f_out - - log.debug(f"set dataset json creationPropries: {creationProperties}") - - parent_id = None - link_title = None - h5path = None - if "link" in body: - if "h5path" in body: - msg = "link can't be used with h5path" + kwargs = getDatasetCreateArgs(item, + root_id=root_id, + type=datatype_json, + bucket=bucket) + _updateInitValuesList(kwargs) + kwargs["ignore_link"] = True + kwarg_list.append(kwargs) + kwargs = {"bucket": bucket, "root_id": root_id} + if datatype_json: + kwargs["type"] = datatype_json + log.debug(f"createDatasetObjects, items: {kwarg_list}") + post_rsp = await createDatasets(app, kwarg_list, **kwargs) + else: + # single object create + kwargs = getDatasetCreateArgs(body, + root_id=root_id, + type=datatype_json, + bucket=bucket, + implicit=implicit) + _updateInitValuesList(kwargs) + log.debug(f"kwargs for dataset create: {kwargs}") + + if post_rsp is None: + # Handle cases other than multi ctype create here + post_rsp = await createDataset(app, **kwargs) + + log.debug(f"returning resp: {post_rsp}") + + if "objects" in post_rsp: + # add any links in multi request + objects = post_rsp["objects"] + obj_count = len(objects) + log.debug(f"Post dataset multi create: {obj_count} objects") + if len(body) != obj_count: + msg = f"Expected {obj_count} objects but got {len(body)}" log.warn(msg) raise HTTPBadRequest(reason=msg) - link_body = body["link"] - if "id" in link_body: - parent_id = link_body["id"] - if "name" in link_body: - link_title = link_body["name"] - try: - # will throw exception if there's a slash in the name - validateLinkName(link_title) - except ValueError: - msg = f"invalid link title: {link_title}" - log.warn(msg) - raise HTTPBadRequest(reason=msg) - - if parent_id and link_title: - log.debug(f"parent id: {parent_id}, link_title: {link_title}") - h5path = link_title # just use the link name as the h5path - - if "h5path" in body: - h5path = body["h5path"] - if "parent_id" not in body: - parent_id = root_id - else: - parent_id = body["parent_id"] - - # setup args to createObject - kwargs = {"bucket": bucket, "obj_type": datatype, "obj_shape": shape_json} - if creationProperties: - kwargs["creation_props"] = creationProperties - if layout: - kwargs["layout"] = layout - - if parent_id: - kwargs["parent_id"] = parent_id - kwargs["h5path"] = h5path - # allow parent group creation or not - implicit = getBooleanParam(params, "implicit") - if implicit: - kwargs["implicit"] = True - dset_json = await createObjectByPath(app, **kwargs) else: - # create an anonymous datatype - kwargs["root_id"] = root_id - dset_json = await createObject(app, **kwargs) + obj_count = 1 # single object create + objects = [post_rsp, ] # treat as an array to make the following code more consistent + + if len(init_values) != obj_count: + msg = f"Expected {obj_count} init values" + log.error(msg) + raise HTTPInternalServerError() + + # write any init data values + init_chunks = {} + for index in range(obj_count): + init_data = init_values[index] + if init_data is None: + continue # no data to initialize + log.debug(f"init data: {init_data}") + dset_json = objects[index] + dset_id = dset_json["id"] + log.debug(f"init value, post_rsp: {dset_json}") + layout_class = getDatasetLayoutClass(dset_json) + log.debug(f"layout_class: {layout_class}") + if layout_class not in ("H5D_CONTIGUOUS", "H5D_CHUNKED"): + msg = f"dataset init_data used with unsupported layout_class: {layout_class}" + log.error(msg) + raise HTTPInternalServerError() + layout_dims = getChunkDims(dset_json) + log.debug(f"init data layout is: {layout_dims}") + # make selection for entire dataspace + dims = getShapeDims(dset_json) + slices = selections.select(tuple(dims), ...) + + chunk_ids = getChunkIds(dset_id, slices, layout_dims) + log.debug(f"init data, got chunk_ids: {chunk_ids}") + if not chunk_ids or len(chunk_ids) != 1: + msg = f"expected one chunk for init_data but got: {chunk_ids}" + log.error(msg) + raise HTTPInternalServerError() + chunk_id = chunk_ids[0] + shape_json = dset_json["shape"] + type_json = dset_json["type"] + arr_dtype = createDataType(type_json) + dims = getShapeDims(shape_json) + try: + input_arr = jsonToArray(dims, arr_dtype, init_data) + except ValueError: + log.warn(f"ValueError: {msg}") + raise HTTPBadRequest(reason=msg) + except TypeError: + log.warn(f"TypeError: {msg}") + raise HTTPBadRequest(reason=msg) + except IndexError: + log.warn(f"IndexError: {msg}") + raise HTTPBadRequest(reason=msg) + log.debug(f"got json arr: {input_arr.shape}") + init_chunks[chunk_id] = input_arr + + if init_chunks: + # write dataset init values using the Domain Crawler + log.debug(f"POST dataset - setting init values: {list(init_chunks.keys())}") + kwargs = {"action": "put_data", "bucket": bucket} + + crawler = DomainCrawler(app, init_chunks, **kwargs) + + # will raise exception on not found, server busy, etc. + await crawler.crawl() + status = crawler.get_status() + log.info(f"DomainCrawler done for put_data action, status: {status}") + + if "objects" in post_rsp: + # add any links in multi request + objects = post_rsp["objects"] + obj_count = len(objects) + log.debug(f"Post datatype multi create: {obj_count} objects") + if len(body) != obj_count: + msg = f"Expected {obj_count} objects but got {len(body)}" + log.warn(msg) + raise HTTPBadRequest(reason=msg) + parent_ids = {} + for index in range(obj_count): + item = body[index] + if "link" in item: + link_item = item["link"] + parent_id = link_item.get("id") + title = link_item.get("name") + if parent_id and title: + # add a hard link + object = objects[index] + obj_id = object["id"] + if parent_id not in parent_ids: + parent_ids[parent_id] = {} + links = parent_ids[parent_id] + links[title] = {"id": obj_id} + if parent_ids: + log.debug(f"POST dataset multi - adding links: {parent_ids}") + kwargs = {"action": "put_link", "bucket": bucket} + kwargs["replace"] = True + + crawler = DomainCrawler(app, parent_ids, **kwargs) + + # will raise exception on not found, server busy, etc. + await crawler.crawl() + status = crawler.get_status() + + log.info(f"DomainCrawler done for put_links action, status: {status}") # dataset creation successful - resp = await jsonResponse(request, dset_json, status=201) + resp = await jsonResponse(request, post_rsp, status=201) log.response(request, resp=resp) return resp @@ -1169,7 +724,7 @@ async def DELETE_Dataset(request): msg = "Missing dataset id" log.warn(msg) raise HTTPBadRequest(reason=msg) - if not isValidUuid(dset_id, "Dataset"): + if not isValidUuid(dset_id, obj_class="datasets"): msg = f"Invalid dataset id: {dset_id}" log.warn(msg) raise HTTPBadRequest(reason=msg) diff --git a/hsds/folder_crawl.py b/hsds/folder_crawl.py index 48f37ce6..f1b3fcf2 100644 --- a/hsds/folder_crawl.py +++ b/hsds/folder_crawl.py @@ -13,14 +13,15 @@ # service node of hsds cluster # -import time import asyncio from aiohttp.web_exceptions import HTTPBadRequest, HTTPForbidden, HTTPNotFound from aiohttp.web_exceptions import HTTPGone, HTTPInternalServerError from aiohttp.web_exceptions import HTTPServiceUnavailable -from .util.idUtil import getNodeCount +from h5json.time_util import getNow from .servicenode_lib import getObjectJson, getDomainResponse, getDomainJson +from .util.nodeUtil import getNodeCount + from . import hsds_logger as log @@ -50,6 +51,9 @@ def __init__( else: self._max_tasks = len(domains) + def now(self): + return getNow(app=self._app) + async def crawl(self): workers = [asyncio.Task(self.work()) for _ in range(self._max_tasks)] # When all work is done, exit. @@ -67,11 +71,11 @@ async def crawl(self): async def work(self): while True: - start = time.time() + start = self.now() domain = await self._q.get() await self.fetch(domain) self._q.task_done() - elapsed = time.time() - start + elapsed = self.now() - start msg = f"FolderCrawler - task {domain} start: {start:.3f} " msg += f"elapsed: {elapsed:.3f}" log.debug(msg) diff --git a/hsds/group_dn.py b/hsds/group_dn.py index 0a6bb937..8e5022b9 100755 --- a/hsds/group_dn.py +++ b/hsds/group_dn.py @@ -19,9 +19,10 @@ from aiohttp.web_exceptions import HTTPNotFound, HTTPServiceUnavailable from aiohttp.web import json_response -from .util.idUtil import isValidUuid, isSchema2Id, isRootObjId, getRootObjId +from h5json.objid import isValidUuid, isSchema2Id, isRootObjId, getRootObjId +from h5json.time_util import getNow + from .util.domainUtil import isValidBucketName -from .util.timeUtil import getNow from .datanode_lib import get_obj_id, check_metadata_obj, get_metadata_obj from .datanode_lib import save_metadata_obj, delete_metadata_obj from . import hsds_logger as log @@ -46,7 +47,7 @@ async def GET_Group(request): log.info(f"GET group: {group_id} bucket: {bucket}") - if not isValidUuid(group_id, obj_class="group"): + if not isValidUuid(group_id, obj_class="groups"): log.error(f"Unexpected group_id: {group_id}") raise HTTPInternalServerError() @@ -97,9 +98,13 @@ async def POST_Group(request): raise HTTPBadRequest(reason=msg) group_id = get_obj_id(request, body=body) + deleted_ids = app["deleted_ids"] + if group_id in deleted_ids: + log.warn(f"POST Group has id: {group_id} that has previously been deleted") + deleted_ids.remove(group_id) - log.info(f"POST group: {group_id} bucket: {bucket}") - if not isValidUuid(group_id, obj_class="group"): + log.info(f"POST group: {group_id} bucket: {bucket} body: {body}") + if not isValidUuid(group_id, obj_class="groups"): log.error(f"Unexpected group_id: {group_id}") raise HTTPInternalServerError() if "root" not in body: @@ -110,12 +115,13 @@ async def POST_Group(request): # verify the id doesn't already exist obj_found = await check_metadata_obj(app, group_id, bucket=bucket) if obj_found: - log.error(f"Post with existing group_id: {group_id}") - raise HTTPInternalServerError() + msg = f"Post with existing group_id: {group_id}" + log.warn(msg) + raise HTTPBadRequest(reason=msg) root_id = body["root"] - if not isValidUuid(root_id, obj_class="group"): + if not isValidUuid(root_id, obj_class="groups"): msg = "Invalid root_id: " + root_id log.error(msg) raise HTTPInternalServerError() @@ -123,13 +129,27 @@ async def POST_Group(request): # ok - all set, create group obj now = getNow(app) + if "attributes" in body: + # initialize attributes + attrs = body["attributes"] + log.debug(f"POST Group with attributes: {attrs}") + else: + attrs = {} + + if "links" in body: + # initialize links + links = body["links"] + log.debug(f"POST Group with links: {links}") + else: + links = {} + group_json = { "id": group_id, "root": root_id, "created": now, "lastModified": now, - "links": {}, - "attributes": {}, + "links": links, + "attributes": attrs, } if "creationProperties" in body: @@ -144,8 +164,8 @@ async def POST_Group(request): resp_json["root"] = root_id resp_json["created"] = group_json["created"] resp_json["lastModified"] = group_json["lastModified"] - resp_json["linkCount"] = 0 - resp_json["attributeCount"] = 0 + resp_json["linkCount"] = len(links) + resp_json["attributeCount"] = len(attrs) resp = json_response(resp_json, status=201) log.response(request, resp=resp) @@ -178,7 +198,7 @@ async def PUT_Group(request): # don't really need bucket param since the dirty ids know which bucket # they should write too - if not isValidUuid(root_id, obj_class="group"): + if not isValidUuid(root_id, obj_class="groups"): log.error(f"Unexpected group_id: {root_id}") raise HTTPInternalServerError() @@ -247,7 +267,7 @@ async def DELETE_Group(request): params = request.rel_url.query group_id = get_obj_id(request) - if not isValidUuid(group_id, obj_class="group"): + if not isValidUuid(group_id, obj_class="groups"): log.error(f"Unexpected group_id: {group_id}") raise HTTPInternalServerError() @@ -319,7 +339,7 @@ async def POST_Root(request): try: timestamp = int(params["timestamp"]) except ValueError: - log.error("unexpected value for timestamp: {params}") + log.error(f"unexpected value for timestamp: {params}") raise HTTPInternalServerError() else: timestamp = getNow(app) diff --git a/hsds/group_sn.py b/hsds/group_sn.py index 2b573985..991b50bd 100755 --- a/hsds/group_sn.py +++ b/hsds/group_sn.py @@ -16,17 +16,19 @@ from aiohttp.web_exceptions import HTTPBadRequest, HTTPForbidden, HTTPNotFound from json import JSONDecodeError +from h5json.objid import isValidUuid + from .util.httpUtil import getHref, jsonResponse, getBooleanParam -from .util.idUtil import isValidUuid from .util.authUtil import getUserPasswordFromRequest, aclCheck from .util.authUtil import validateUserPassword from .util.domainUtil import getDomainFromRequest, isValidDomain from .util.domainUtil import getBucketForDomain, getPathForDomain, verifyRoot -from .util.linkUtil import validateLinkName from .servicenode_lib import getDomainJson, getObjectJson, validateAction -from .servicenode_lib import getObjectIdByPath, getPathForObjectId -from .servicenode_lib import createObject, createObjectByPath, deleteObject +from .servicenode_lib import getObjectIdByPath, getPathForObjectId, deleteObject +from .servicenode_lib import getCreateArgs, createGroup from . import hsds_logger as log +from .post_crawl import createGroups +from .domain_crawl import DomainCrawler async def GET_Group(request): @@ -50,7 +52,7 @@ async def GET_Group(request): if group_id: log.info(f"GET_Group, id: {group_id}") # is the id a group id and not something else? - if not isValidUuid(group_id, "Group"): + if not isValidUuid(group_id, obj_class="groups"): msg = f"Invalid group id: {group_id}" log.warn(msg) raise HTTPBadRequest(reason=msg) @@ -97,7 +99,7 @@ async def GET_Group(request): kwargs = {"bucket": bucket, "domain": domain} group_id, domain, obj_json = await getObjectIdByPath(app, group_id, h5path, **kwargs) - if not isValidUuid(group_id, "Group"): + if not isValidUuid(group_id, obj_class="groups"): msg = f"No group exist with the path: {h5path}" log.warn(msg) raise HTTPNotFound() @@ -173,6 +175,7 @@ async def POST_Group(request): bucket = getBucketForDomain(domain) domain_json = await getDomainJson(app, domain, reload=True) + log.debug(f"got domain_json: {domain_json}") # throws exception if not allowed aclCheck(app, domain_json, "create", username) @@ -182,11 +185,8 @@ async def POST_Group(request): # allow parent group creation or not implicit = getBooleanParam(params, "implicit") - - parent_id = None - h5path = None - creation_props = None - + kwargs = {} + post_rsp = None if request.has_body: try: body = await request.json() @@ -197,55 +197,92 @@ async def POST_Group(request): log.info(f"POST Group body: {body}") if body: - if "link" in body: - if "h5path" in body: - msg = "link can't be used with h5path" - log.warn(msg) - raise HTTPBadRequest(reason=msg) - link_body = body["link"] - log.debug(f"link_body: {link_body}") - if "id" in link_body: - parent_id = link_body["id"] - if "name" in link_body: - link_title = link_body["name"] - try: - # will throw exception if there's a slash in the name - validateLinkName(link_title) - except ValueError: - msg = f"invalid link title: {link_title}" - log.warn(msg) - raise HTTPBadRequest(reason=msg) - - if parent_id and link_title: - log.debug(f"parent id: {parent_id}, link_title: {link_title}") - h5path = link_title # just use the link name as the h5path - - if "h5path" in body: - h5path = body["h5path"] - if "parent_id" not in body: - parent_id = root_id + if isinstance(body, list): + count = len(body) + log.debug(f"multiple group create: {count} items") + if count == 0: + # equivalent to no body, anonymous group case + kwargs = {"root_id": root_id, "bucket": bucket} + elif count == 1: + # just create one object in typical way + kwargs = getCreateArgs(body[0], + root_id=root_id, + bucket=bucket, + implicit=implicit) else: - parent_id = body["parent_id"] - if "creationProperties" in body: - creation_props = body["creationProperties"] - - if parent_id: - kwargs = {"bucket": bucket, "parent_id": parent_id, "h5path": h5path} - if creation_props: - kwargs["creation_props"] = creation_props - if implicit: - kwargs["implicit"] = True - group_json = await createObjectByPath(app, **kwargs) + # create multiple group objects + kwarg_list = [] # list of kwargs for each object + + for item in body: + log.debug(f"item: {item}") + if not isinstance(item, dict): + msg = f"PostGroup - invalid item type: {type(item)}" + log.warn(msg) + raise HTTPBadRequest(reason=msg) + kwargs = getCreateArgs(item, root_id=root_id, bucket=bucket) + kwargs["ignore_link"] = True + kwarg_list.append(kwargs) + kwargs = {"bucket": bucket, "root_id": root_id} + post_rsp = await createGroups(app, kwarg_list, **kwargs) + else: + kwargs = getCreateArgs(body, root_id=root_id, bucket=bucket, implicit=implicit) + else: + kwargs["root_id"] = root_id + kwargs["bucket"] = bucket else: - # create an anonymous group - kwargs = {"bucket": bucket, "root_id": root_id} - if creation_props: - kwargs["creation_props"] = creation_props - group_json = await createObject(app, **kwargs) + kwargs = {"root_id": root_id, "bucket": bucket} + + if post_rsp is None: + log.debug(f"post_rsp is None, call createGroup with kwargs: {kwargs}") + # Handle cases other than multi-group create here + if "type" in kwargs: + msg = "type key is not allowed for Group creation" + log.warn(msg) + raise HTTPBadRequest(reason=msg) + post_rsp = await createGroup(app, **kwargs) + + log.debug(f"returning resp: {post_rsp}") + + if "objects" in post_rsp: + # add any links in multi request + objects = post_rsp["objects"] + obj_count = len(objects) + log.debug(f"Post group multi create: {obj_count} objects") + if len(body) != obj_count: + msg = f"Expected {obj_count} objects but got {len(body)}" + log.warn(msg) + raise HTTPBadRequest(reason=msg) + parent_ids = {} + for index in range(obj_count): + item = body[index] + if "link" in item: + link_item = item["link"] + parent_id = link_item.get("id") + title = link_item.get("name") + if parent_id and title: + # add a hard link + object = objects[index] + obj_id = object["id"] + if parent_id not in parent_ids: + parent_ids[parent_id] = {} + links = parent_ids[parent_id] + links[title] = {"id": obj_id} + if parent_ids: + log.debug(f"POST group multi - adding links: {parent_ids}") + kwargs = {"action": "put_link", "bucket": bucket} + kwargs["replace"] = True + + crawler = DomainCrawler(app, parent_ids, **kwargs) + + # will raise exception on not found, server busy, etc. + await crawler.crawl() + + status = crawler.get_status() + + log.info(f"DomainCrawler done for put_links action, status: {status}") - log.debug(f"returning resp: {group_json}") # group creation successful - resp = await jsonResponse(request, group_json, status=201) + resp = await jsonResponse(request, post_rsp, status=201) log.response(request, resp=resp) return resp @@ -260,7 +297,7 @@ async def DELETE_Group(request): msg = "Missing group id" log.warn(msg) raise HTTPBadRequest(reason=msg) - if not isValidUuid(group_id, "Group"): + if not isValidUuid(group_id, obj_class="groups"): msg = f"Invalid group id: {group_id}" log.warn(msg) raise HTTPBadRequest(reason=msg) diff --git a/hsds/headnode.py b/hsds/headnode.py index 9b49517d..205e4a11 100755 --- a/hsds/headnode.py +++ b/hsds/headnode.py @@ -15,14 +15,13 @@ import asyncio import os -import time from aiohttp.web import Application, StreamResponse, run_app, json_response from aiohttp.web_exceptions import HTTPBadRequest, HTTPInternalServerError +from h5json.time_util import unixTimeToUTC, elapsedTime, getNow from . import config -from .util.timeUtil import unixTimeToUTC, elapsedTime -from .util.idUtil import createNodeId +from .util.nodeUtil import createNodeId from . import hsds_logger as log from .util import query_marathon as marathonClient @@ -46,7 +45,7 @@ def __init__(self, node_id=None, node_type=None, node_host=None, node_port=None) self._type = node_type self._host = node_host self._port = node_port - now = time.time() + now = getNow() self._create_time = now self._last_poll = now self._stats = {} @@ -87,13 +86,13 @@ def get_info(self): return info def poll_update(self): - now = time.time() + now = getNow() self._last_poll = now def is_healthy(self): sleep_sec = int(config.get("node_sleep_time")) - now = time.time() + now = getNow() if now - self._last_poll < sleep_sec * 2: return True else: @@ -301,7 +300,7 @@ async def register(request): answer["dn_ids"] = dn_ids answer["req_ip"] = node_host log.debug(f"register returning: {answer}") - app["last_health_check"] = int(time.time()) + app["last_health_check"] = int(getNow()) resp = json_response(answer) log.response(request, resp=resp) @@ -410,18 +409,18 @@ async def nodeinfo(request): async def getTargetNodeCount(app, node_type): if node_type == "dn": - key = "target_sn_count" + key = "target_dn_count" elif node_type == "sn": key = "target_sn_count" else: raise KeyError() - if "key" not in app: + if key not in app: if "is_dcos" in app: marathon = marathonClient.MarathonClient(app) if node_type == "dn": app[key] = int(await marathon.getDNInstances()) else: - app[key] = int(await marathon.getDNInstances()) + app[key] = int(await marathon.getSNInstances()) else: app[key] = config.get(key) return app[key] @@ -440,7 +439,7 @@ def getActiveNodeCount(app, node_type): async def init(): - """Intitialize application and return app object""" + """Initialize application and return app object""" # setup log config log_level = config.get("log_level") @@ -475,7 +474,7 @@ async def init(): app["nodes"] = nodes app["dead_node_ids"] = set() - app["start_time"] = int(time.time()) # seconds after epoch + app["start_time"] = int(getNow()) # seconds after epoch app["last_health_check"] = 0 app["max_task_count"] = config.get("max_task_count") app.router.add_get("/", info) diff --git a/hsds/hsds_app.py b/hsds/hsds_app.py index e690b68d..7bef1a9d 100644 --- a/hsds/hsds_app.py +++ b/hsds/hsds_app.py @@ -3,12 +3,14 @@ from pathlib import Path import site import subprocess -import time import queue import threading +import time import logging from shutil import which +from h5json.time_util import getNow + def _enqueue_output(out, queue, loglevel): try: @@ -99,7 +101,6 @@ def __init__( sn_port=None, config_dir=None, readonly=False, - islambda=False, ): """ Initializer for class @@ -117,7 +118,6 @@ def __init__( self._logfile = logfile self._loglevel = log_level self._readonly = readonly - self._islambda = islambda self._ready = False self._config_dir = config_dir self._cmd_dir = get_cmd_dir() @@ -244,12 +244,6 @@ def run(self): ] common_args.append(f"--dn_urls={dn_urls_arg}") common_args.append(f"--hsds_endpoint={self._endpoint}") - if self._islambda: - # base boto packages installed in AWS image conflicting with aiobotocore - # see: https://github.com/aio-libs/aiobotocore/issues/862 - # This command line argument will tell the sub-processes to remove - # sitepackage libs from their path before importing aiobotocore - common_args.append("--removesitepackages") # common_args.append("--server_name=Direct Connect (HSDS)") if len(self._socket_paths) > 0: common_args.append("--use_socket") @@ -318,7 +312,7 @@ def run(self): self._threads.append(t) # wait to sockets are initialized - start_ts = time.time() + start_ts = getNow() SLEEP_TIME = 1 # time to sleep between checking on socket connection MAX_INIT_TIME = 10.0 # max time to wait for socket to be initialized @@ -329,7 +323,7 @@ def run(self): if os.path.exists(socket_path): ready += 1 else: - if time.time() > start_ts + 5: + if getNow() > start_ts + 5: # TBD - put a real ready check here ready = count if ready == count: @@ -339,12 +333,12 @@ def run(self): self.log.debug(f"{ready}/{count} ready") self.log.debug(f"sleeping for {SLEEP_TIME}") time.sleep(SLEEP_TIME) - if time.time() > start_ts + MAX_INIT_TIME: + if getNow() > start_ts + MAX_INIT_TIME: msg = f"failed to initialize after {MAX_INIT_TIME} seconds" self.log.error(msg) raise IOError(msg) - self.log.info(f"Ready after: {(time.time() - start_ts):4.2f} s") + self.log.info(f"Ready after: {(getNow() - start_ts):4.2f} s") self._ready = True def stop(self): @@ -352,7 +346,7 @@ def stop(self): if not self._processes: return - now = time.time() + now = getNow() logging.info(f"hsds app stop at {now}") for pname in self._processes: @@ -360,10 +354,10 @@ def stop(self): logging.info(f"terminating sub-process: {pname}") p.terminate() - # wait for sub-proccesses to exit + # wait for sub-processes to exit SLEEP_TIME = 0.1 # time to sleep between checking on process state MAX_WAIT_TIME = 10.0 # max time to wait for sub-process to terminate - start_ts = time.time() + start_ts = getNow() while True: is_alive_cnt = 0 for pname in self._processes: @@ -380,7 +374,7 @@ def stop(self): else: logging.debug("all subprocesses exited") break - if time.time() > start_ts + MAX_WAIT_TIME: + if getNow() > start_ts + MAX_WAIT_TIME: msg = f"failed to terminate after {MAX_WAIT_TIME} seconds" self.log.error(msg) break diff --git a/hsds/hsds_logger.py b/hsds/hsds_logger.py index 3421d981..743f1408 100644 --- a/hsds/hsds_logger.py +++ b/hsds/hsds_logger.py @@ -10,7 +10,7 @@ # request a copy from help@hdfgroup.org. # ############################################################################## # -# Simple looger for hsds +# Simple logger for hsds # import asyncio @@ -138,7 +138,7 @@ def request(req): "/register", ) if req.path in INFO_METHODS: - # always service these state requests regardles of node state and + # always service these state requests regardless of node state and # task load return node_state = app["node_state"] if "node_state" in app else None diff --git a/hsds/link_dn.py b/hsds/link_dn.py index f7ec5956..551c2333 100755 --- a/hsds/link_dn.py +++ b/hsds/link_dn.py @@ -20,19 +20,21 @@ from aiohttp.web_exceptions import HTTPInternalServerError from aiohttp.web import json_response -from .util.idUtil import isValidUuid +from h5json.objid import isValidUuid +from h5json.time_util import getNow +from h5json.link_util import validateLinkName, getLinkClass, isEqualLink + from .util.globparser import globmatch -from .util.linkUtil import validateLinkName, getLinkClass, isEqualLink from .util.domainUtil import isValidBucketName -from .util.timeUtil import getNow from .datanode_lib import get_obj_id, get_metadata_obj, save_metadata_obj +from . import config from . import hsds_logger as log def _index(items, marker, create_order=False): """Locate the leftmost value exactly equal to x""" if create_order: - # list is not ordered, juse search linearly + # list is not ordered, just search linearly for i in range(len(items)): if items[i] == marker: return i @@ -74,7 +76,7 @@ async def GET_Links(request): log.debug(f"GET_Links params: {params}") group_id = get_obj_id(request) log.info(f"GET links: {group_id}") - if not isValidUuid(group_id, obj_class="group"): + if not isValidUuid(group_id, obj_class="groups"): log.error(f"Unexpected group_id: {group_id}") raise HTTPInternalServerError() @@ -154,6 +156,10 @@ async def GET_Links(request): link = copy(link_dict[title]) log.debug(f"link list[{i}: {link}") link["title"] = title + if link.get("h5domain"): + # deprecated key, replace with file + link["file"] = link["h5domain"] + del link["h5domain"] link_list.append(link) resp_json = {"links": link_list} @@ -170,7 +176,7 @@ async def POST_Links(request): group_id = get_obj_id(request) log.info(f"POST_Links: {group_id}") - if not isValidUuid(group_id, obj_class="group"): + if not isValidUuid(group_id, obj_class="groups"): log.error(f"Unexpected group_id: {group_id}") raise HTTPInternalServerError() @@ -216,6 +222,7 @@ async def POST_Links(request): log.info(f"Link name {title} not found in group: {group_id}") continue link_json = links[title] + log.debug(f"POST Links got link_json: {link_json}") item = {} if "class" not in link_json: log.warn(f"expected to find class key for link: {title}") @@ -243,15 +250,19 @@ async def POST_Links(request): log.warn(f"expected to find h5path for external link: {title}") continue item["h5path"] = link_json["h5path"] - if "h5domain" not in link_json: - log.warn(f"expted to find h5domain for external link: {title}") + if "h5domain" in link_json: + item["file"] = link_json["h5domain"] + elif "file" in link_json: + item["file"] = link_json["file"] + else: + log.warn(f"expected to find h5domain or file for external link: {title}") continue - item["h5domain"] = link_json["h5domain"] else: log.warn(f"unexpected to link class {link_class} for link: {title}") continue item["title"] = title + log.debug(f"adding link item: {item}") link_list.append(item) @@ -284,8 +295,10 @@ async def PUT_Links(request): params = request.rel_url.query group_id = get_obj_id(request) log.info(f"PUT links: {group_id}") + now = getNow(app) + max_timestamp_drift = int(config.get("max_timestamp_drift", default=300)) - if not isValidUuid(group_id, obj_class="group"): + if not isValidUuid(group_id, obj_class="groups"): log.error(f"Unexpected group_id: {group_id}") raise HTTPInternalServerError() @@ -364,11 +377,16 @@ async def PUT_Links(request): link_delete_set = deleted_links[group_id] else: link_delete_set = set() - - create_time = getNow(app) - for title in new_links: item = items[title] + if item.get("created"): + create_time = item["created"] + log.debug(f"link {title} has create time: {create_time}") + if abs(create_time - now) > max_timestamp_drift: + log.warn(f"link {title} create time stale, ignoring") + create_time = now + else: + create_time = now item["created"] = create_time links[title] = item log.debug(f"added link {title}: {item}") @@ -377,8 +395,7 @@ async def PUT_Links(request): if new_links: # update the group lastModified - group_json["lastModified"] = create_time - log.debug(f"tbd: group_json: {group_json}") + group_json["lastModified"] = now # write back to S3, save to metadata cache await save_metadata_obj(app, group_id, group_json, bucket=bucket) @@ -405,7 +422,7 @@ async def DELETE_Links(request): group_id = get_obj_id(request) log.info(f"DELETE links: {group_id}") - if not isValidUuid(group_id, obj_class="group"): + if not isValidUuid(group_id, obj_class="groups"): msg = f"Unexpected group_id: {group_id}" log.warn(msg) raise HTTPBadRequest(reason=msg) diff --git a/hsds/link_sn.py b/hsds/link_sn.py index 71e39246..a7dcc6a0 100755 --- a/hsds/link_sn.py +++ b/hsds/link_sn.py @@ -13,21 +13,27 @@ # service node of hsds cluster # -from aiohttp.web_exceptions import HTTPBadRequest +from aiohttp.web_exceptions import HTTPBadRequest, HTTPInternalServerError from json import JSONDecodeError +from h5json.objid import isValidUuid, getCollectionForId +from h5json.link_util import validateLinkName, getLinkClass, getLinkId +from h5json.link_util import getLinkPath, getLinkFilePath + +from .util.nodeUtil import getDataNodeUrl from .util.httpUtil import getHref, getBooleanParam from .util.httpUtil import jsonResponse from .util.globparser import globmatch -from .util.idUtil import isValidUuid, getDataNodeUrl, getCollectionForId from .util.authUtil import getUserPasswordFromRequest, validateUserPassword -from .util.domainUtil import getDomainFromRequest, isValidDomain, verifyRoot -from .util.domainUtil import getBucketForDomain -from .util.linkUtil import validateLinkName, getLinkClass +from .util.domainUtil import getDomainFromRequest, isValidDomain, verifyRoot, getBucketForDomain +from .util.linkUtil import getRequestLink + + from .servicenode_lib import getDomainJson, validateAction from .servicenode_lib import getLink, putLink, putLinks, getLinks, deleteLinks from .domain_crawl import DomainCrawler from . import hsds_logger as log +from . import config async def GET_Links(request): @@ -43,7 +49,7 @@ async def GET_Links(request): msg = "Missing group id" log.warn(msg) raise HTTPBadRequest(reason=msg) - if not isValidUuid(group_id, obj_class="Group"): + if not isValidUuid(group_id, obj_class="groups"): msg = f"Invalid group id: {group_id}" log.warn(msg) raise HTTPBadRequest(reason=msg) @@ -140,7 +146,15 @@ async def GET_Links(request): # mix in collection key, target and hrefs for link in links: + for key in ("class", "title"): + if key not in link: + log.error(f"expected to find {key} key in link") + raise HTTPInternalServerError() + if link["class"] == "H5L_TYPE_HARD": + if "id" not in link: + log.error("expected to id key in hard link") + raise HTTPInternalServerError() collection_name = getCollectionForId(link["id"]) link["collection"] = collection_name target_uri = "/" + collection_name + "/" + link["id"] @@ -175,7 +189,7 @@ async def GET_Link(request): msg = "Missing group id" log.warn(msg) raise HTTPBadRequest(reason=msg) - if not isValidUuid(group_id, obj_class="Group"): + if not isValidUuid(group_id, obj_class="groups"): msg = f"Invalid group id: {group_id}" log.warn(msg) raise HTTPBadRequest(reason=msg) @@ -211,13 +225,13 @@ async def GET_Link(request): link_class = link_json["class"] resp_link["class"] = link_class if link_class == "H5L_TYPE_HARD": - resp_link["id"] = link_json["id"] + resp_link["id"] = getLinkId(link_json) resp_link["collection"] = getCollectionForId(link_json["id"]) elif link_class == "H5L_TYPE_SOFT": - resp_link["h5path"] = link_json["h5path"] + resp_link["h5path"] = getLinkPath(link_json) elif link_class == "H5L_TYPE_EXTERNAL": - resp_link["h5path"] = link_json["h5path"] - resp_link["h5domain"] = link_json["h5domain"] + resp_link["h5path"] = getLinkPath(link_json) + resp_link["file"] = getLinkFilePath(link_json) else: log.warn(f"Unexpected link class: {link_class}") resp_json = {} @@ -281,14 +295,32 @@ async def PUT_Link(request): msg = f"Invalid domain: {domain}" log.warn(msg) raise HTTPBadRequest(reason=msg) - bucket = getBucketForDomain(domain) await validateAction(app, domain, group_id, username, "create") - # putLink will validate these arguments - kwargs = {"bucket": bucket} - kwargs["tgt_id"] = body.get("id") - kwargs["h5path"] = body.get("h5path") - kwargs["h5domain"] = body.get("h5domain") + + predate_max_time = config.get("predate_max_time", default=10.0) + + try: + link_json = getRequestLink(link_title, body, predate_max_time=predate_max_time) + except (KeyError, TypeError, ValueError) as e: + raise HTTPBadRequest(reason=str(e)) + + link_class = getLinkClass(link_json) + + kwargs = {} + kwargs["bucket"] = getBucketForDomain(domain) + if link_class == "H5L_TYPE_HARD": + kwargs["tgt_id"] = getLinkId(link_json) + elif link_class == "H5L_TYPE_SOFT": + kwargs["h5path"] = getLinkPath(link_json) + elif link_class == "H5L_TYPE_EXTERNAL": + kwargs["h5path"] = getLinkPath(link_json) + kwargs["h5domain"] = getLinkFilePath(link_json) + else: + raise HTTPBadRequest(reason=f"unexpected link class: {link_class}") + + if "created" in link_json: + kwargs["created"] = link_json["created"] status = await putLink(app, group_id, link_title, **kwargs) @@ -327,6 +359,16 @@ async def PUT_Links(request): log.warn(msg) raise HTTPBadRequest(reason=msg) + if not body: + msg = "PUT links with empty body" + log.warn(msg) + raise HTTPBadRequest(reason=msg) + + if not isinstance(body, dict): + msg = f"PUT links expected dictionary body but got: {type(body)}" + log.warn(msg) + raise HTTPBadRequest(reason=msg) + domain = getDomainFromRequest(request) if not isValidDomain(domain): msg = f"Invalid domain: {domain}" @@ -422,6 +464,7 @@ async def PUT_Links(request): link_item = link_items[title] getLinkClass(link_item) except ValueError: + log.warn(f"invalid link for {title}: {link_item}") raise HTTPBadRequest(reason="invalid link item") grp_ids[grp_id] = link_items @@ -447,7 +490,7 @@ async def PUT_Links(request): count = len(grp_ids) if count == 0: msg = "no grp_ids defined" - log.warn(f"PUT_Attributes: {msg}") + log.warn(f"PUT_Links: {msg}") raise HTTPBadRequest(reason=msg) elif count == 1: # just send one PUT Attributes request to the dn @@ -493,7 +536,7 @@ async def DELETE_Links(request): msg = "Missing group id" log.warn(msg) raise HTTPBadRequest(reason=msg) - if not isValidUuid(group_id, obj_class="Group"): + if not isValidUuid(group_id, obj_class="groups"): msg = f"Invalid group id: {group_id}" log.warn(msg) raise HTTPBadRequest(reason=msg) @@ -640,7 +683,7 @@ async def POST_Links(request): # do a check that everything is as it should with the item list for group_id in items: - if not isValidUuid(group_id, obj_class="Group"): + if not isValidUuid(group_id, obj_class="groups"): msg = f"Invalid group id: {group_id}" log.warn(msg) @@ -747,7 +790,7 @@ async def DELETE_Link(request): msg = "Missing group id" log.warn(msg) raise HTTPBadRequest(reason=msg) - if not isValidUuid(group_id, obj_class="Group"): + if not isValidUuid(group_id, obj_class="groups"): msg = f"Invalid group id: {group_id}" log.warn(msg) raise HTTPBadRequest(reason=msg) diff --git a/hsds/node_runner.py b/hsds/node_runner.py index 4790e07b..8afd0ae1 100644 --- a/hsds/node_runner.py +++ b/hsds/node_runner.py @@ -1,5 +1,3 @@ -import pkg_resources -import site import sys from . import config from . import servicenode @@ -7,25 +5,6 @@ from . import headnode -def removeSitePackages(): - - # site_packages = "/var/lang/lib/python3.9/site-packages" - # but this is removing: "/home/sbx_user1051/.local/lib/python3.9/site-packages" on lambda? - site_packages = site.getusersitepackages() - if not site_packages: - return - print("sitepackages:", site_packages) - - try: - sys.path.remove(site_packages) - except ValueError as ve: - print(f"site_package remove error: {ve}") - else: - sys.path.insert(0, site_packages) - for dist in pkg_resources.find_distributions(site_packages, True): - pkg_resources.working_set.add(dist, site_packages, False, replace=True) - - def main(): node_type = config.getCmdLineArg("node_type") if node_type is None: @@ -34,9 +13,6 @@ def main(): raise ValueError(f"Unexpected node type: {node_type}") print(f"hsds node main for node_type: {node_type}") print(f"python version: {sys.version}") - print(f"sys path: {sys.path}") - if config.getCmdLineArg("removesitepackages"): - removeSitePackages() if node_type == "sn": servicenode.main() diff --git a/hsds/post_crawl.py b/hsds/post_crawl.py new file mode 100644 index 00000000..1e622afc --- /dev/null +++ b/hsds/post_crawl.py @@ -0,0 +1,288 @@ +############################################################################## +# Copyright by The HDF Group. # +# All rights reserved. # +# # +# This file is part of HSDS (HDF5 Scalable Data Service), Libraries and # +# Utilities. The full HSDS copyright notice, including # +# terms governing use, modification, and redistribution, is contained in # +# the file COPYING, which can be found at the root of the source code # +# distribution tree. If you do not have access to this file, you may # +# request a copy from help@hdfgroup.org. # +############################################################################## +# +# post crawler +# + +import asyncio + +from aiohttp.web_exceptions import HTTPServiceUnavailable, HTTPConflict, HTTPBadRequest +from aiohttp.web_exceptions import HTTPInternalServerError, HTTPNotFound, HTTPGone + +from .util.httpUtil import isOK +from .servicenode_lib import createObject +from . import hsds_logger as log + + +class PostCrawler: + def __init__( + self, + app, + items=None, + root_id=None, + bucket=None, + max_tasks=40, + ignore_error=False + ): + log.info("PostCrawler.__init__") + self._app = app + self._root_id = root_id + self._bucket = bucket + self._max_tasks = max_tasks + self._ignore_error = ignore_error + + if not items: + log.error("no post requests for crawler to crawl!") + raise ValueError() + if not bucket: + log.error("bucket not set for PostCrawler") + raise ValueError() + self._count = len(items) + self._items = items + self._rsp_objs = [None,] * self._count + self._q = asyncio.Queue() + log.debug(f"PostCrawler adding index 0 - {self._count} to queue") + for i in range(self._count): + self._q.put_nowait(i) + + def get_rsp_objs(self): + """ return list of object responses """ + + return self._rsp_objs + + def get_status(self): + """ return the highest status of any of the returned objects """ + status = None + for i in range(self._count): + item = self._rsp_objs[i] + if not item: + continue # resp not filled in yet + if "status_code" in item: + item_status = item["status_code"] + if status is None or item_status > status: + # return the more severe error + log.debug(f"setting status to {item_status}") + status = item_status + elif "id" in item: + # post request succeeded + if status is None: + status = 201 + else: + log.error(f"PostCrawler unexpected response for item {i}: {item}") + status = 500 + + return status + + async def crawl(self): + max_tasks = min(self._max_tasks, self._count) + workers = [asyncio.Task(self.work()) for _ in range(max_tasks)] + # When all work is done, exit. + msg = "PostCrawler - await queue.join - " + msg += f"count: {self._count} with {max_tasks} workers" + log.info(msg) + await self._q.join() + msg = "PostCrawler - join complete - " + msg += f"count: {self._count}" + log.info(msg) + + for w in workers: + w.cancel() + log.debug("PostCrawler - workers canceled") + + status = self.get_status() + if status: + log.debug(f"PostCrawler -- status: {status}") + log.debug(f"ignore_error: {self._ignore_error}") + if not self._ignore_error: + # throw the appropriate exception if other than 200, 201 + if isOK(status): + pass # ok + elif status == 400: + log.warn("PostCrawler - BadRequest") + raise HTTPBadRequest(reason="unknown") + elif status == 404: + log.warn("PostCrawler - not found") + raise HTTPNotFound() + elif status == 409: + log.warn("PostCrawler - conflict") + raise HTTPConflict() + elif status == 410: + log.warn("PostCrawler - gone") + raise HTTPGone() + elif status == 500: + log.error("PostCrawler - internal server error") + raise HTTPInternalServerError() + elif status == 503: + log.error("PostCrawler - server busy") + raise HTTPServiceUnavailable() + else: + log.error(f"PostCrawler - unexpected status: {status}") + raise HTTPInternalServerError() + else: + # no tasks returned anything + log.error("PostCrawler - no results returned") + if not self._ignore_error: + raise HTTPInternalServerError() + + async def work(self): + while True: + index = await self._q.get() + await self.create(index) + self._q.task_done() + + async def create(self, index): + log.debug(f"PostCrawler fetch for index: {index}") + item = self._items[index] + log.debug(f"got item: {item}") + kwargs = {"bucket": self._bucket} + + if "obj_id" in item: + kwargs["obj_id"] = item["obj_id"] + if "type" in item: + kwargs["type"] = item["type"] + if "shape" in item: + kwargs["shape"] = item["shape"] + if "layout" in item: + kwargs["layout"] = item["layout"] + if "creation_props" in item: + kwargs["creation_props"] = item["creation_props"] + if "attrs" in item: + kwargs["attrs"] = item["attrs"] + if "parent_id" in item: + kwargs["parent_id"] = item["parent_id"] + elif "root_id" in item: + kwargs["root_id"] = item["root_id"] + if "h5path" in item: + kwargs["h5path"] = item["h5path"] + if "links" in item: + kwargs["links"] = item["links"] + + log.debug(f"PostCrawler index {index} kwargs: {kwargs}") + rsp_json = None + try: + rsp_json = await createObject(self._app, **kwargs) + except HTTPConflict: + log.warn("PostCrawler - got HTTPConflict from http_post") + rsp_json = {"status_code": 409} + except HTTPServiceUnavailable: + rsp_json = {"status_code": 503} + except HTTPInternalServerError: + rsp_json = {"status_code": 500} + except Exception as e: + log.error(f"unexpected exception {e}") + rsp_json = {"status_code": 500} + + log.info(f"PostCrawler - index: {index} post rsp: {rsp_json}") + + self._rsp_objs[index] = rsp_json + + +async def _createObjects(app, items: list, root_id=None, bucket=None): + """ generic create function """ + + post_crawler = PostCrawler(app, root_id=root_id, bucket=bucket, items=items) + await post_crawler.crawl() + if post_crawler.get_status() > 201: + msg = f"_createObjects returning status from crawler: {post_crawler.get_status()}" + log.error(msg) + raise HTTPInternalServerError() + + obj_list = post_crawler.get_rsp_objs() + if not isinstance(obj_list, list): + msg = f"_createObjects expected list but got: {type(obj_list)}" + log.error(msg) + raise HTTPInternalServerError() + return {"objects": obj_list} + + +async def createGroups(app, items: list, root_id=None, bucket=None): + """ create an group objects based on parameters in items list """ + + if not root_id: + msg = "no root_id given for createObjects" + log.warn(msg) + raise HTTPBadRequest(reason=msg) + + for item in items: + if not isinstance(item, dict): + msg = "expected list of dictionary objects for multi-object create" + log.warn(msg) + raise HTTPBadRequest(reason=msg) + if "type" in item: + msg = "type key not allowed for multi-group create" + log.warn(msg) + raise HTTPBadRequest(reason=msg) + if "shape" in item: + msg = "shape key not allowed for multi-group create" + log.warn(msg) + raise HTTPBadRequest(reason=msg) + + log.info(f"createGroups with {len(items)} items, root_id: {root_id}") + + rsp_json = await _createObjects(app, items=items, root_id=root_id, bucket=bucket) + return rsp_json + + +async def createDatatypeObjs(app, items: list, root_id=None, bucket=None): + """ create datatype objects based on parameters in items list """ + + if not root_id: + msg = "no root_id given for createDatatypeObjs" + log.warn(msg) + raise HTTPBadRequest(reason=msg) + + for item in items: + if not isinstance(item, dict): + msg = "expected list of dictionary objects for multi-object create" + log.warn(msg) + raise HTTPBadRequest(reason=msg) + if "type" not in item: + msg = "type key not provided for multi-datatype create" + log.warn(msg) + raise HTTPBadRequest(reason=msg) + if "shape" in item: + msg = "shape key not allowed for multi-datatype create" + log.warn(msg) + raise HTTPBadRequest(reason=msg) + + log.info(f"createDatatypes with {len(items)} items, root_id: {root_id}") + + rsp_json = await _createObjects(app, items=items, root_id=root_id, bucket=bucket) + return rsp_json + + +async def createDatasets(app, items: list, root_id=None, bucket=None): + """ create dataset objects based on parameters in items list """ + + if not root_id: + msg = "no root_id given for createDatasets" + log.warn(msg) + raise HTTPBadRequest(reason=msg) + + for item in items: + if not isinstance(item, dict): + msg = "expected list of dictionary objects for multi-object create" + log.warn(msg) + raise HTTPBadRequest(reason=msg) + if "type" not in item: + msg = "type key not provided for multi-dataset create" + log.warn(msg) + raise HTTPBadRequest(reason=msg) + if "shape" not in item: + msg = "shape key not provided for multi-dataset create" + log.warn(msg) + raise HTTPBadRequest(reason=msg) + + log.info(f"createDatasets with {len(items)} items, root_id: {root_id}") + + rsp_json = await _createObjects(app, items=items, root_id=root_id, bucket=bucket) + return rsp_json diff --git a/hsds/servicenode.py b/hsds/servicenode.py index 8a5ddaee..f9e528eb 100755 --- a/hsds/servicenode.py +++ b/hsds/servicenode.py @@ -14,9 +14,10 @@ # import asyncio -import time from aiohttp.web import run_app import aiohttp_cors +from h5json.time_util import getNow + from .util.lruCache import LruCache from .util.httpUtil import isUnixDomainUrl, bindToSocket, getPortFromUrl from .util.httpUtil import release_http_client, jsonResponse @@ -36,11 +37,11 @@ from .ctype_sn import GET_Datatype, POST_Datatype, DELETE_Datatype from .dset_sn import GET_Dataset, POST_Dataset, DELETE_Dataset from .dset_sn import GET_DatasetShape, PUT_DatasetShape, GET_DatasetType -from .chunk_sn import PUT_Value, GET_Value, POST_Value +from .chunk_sn import PUT_Value, GET_Value, POST_Value, GET_Query async def init(): - """Intitialize application and return app object""" + """Initialize application and return app object""" app = baseInit("sn") # call app.router.add_get() here to add node-specific routes @@ -178,6 +179,9 @@ async def init(): app.router.add_route("GET", path, GET_Value) app.router.add_route("POST", path, POST_Value) + path = "/datasets/{id}/query" + app.router.add_route("GET", path, GET_Query) + # Add CORS to all routes cors_domain = config.get("cors_domain") if cors_domain: @@ -217,10 +221,10 @@ async def preStop(request): log.request(request) app = request.app - shutdown_start = time.time() + shutdown_start = getNow() log.warn(f"preStop request calling on_shutdown at {shutdown_start:.2f}") await on_shutdown(app) - shutdown_elapse_time = time.time() - shutdown_start + shutdown_elapse_time = getNow() - shutdown_start msg = f"shutdown took: {shutdown_elapse_time:.2f} seconds" if shutdown_elapse_time > 2.0: # 2.0 is the default grace period for kubernetes @@ -297,7 +301,7 @@ def main(): sn_url = f"http://localhost:{sn_port}" if isUnixDomainUrl(sn_url): - print("binding to socket:", sn_url) + log.info(f"binding to socket: {sn_url}") try: s = bindToSocket(sn_url) except OSError as oe: diff --git a/hsds/servicenode_lib.py b/hsds/servicenode_lib.py index c8c84f75..0fb7c99a 100644 --- a/hsds/servicenode_lib.py +++ b/hsds/servicenode_lib.py @@ -15,25 +15,37 @@ import asyncio import json +import numpy as np from aiohttp.web_exceptions import HTTPBadRequest, HTTPForbidden, HTTPGone, HTTPConflict from aiohttp.web_exceptions import HTTPNotFound, HTTPInternalServerError + from aiohttp.client_exceptions import ClientOSError, ClientError from aiohttp import ClientResponseError +from h5json.array_util import encodeData, decodeData, bytesToArray, bytesArrayToList +from h5json.array_util import jsonToArray +from h5json.objid import getCollectionForId, createObjId, getRootObjId +from h5json.objid import isSchema2Id, getS3Key, isValidUuid +from h5json.hdf5dtype import getBaseTypeJson, validateTypeItem, createDataType, getItemSize +from h5json.shape_util import getShapeDims, getShapeClass, getShapeJson +from h5json.dset_util import getChunkSize, generateLayout +from h5json.dset_util import getDataSize, validateDatasetCreationProps +from h5json.link_util import h5Join, validateLinkName, getLinkClass, getLinkFilePath +from h5json.time_util import getNow + +from .util.nodeUtil import getDataNodeUrl from .util.authUtil import getAclKeys -from .util.arrayUtil import encodeData -from .util.idUtil import getDataNodeUrl, getCollectionForId, createObjId, getRootObjId -from .util.idUtil import isSchema2Id, getS3Key, isValidUuid -from .util.linkUtil import h5Join, validateLinkName, getLinkClass -from .util.storUtil import getStorJSONObj, isStorObj +from .util.linkUtil import getRequestLinks +from .util.storUtil import getStorJSONObj, isStorObj, getSupportedFilters from .util.authUtil import aclCheck from .util.httpUtil import http_get, http_put, http_post, http_delete from .util.domainUtil import getBucketForDomain, verifyRoot, getLimits from .util.storUtil import getCompressors -from .basenode import getVersion +from .basenode import getVersion from . import hsds_logger as log +from . import config async def getDomainJson(app, domain, reload=False): @@ -104,7 +116,7 @@ async def getDomainJson(app, domain, reload=False): return domain_json -async def getDomainResponse(app, domain_json, bucket=None, verbose=False): +async def getDomainResponse(app, domain_json, bucket=None, verbose=False, getobjs=False): """ construct JSON response for domain request """ rsp_json = {} if "root" in domain_json: @@ -177,6 +189,13 @@ async def getDomainResponse(app, domain_json, bucket=None, verbose=False): rsp_json["num_linked_chunks"] = num_linked_chunks rsp_json["md5_sum"] = md5_sum + if getobjs and "root" in domain_json: + root_id = domain_json["root"] + domain_objs = await getDomainObjs(app, root_id, bucket=bucket) + if domain_objs: + log.debug(f"returning {len(domain_objs)} domain objects for root_id: {root_id}") + rsp_json["domain_objs"] = domain_objs + # pass back config parameters the client may care about rsp_json["limits"] = getLimits() @@ -435,7 +454,12 @@ async def getLink(app, group_id, title, bucket=None): return link_json -async def putLink(app, group_id, title, tgt_id=None, h5path=None, h5domain=None, bucket=None): +async def putLink(app, group_id, title, + tgt_id=None, + h5path=None, + h5domain=None, + bucket=None, + created=None): """ create a new link. Return 201 if this is a new link, or 200 if it's a duplicate of an existing link. """ @@ -455,7 +479,9 @@ async def putLink(app, group_id, title, tgt_id=None, h5path=None, h5domain=None, if h5path: link_json["h5path"] = h5path if h5domain: - link_json["h5domain"] = h5domain + link_json["file"] = h5domain + if created: + link_json["created"] = created try: link_class = getLinkClass(link_json) @@ -525,8 +551,7 @@ async def putLinks(app, group_id, items, bucket=None): """ create a new links. Return 201 if any item is a new link, or 200 if it's a duplicate of an existing link. """ - isValidUuid(group_id, obj_class="group") - group_json = None + isValidUuid(group_id, obj_class="groups") # validate input for title in items: @@ -539,25 +564,23 @@ async def putLinks(app, group_id, items, bucket=None): raise HTTPBadRequest(reason="invalid link") if link_class == "H5L_TYPE_HARD": + if "id" not in item: + msg = "expected id key for hard link class" + log.warn(msg) + raise HTTPBadRequest(reason=msg) tgt_id = item["id"] - isValidUuid(tgt_id) - # for hard links, verify that the referenced id exists and is in - # this domain - ref_json = await getObjectJson(app, tgt_id, bucket=bucket) - if not group_json: - # just need to fetch this once - group_json = await getObjectJson(app, group_id, bucket=bucket) - if ref_json["root"] != group_json["root"]: - msg = "Hard link must reference an object in the same domain" + try: + isValidUuid(tgt_id) + except ValueError: + msg = f"invalid object id: {tgt_id}" log.warn(msg) raise HTTPBadRequest(reason=msg) # ready to add links now req = getDataNodeUrl(app, group_id) req += "/groups/" + group_id + "/links" - log.debug(f"PUT links - PUT request: {req}") + log.debug(f"PUT links {len(items)} items - PUT request: {req}") params = {"bucket": bucket} - data = {"links": items} put_rsp = await http_put(app, req, data=data, params=params) @@ -636,7 +659,7 @@ async def getObjectIdByPath(app, obj_id, h5path, bucket=None, refresh=False, dom raise HTTPBadRequest(reason=msg) # find domain object is stored under - domain = link_json["h5domain"] + domain = getLinkFilePath(link_json) if domain.startswith("hdf5:/"): # strip off prefix @@ -833,8 +856,32 @@ async def getRootInfo(app, root_id, bucket=None): return info_json +async def getDomainObjs(app, root_id, bucket=None): + """ Return domain objects if available for this root id """ + log.debug(f"getDomainObjs {root_id}") + + s3_key = getS3Key(root_id) + + parts = s3_key.split("/") + # dset_key is in the format db//d//.dataset.json + # get the key for the root info object as: db//.summary.json + if len(parts) != 3: + log.error(f"Unexpected s3key format: {s3_key}") + return None + + summary_key = f"db/{parts[1]}/.summary.json" + + try: + summary_json = await getStorJSONObj(app, summary_key, bucket=bucket) + except HTTPNotFound: + log.warn(f".summary.json not found for key: {summary_key}") + return None + + return summary_json + + async def doFlush(app, root_id, bucket=None): - """return wnen all DN nodes have wrote any pending changes to S3""" + """return wnen all DN nodes have wrote any pending changes to S3 """ log.info(f"doFlush {root_id}") params = {"flush": 1} if bucket: @@ -886,6 +933,245 @@ async def doFlush(app, root_id, bucket=None): return dn_ids +async def getTypeFromRequest(app, body, obj_id=None, bucket=None): + """ return a type json from the request body """ + if "type" not in body: + msg = "expected type in body" + log.warn(msg) + raise HTTPBadRequest(reason=msg) + datatype = body["type"] + + if isinstance(datatype, str) and datatype.startswith("t-"): + # Committed type - fetch type json from DN + ctype_id = datatype + log.debug(f"got ctypeid: {ctype_id}") + ctype_json = await getObjectJson(app, ctype_id, bucket=bucket) + log.debug(f"ctype {ctype_id}: {ctype_json}") + root_id = getRootObjId(obj_id) + if ctype_json["root"] != root_id: + msg = "Referenced committed datatype must belong in same domain" + log.warn(msg) + raise HTTPBadRequest(reason=msg) + datatype = ctype_json["type"] + # add the ctype_id to the type + datatype["id"] = ctype_id + elif isinstance(datatype, str): + try: + # convert predefined type string (e.g. "H5T_STD_I32LE") to + # corresponding json representation + datatype = getBaseTypeJson(datatype) + except TypeError: + msg = "PUT attribute with invalid predefined type" + log.warn(msg) + raise HTTPBadRequest(reason=msg) + + try: + validateTypeItem(datatype) + except KeyError as ke: + msg = f"KeyError creating type: {ke}" + log.warn(msg) + raise HTTPBadRequest(reason=msg) + except TypeError as te: + msg = f"TypeError creating type: {te}" + log.warn(msg) + raise HTTPBadRequest(reason=msg) + except ValueError as ve: + msg = f"ValueError creating type: {ve}" + log.warn(msg) + raise HTTPBadRequest(reason=msg) + + return datatype + + +def getShapeFromRequest(body): + """ get shape json from request body """ + shape_json = {} + if "shape" in body: + shape_body = body["shape"] + shape_class = None + if isinstance(shape_body, dict) and "class" in shape_body: + shape_class = shape_body["class"] + elif isinstance(shape_body, str): + shape_class = shape_body + if shape_class: + if shape_class == "H5S_NULL": + shape_json["class"] = "H5S_NULL" + if isinstance(shape_body, dict) and "dims" in shape_body: + msg = "can't include dims with null shape" + log.warn(msg) + raise HTTPBadRequest(reason=msg) + if isinstance(shape_body, dict) and "value" in shape_body: + if shape_body["value"] is not None: + msg = "can't have H5S_NULL shape with value" + log.warn(msg) + raise HTTPBadRequest(reason=msg) + elif shape_class == "H5S_SCALAR": + shape_json["class"] = "H5S_SCALAR" + if "dims" in shape_body: + msg = "dimensions aren't valid for scalar attribute" + log.warn(msg) + raise HTTPBadRequest(reason=msg) + elif shape_class == "H5S_SIMPLE": + shape_json["class"] = "H5S_SIMPLE" + dims = getShapeDims(shape_body) + shape_json["dims"] = dims + else: + msg = f"Unknown shape class: {shape_class}" + log.warn(msg) + raise HTTPBadRequest(reason=msg) + else: + # no class, interpret shape value as dimensions and + # use H5S_SIMPLE as class + if isinstance(shape_body, list) and len(shape_body) == 0: + shape_json["class"] = "H5S_SCALAR" + else: + shape_json["class"] = "H5S_SIMPLE" + dims = getShapeDims(shape_body) + shape_json["dims"] = dims + else: + shape_json["class"] = "H5S_SCALAR" + + return shape_json + + +async def getAttributeFromRequest(app, req_json, obj_id=None, bucket=None): + """ return attribute from given request json """ + attr_item = {} + log.debug(f"getAttributeFromRequest req_json: {req_json} obj_id: {obj_id}") + attr_type = await getTypeFromRequest(app, req_json, obj_id=obj_id, bucket=bucket) + attr_shape = getShapeFromRequest(req_json) + attr_item = {"type": attr_type, "shape": attr_shape} + attr_value = getValueFromRequest(req_json, attr_type, attr_shape) + if attr_value is not None: + if isinstance(attr_value, bytes): + attr_value = encodeData(attr_value) # store as base64 + attr_item["encoding"] = "base64" + else: + # just store the JSON dict or primitive value + attr_item["value"] = attr_value + else: + attr_item["value"] = None + + now = getNow() + if "created" in req_json: + created = req_json["created"] + # allow "pre-dated" attributes if the timestamp is within the last 10 seconds + predate_max_time = config.get("predate_max_time", default=10.0) + if now - created < predate_max_time: + attr_item["created"] = created + else: + msg = "stale created timestamp for attribute, ignoring " + msg += f"predate config: {predate_max_time:6.2f} " + msg += f"age: {(now - created):6.2f}" + log.warn(msg) + if "created" not in attr_item: + attr_item["created"] = now + + return attr_item + + +async def getAttributesFromRequest(app, req_json, obj_id=None, bucket=None): + """ read the given JSON dictionary and return dict of attribute json """ + + attr_items = {} + kwargs = {"obj_id": obj_id} + if bucket: + kwargs["bucket"] = bucket + if "attributes" in req_json: + attributes = req_json["attributes"] + if not isinstance(attributes, dict): + msg = f"expected list for attributes but got: {type(attributes)}" + log.warn(msg) + raise HTTPBadRequest(reason=msg) + # read each attr_item and canonicalize the shape, type, verify value + for attr_name in attributes: + attr_json = attributes[attr_name] + attr_item = await getAttributeFromRequest(app, attr_json, **kwargs) + attr_items[attr_name] = attr_item + else: + log.debug(f"getAttributesFromRequest - no attribute defined in {req_json}") + + return attr_items + + +def getValueFromRequest(body, data_type, data_shape): + """ Get attribute value from request json """ + dims = getShapeDims(data_shape) + if "value" in body and body["value"] is not None: + if dims is None: + msg = "Bad Request: data can not be included with H5S_NULL space" + log.warn(msg) + raise HTTPBadRequest(reason=msg) + value = body["value"] + # validate that the value agrees with type/shape + arr_dtype = createDataType(data_type) # np datatype + if len(dims) == 0: + np_dims = [1, ] + else: + np_dims = dims + + if "encoding" in body: + encoding = body["encoding"] + log.debug(f"using encoding: {encoding}") + item_size = getItemSize(data_type) + if item_size == "H5T_VARIABLE": + msg = "base64 encoding is not support for variable length attributes" + log.warn(msg) + raise HTTPBadRequest(reason=msg) + try: + data = decodeData(value) + except ValueError: + msg = "unable to decode data" + log.warn(msg) + raise HTTPBadRequest(reason=msg) + + expected_byte_count = arr_dtype.itemsize * np.prod(dims) + if len(data) != expected_byte_count: + msg = f"expected: {expected_byte_count} but got: {len(data)}" + log.warn(msg) + raise HTTPBadRequest(reason=msg) + + # check to see if this works with our shape and type + try: + arr = bytesToArray(data, arr_dtype, np_dims) + except ValueError as e: + log.debug(f"data: {data}") + log.debug(f"type: {arr_dtype}") + log.debug(f"np_dims: {np_dims}") + msg = f"Bad Request: encoded input data doesn't match shape and type: {e}" + log.warn(msg) + raise HTTPBadRequest(reason=msg) + + value_json = None + # now try converting to JSON + try: + value_json = bytesArrayToList(arr) + except ValueError as err: + msg = f"Cannot decode bytes to list: {err}, will store as encoded bytes" + log.warn(msg) + if value_json: + log.debug("will store base64 input as json") + if data_shape["class"] == "H5S_SCALAR": + # just use the scalar value + value = value_json[0] + else: + value = value_json # return this + else: + value = data # return bytes to signal that this needs to be encoded + else: + # verify that the input data matches the array shape and type + try: + jsonToArray(np_dims, arr_dtype, value) + except ValueError as e: + msg = f"Bad Request: input data doesn't match selection: {e}" + log.warn(msg) + raise HTTPBadRequest(reason=msg) + else: + value = None + + return value + + async def getAttributes(app, obj_id, attr_names=None, include_data=False, @@ -973,7 +1259,7 @@ async def putAttributes(app, req = getDataNodeUrl(app, obj_id) collection = getCollectionForId(obj_id) req += f"/{collection}/{obj_id}/attributes" - log.info(f"putAttribute: {req}") + log.info(f"putAttributes: {req}") params = {} if replace: @@ -1044,99 +1330,372 @@ async def deleteObject(app, obj_id, bucket=None): del meta_cache[obj_id] # remove from cache -async def createObject(app, - root_id=None, - obj_type=None, - obj_shape=None, - layout=None, - creation_props=None, - bucket=None): - """ create a group, ctype, or dataset object and return object json - Determination on whether a group, ctype, or dataset is created is based on: - 1) if obj_type and obj_shape are set, a dataset object will be created - 2) if obj_type is set but not obj_shape, a datatype object will be created - 3) otherwise (type and shape are both None), a group object will be created - The layout parameter only applies to dataset creation - """ - if obj_type and obj_shape: - collection = "datasets" - elif obj_type: - collection = "datatypes" +def getCreateArgs(body, + root_id=None, + bucket=None, + type=None, + implicit=False, + chunk_table=None, + ignore_link=False): + """ get args for createObject from request body """ + + log.debug(f"getCreateArgs with body keys: {list(body.keys())}") + if ignore_link: + log.debug("getCreateArgs, ignore_link is set") + + kwargs = {"bucket": bucket} + predate_max_time = config.get("predate_max_time", default=10.0) + + parent_id = None + obj_id = None + h5path = None + + if "parent_id" not in body: + parent_id = root_id else: - collection = "groups" - log.info(f"createObject for {collection} collection, root: {root_id}, bucket: {bucket}") - if obj_type: - log.debug(f" obj_type: {obj_type}") - if obj_shape: - log.debug(f" obj_shape: {obj_shape}") - if layout: - log.debug(f" layout: {layout}") - if creation_props: - log.debug(f" cprops: {creation_props}") + parent_id = body["parent_id"] + + if "h5path" in body: + h5path = body["h5path"] + # normalize the h5path + if h5path.startswith("/"): + if parent_id == root_id: + # just adjust the path to be relative + h5path = h5path[1:] + else: + msg = f"PostCrawler expecting relative h5path, but got: {h5path}" + log.warn(msg) + raise HTTPBadRequest(reason=msg) - obj_id = createObjId(collection, rootid=root_id) - log.info(f"new obj id: {obj_id}") - obj_json = {"id": obj_id, "root": root_id} - if obj_type: - obj_json["type"] = obj_type - if obj_shape: - obj_json["shape"] = obj_shape - if layout: - obj_json["layout"] = layout - if creation_props: - obj_json["creationProperties"] = creation_props - log.debug(f"create {collection} obj, body: {obj_json}") - dn_url = getDataNodeUrl(app, obj_id) - req = f"{dn_url}/{collection}" - params = {"bucket": bucket} - rsp_json = await http_post(app, req, data=obj_json, params=params) + if h5path.endswith("/"): + h5path = h5path[:-1] # makes iterating through the links a bit easier - return rsp_json + if "link" in body: + if "h5path" in body: + msg = "'link' key in body can't be used with h5path" + log.warn(msg) + raise HTTPBadRequest(reason=msg) + # if ignore_link is set, parent_links will be created post object creation + link_body = body["link"] + log.debug(f"link_body: {link_body}") + if "id" in link_body and not ignore_link: + parent_id = link_body["id"] + if "name" in link_body: + link_title = link_body["name"] + try: + # will throw exception if there's a slash in the name + validateLinkName(link_title) + except ValueError: + msg = f"invalid link title: {link_title}" + log.warn(msg) + raise HTTPBadRequest(reason=msg) + if parent_id and link_title: + log.debug(f"parent id: {parent_id}, link_title: {link_title}") + if not ignore_link: + h5path = link_title # just use the link name as the h5path + log.debug(f"set h5path to {link_title}") + + if parent_id and h5path: + # these are used by createObjectByPath + kwargs["parent_id"] = parent_id + kwargs["implicit"] = implicit + kwargs["h5path"] = h5path + else: + kwargs["root_id"] = root_id -async def createObjectByPath(app, - parent_id=None, - h5path=None, - implicit=False, - obj_type=None, - obj_shape=None, - layout=None, - creation_props=None, - bucket=None): + if "id" in body: + obj_id = body["id"] + if not isValidUuid(obj_id): + msg = f"Invalid id: {obj_id}" + log.warn(msg) + raise HTTPBadRequest(reason=msg) - """ create an object at the designated path relative to the parent. - If implicit is True, make any intermediate groups needed in the h5path. """ + kwargs["obj_id"] = obj_id + log.debug(f"createObject will use client id: {obj_id}") - if not parent_id: - msg = "no parent_id given for createObjectByPath" + if "creationProperties" in body: + creation_props = body["creationProperties"] + # validate after we've checked for shape and type + else: + creation_props = {} + kwargs["creation_props"] = creation_props + + if "attributes" in body: + attrs = body["attributes"] + if not isinstance(attrs, dict): + msg = f"expected dict for for attributes, but got: {type(attrs)}" + log.warn(msg) + raise HTTPBadRequest(reason=msg) + log.debug(f"createObject attributes: {attrs}") + + # tbd: validate attributes + kwargs["attrs"] = attrs + + if "links" in body: + body_links = body["links"] + log.debug(f"got links for new group: {body_links}") + try: + links = getRequestLinks(body["links"], predate_max_time=predate_max_time) + except ValueError: + msg = "invalid link item sent in request" + raise HTTPBadRequest(reason=msg) + log.debug(f"adding links to createObject request: {links}") + kwargs["links"] = links + + if type: + kwargs["type"] = type + type_json = type + elif "type" in body: + type_json = body["type"] + if isinstance(type_json, str): + try: + # convert predefined type string (e.g. "H5T_STD_I32LE") to + # corresponding json representation + type_json = getBaseTypeJson(type_json) + log.debug(f"got type: {type_json}") + except TypeError: + msg = f"POST with invalid predefined type: {type_json}" + log.warn(msg) + raise HTTPBadRequest(reason=msg) + else: + type_json = None + + if type_json: + try: + validateTypeItem(type_json) + except KeyError as ke: + msg = f"KeyError creating type: {ke}" + log.warn(msg) + raise HTTPBadRequest(reason=msg) + except TypeError as te: + msg = f"TypeError creating type: {te}" + log.warn(msg) + raise HTTPBadRequest(reason=msg) + except ValueError as ve: + msg = f"ValueError creating type: {ve}" + log.warn(msg) + raise HTTPBadRequest(reason=msg) + kwargs["type"] = type_json + else: + pass # no type + return kwargs + + +def genLayout(shape_json, type_json, has_filters=False): + """ create a chunked or contiguous layout based on shape and itemsize """ + + min_chunk_size = int(config.get("min_chunk_size")) + max_chunk_size = int(config.get("max_chunk_size")) + max_chunks_per_folder = int(config.get("max_chunks_per_folder", default=0)) + kwargs = {"chunk_min": min_chunk_size, "chunk_max": max_chunk_size} + max_chunks_per_folder = int(config.get("max_chunks_per_folder")) + if max_chunks_per_folder > 0: + kwargs["max_chunks_per_folder"] = max_chunks_per_folder + if has_filters: + kwargs["chunks"] = True # force a chunked layout to support compression + + layout_json = generateLayout(shape_json, type_json, **kwargs) + return layout_json + + +def getDatasetCreateArgs(body, + root_id=None, + bucket=None, + type=None, + implicit=False, + ignore_link=False): + + """ get args for createDataset from request body """ + + # call getCreateArgs for group, datatype objects, then fill in for dataset specific options + kwargs = getCreateArgs(body, + root_id=root_id, + bucket=bucket, + type=type, + implicit=implicit, + ignore_link=ignore_link) + + if "type" not in kwargs: + msg = "no type specified for create dataset" log.warn(msg) raise HTTPBadRequest(reason=msg) - if not h5path: - msg = "no h5path given for createObjectByPath" - log.warn(msg) + + type_json = kwargs["type"] + # + # Validate shape if present + # + + # will return scalar shape if no shape key in body + dims = () # default to scalar shape + maxdims = None + if "shape" in body: + body_shape = body["shape"] + if isinstance(body_shape, int): + dims = [body_shape, ] + elif isinstance(body_shape, str): + # only valid string value is H5S_NULL or H5S_SCALAR + if body_shape == "H5S_NULL": + dims = None # use None for null space + elif body_shape == "H5S_SCALAR": + pass # keep empty tuple for scalar dims + else: + msg = f"invalid value for 'shape' key: {body_shape}" + log.warn(msg) + raise HTTPBadRequest(reason=msg) + elif isinstance(body_shape, (list, tuple)): + dims = body_shape + else: + msg = f"Unexpected type for 'shape' key: {body_shape}" + log.warn(msg) + raise HTTPBadRequest(reason=msg) + + if "maxdims" in body: + maxdims = body["maxdims"] + if isinstance(maxdims, int): + maxdims = [maxdims,] + + try: + shape_json = getShapeJson(dims, maxdims=maxdims) + except (TypeError, ValueError) as e: + if maxdims is None: + msg = f"Invalid dimensions for dataset: {dims}" + else: + msg = f"Invalid dims: {dims}/maxdims: {maxdims} for dataset" + log.warn(f"Error in getShapeJson: {e}, msg: {msg}") raise HTTPBadRequest(reason=msg) - log.debug(f"createObjectByPath - parent_id: {parent_id}, h5path: {h5path}") - root_id = getRootObjId(parent_id) + shape_class = getShapeClass(shape_json) + shape_dims = getShapeDims(shape_json) + log.debug(f"shape_class: {shape_class}, shape_dims: {shape_dims}") + + log.debug(f"got createArgs: {list(kwargs.keys())}") + + kwargs["shape"] = shape_json + + # get layout for dataset creation + log.debug("getting dataset creation settings") + min_chunk_size = int(config.get("min_chunk_size")) + max_chunk_size = int(config.get("max_chunk_size")) + type_json = kwargs["type"] + + item_size = getItemSize(type_json) + if item_size == "H5T_VARIABLE": + item_size = config.get("default_vlen_type_size", default=128) + if shape_dims is None: + dset_size = 0 + else: + dset_size = getDataSize(shape_dims, item_size) + + if "creationProperties" in body: + creation_props = body["creationProperties"] + else: + creation_props = None + layout_json = None + chunk_dims = None + has_filters = False - if h5path.startswith("/"): - if parent_id == root_id: - # just adjust the path to be relative - h5path = h5path[1:] + if creation_props: + log.debug(f"POST_Dataset creation props: {creation_props}") + try: + validateDatasetCreationProps(creation_props, type_json=type_json, shape=shape_json) + except (KeyError, TypeError, ValueError) as e: + msg = f"Provided creation properties are invalid: {e}" + log.warn(msg) + raise HTTPBadRequest(reason=msg) + log.debug(f"create_props after validation: {creation_props}") + + if "filters" in creation_props: + # check that the given filters are supported by HSDS + filters = creation_props["filters"] + supported_filters = getSupportedFilters() + log.debug(f"supported filters: {supported_filters}") + for filter_item in filters: + if filter_item["name"] not in supported_filters: + msg = f"Unsupported filter id: {filter_item['id']}" + log.warn(msg) + raise HTTPBadRequest(reason=msg) + if filters: + has_filters = True + creation_props["filters"] = filters + log.debug(f"post validate creation properties: {creation_props}") + if "layout" in creation_props: + layout_json = creation_props["layout"] + else: + creation_props = {} + + if "layout" in creation_props: + layout_json = creation_props["layout"] + # layout_json was validated in validateDatasetCreationProps, but issue + # a warning if the chunk size is outside suggested range + if "dims" in layout_json: + chunk_dims = layout_json["dims"] + # log warning if the chunk shape if chunk size is too small or too big + chunk_size = getChunkSize(chunk_dims, item_size) + if chunk_size < min_chunk_size: + msg = f"chunk size: {chunk_size} less than recommended min size: {min_chunk_size}" + log.warn(msg) + elif chunk_size > max_chunk_size: + msg = f"chunk size: {chunk_size} greater than recommended " + msg += f"max size: {max_chunk_size}" + log.debug(msg) else: - msg = f"createObjectByPath expecting relative h5path, but got: {h5path}" + # log warning if contiguous layout used with too large datadset + if dset_size > max_chunk_size: + msg = f"dataset larger than recommended {max_chunk_size} for CONTIGUOUS storage" + log.warn(msg) + else: + # no layout, create one based on shape and itemsize + layout_json = genLayout(shape_json, type_json, has_filters=has_filters) + log.info(f"created chunk layout for new dset: {layout_json}") + creation_props["layout"] = layout_json + + kwargs["creation_props"] = creation_props + log.debug(f"updated creation props: {creation_props}") + + # + # get input data if present + # + if "value" in body and body["value"] and body["value"] is not None: + # data to initialize dataset included in request + if shape_json["class"] == "H5S_NULL": + msg = "null shape datasets can not have initial values" log.warn(msg) raise HTTPBadRequest(reason=msg) - if h5path.endswith("/"): - h5path = h5path[:-1] # makes iterating through the links a bit easier + input_data = body["value"] + msg = "input data doesn't match request type and shape" + dims = getShapeDims(shape_json) - if not h5path: - msg = "h5path for createObjectByPath invalid" - log.warn(msg) - raise HTTPBadRequest(reason=msg) + arr_dtype = createDataType(type_json) - obj_json = None + try: + input_arr = jsonToArray(dims, arr_dtype, input_data) + except ValueError: + log.warn(f"ValueError: {msg}") + raise HTTPBadRequest(reason=msg) + except TypeError: + log.warn(f"TypeError: {msg}") + raise HTTPBadRequest(reason=msg) + except IndexError: + log.warn(f"IndexError: {msg}") + raise HTTPBadRequest(reason=msg) + log.debug(f"got json arr: {input_arr.shape}") + kwargs["value"] = input_data + + return kwargs + + +async def createLinkFromParent(app, parent_id, h5path, tgt_id=None, bucket=None, implicit=False): + """ create link or links from parentId to tgt_id. + If implicit is True, create any intermediate group objects needed """ + + if not h5path: + log.warn("createLinkFromParent with null h5path") + return + log.info(f"createLinkFromParent, parent_id: {parent_id} h5path: {h5path} tgt_id={tgt_id}") + if implicit: + log.debug("createLinkFromParent - using implicit creation") link_titles = h5path.split("/") log.debug(f"link_titles: {link_titles}") for i in range(len(link_titles)): @@ -1145,7 +1704,7 @@ async def createObjectByPath(app, else: last_link = False link_title = link_titles[i] - log.debug(f"createObjectByPath - processing link: {link_title}") + log.debug(f"createLinkFromParent - processing link: {link_title}") link_json = None try: link_json = await getLink(app, parent_id, link_title, bucket=bucket) @@ -1161,7 +1720,7 @@ async def createObjectByPath(app, raise HTTPConflict() # otherwise, verify that this is a hardlink if link_json.get("class") != "H5L_TYPE_HARD": - msg = "createObjectByPath - h5path must contain only hardlinks" + msg = "createLinkFromParent - h5path must contain only hard links" log.warn(msg) raise HTTPBadRequest(reason=msg) parent_id = link_json["id"] @@ -1174,31 +1733,236 @@ async def createObjectByPath(app, log.debug(f"link: {link_title} to sub-group found") else: log.debug(f"link for link_title {link_title} not found") - if not last_link and not implicit: + if last_link: + # create a link to the new object + await putHardLink(app, parent_id, link_title, tgt_id=tgt_id, bucket=bucket) + parent_id = tgt_id # new parent + elif implicit: + # create a new group object + log.info(f"creating intermediate group object for: {link_title}") + kwargs = {"parent_id": parent_id, "bucket": bucket} + grp_id = createObjId("groups", root_id=getRootObjId(parent_id)) + kwargs["obj_id"] = grp_id + # createObject won't call back to this function since we haven't set the h5path + await createObject(app, **kwargs) + # create a link to the subgroup + await putHardLink(app, parent_id, link_title, tgt_id=grp_id, bucket=bucket) + parent_id = grp_id # new parent + else: if len(link_titles) > 1: - msg = f"createObjectByPath failed: not all groups in {h5path} exist" + msg = f"createLinkFromParent failed: not all groups in {h5path} exist" else: - msg = f"createObjectByPath failed: {h5path} does not exist" + msg = f"createLinkFromParent failed: {h5path} does not exist" log.warn(msg) raise HTTPNotFound(reason=msg) - # create the group or group/datatype/dataset for the last - # item in the path (based on parameters passed in) - kwargs = {"bucket": bucket, "root_id": root_id} - if last_link: - if obj_type: - kwargs["obj_type"] = obj_type - if obj_shape: - kwargs["obj_shape"] = obj_shape - if layout: - kwargs["layout"] = layout - if creation_props: - kwargs["creation_props"] = creation_props - obj_json = await createObject(app, **kwargs) - obj_id = obj_json["id"] - # create a link to the new object - await putHardLink(app, parent_id, link_title, tgt_id=obj_id, bucket=bucket) - parent_id = obj_id # new parent - log.info(f"createObjectByPath {h5path} done, returning obj_json") - return obj_json +async def createObject(app, + parent_id=None, + root_id=None, + h5path=None, + obj_id=None, + type=None, + shape=None, + creation_props=None, + attrs=None, + links=None, + implicit=None, + bucket=None): + """ create a group, ctype, or dataset object and return object json + Determination on whether a group, ctype, or dataset is created is based on: + 1) if type and shape are set, a dataset object will be created + 2) if type is set but not shape, a datatype object will be created + 3) otherwise (type and shape are both None), a group object will be created + The layout parameter only applies to dataset creation + """ + if type and shape: + collection = "datasets" + elif type: + collection = "datatypes" + else: + collection = "groups" + + if not root_id: + root_id = getRootObjId(parent_id) + log.info(f"createObject for {collection} collection, root_id: {root_id}, bucket: {bucket}") + if root_id != parent_id: + log.debug(f" parent_id: {parent_id}") + if obj_id: + log.debug(f" obj_id: {obj_id}") + if h5path: + log.debug(f" h5path: {h5path}") + if type: + log.debug(f" type: {type}") + if shape: + log.debug(f" shape: {shape}") + if creation_props: + log.debug(f" cprops: {creation_props}") + if attrs: + log.debug(f" attrs: {attrs}") + if links: + log.debug(f" links: {links}") + + if h5path: + if h5path.startswith("/"): + if parent_id == root_id: + # just adjust the path to be relative + h5path = h5path[1:] + else: + msg = f"createObject expecting relative h5path, but got: {h5path}" + log.warn(msg) + raise HTTPBadRequest(reason=msg) + + if h5path.endswith("/"): + h5path = h5path[:-1] # makes iterating through the links a bit easier + + if obj_id: + log.debug(f"using client supplied id: {obj_id}") + if not isValidUuid(obj_id, obj_class=collection): + msg = f"invalid id: {obj_id}" + log.warn(msg) + raise HTTPBadRequest(reason=msg) + if getRootObjId(obj_id) != root_id: + msg = f"id: {obj_id} is not valid for root: {root_id}" + log.warn(msg) + raise HTTPBadRequest(reason=msg) + else: + obj_id = createObjId(collection, root_id=root_id) + log.info(f"new obj id: {obj_id}") + obj_json = {"id": obj_id, "root": root_id} + if type: + obj_json["type"] = type + if shape: + obj_json["shape"] = shape + if creation_props: + obj_json["creationProperties"] = creation_props + else: + obj_json["creationProperties"] = {} + if attrs: + kwargs = {"obj_id": obj_id, "bucket": bucket} + attrs_json = {"attributes": attrs} + attr_items = await getAttributesFromRequest(app, attrs_json, **kwargs) + log.debug(f"got attr_items: {attr_items}") + obj_json["attributes"] = attr_items + if links: + if collection != "groups": + msg = "links can only be used with groups" + log.warn(msg) + raise HTTPBadRequest(reason=msg) + obj_json["links"] = links + log.debug(f"create {collection} obj, body: {obj_json}") + dn_url = getDataNodeUrl(app, obj_id) + req = f"{dn_url}/{collection}" + params = {"bucket": bucket} + rsp_json = await http_post(app, req, data=obj_json, params=params) + + log.debug(f"createObject: {req} got rsp_json: {rsp_json}") + + # object creation successful, create link from parent if requested + if h5path: + kwargs = {"tgt_id": obj_id, "bucket": bucket, "implicit": implicit} + await createLinkFromParent(app, parent_id, h5path, **kwargs) + + return rsp_json + + +async def createGroup(app, + parent_id=None, + root_id=None, + h5path=None, + obj_id=None, + creation_props=None, + attrs=None, + links=None, + implicit=None, + bucket=None): + + """ create a new group object """ + + kwargs = {} + kwargs["parent_id"] = parent_id + kwargs["root_id"] = root_id + kwargs["h5path"] = h5path + kwargs["obj_id"] = obj_id + kwargs["creation_props"] = creation_props + kwargs["attrs"] = attrs + kwargs["links"] = links + kwargs["implicit"] = implicit + kwargs["bucket"] = bucket + rsp_json = await createObject(app, **kwargs) + return rsp_json + + +async def createDatatypeObj(app, + parent_id=None, + root_id=None, + type=None, + h5path=None, + obj_id=None, + creation_props=None, + attrs=None, + links=None, + implicit=None, + bucket=None): + + """ create a new committed type object""" + + if not type: + msg = "type not set for committed type creation" + log.warn(msg) + raise HTTPBadRequest(reason=msg) + + kwargs = {} + kwargs["parent_id"] = parent_id + kwargs["root_id"] = root_id + kwargs["type"] = type + kwargs["h5path"] = h5path + kwargs["obj_id"] = obj_id + kwargs["creation_props"] = creation_props + kwargs["attrs"] = attrs + kwargs["links"] = links + kwargs["implicit"] = implicit + kwargs["bucket"] = bucket + rsp_json = await createObject(app, **kwargs) + return rsp_json + + +async def createDataset(app, + parent_id=None, + root_id=None, + type=None, + shape=None, + h5path=None, + obj_id=None, + creation_props=None, + attrs=None, + links=None, + implicit=None, + bucket=None): + + """ create a new dataset object""" + + if not type: + msg = "type not set for dataset creation" + log.warn(msg) + raise HTTPBadRequest(reason=msg) + + if not shape: + # default to a scalar dataset + shape = {"class": "H5S_SCALAR"} + + kwargs = {} + kwargs["parent_id"] = parent_id + kwargs["root_id"] = root_id + kwargs["type"] = type + kwargs["shape"] = shape + kwargs["h5path"] = h5path + kwargs["obj_id"] = obj_id + kwargs["creation_props"] = creation_props + kwargs["attrs"] = attrs + kwargs["links"] = links + kwargs["implicit"] = implicit + kwargs["bucket"] = bucket + dset_json = await createObject(app, **kwargs) + + return dset_json diff --git a/hsds/util/arrayUtil.py b/hsds/util/arrayUtil.py deleted file mode 100644 index 67c847c3..00000000 --- a/hsds/util/arrayUtil.py +++ /dev/null @@ -1,731 +0,0 @@ -############################################################################## -# Copyright by The HDF Group. # -# All rights reserved. # -# # -# This file is part of HSDS (HDF5 Scalable Data Service), Libraries and # -# Utilities. The full HSDS copyright notice, including # -# terms governing use, modification, and redistribution, is contained in # -# the file COPYING, which can be found at the root of the source code # -# distribution tree. If you do not have access to this file, you may # -# request a copy from help@hdfgroup.org. # -############################################################################## - -import math -import base64 -import binascii -import numpy as np - -MAX_VLEN_ELEMENT = 1_000_000 # restrict largest vlen element to one million - - -def bytesArrayToList(data): - """ - Convert list that may contain bytes type elements to list of string elements - - TBD: Need to deal with non-string byte data (hexencode?) - """ - if type(data) in (bytes, str): - is_list = False - elif isinstance(data, (np.ndarray, np.generic)): - if len(data.shape) == 0: - is_list = False - data = data.tolist() # tolist will return a scalar in this case - if type(data) in (list, tuple): - is_list = True - else: - is_list = False - else: - is_list = True - elif type(data) in (list, tuple): - is_list = True - else: - is_list = False - - if is_list: - out = [] - for item in data: - try: - rec_item = bytesArrayToList(item) # recursive call - out.append(rec_item) - except ValueError as err: - raise err - elif type(data) is bytes: - try: - out = data.decode("utf-8") - except UnicodeDecodeError as err: - raise ValueError(err) - else: - out = data - - return out - - -def toTuple(rank, data): - """ - Convert a list to a tuple, recursively. - Example. [[1,2],[3,4]] -> ((1,2),(3,4)) - """ - if type(data) in (list, tuple): - if rank > 0: - return list(toTuple(rank - 1, x) for x in data) - else: - return tuple(toTuple(rank - 1, x) for x in data) - else: - if isinstance(data, str): - data = data.encode("utf8") - return data - - -def getArraySize(arr): - """ - Get size in bytes of a numpy array. - """ - nbytes = arr.dtype.itemsize - for n in arr.shape: - nbytes *= n - return nbytes - - -def getNumElements(dims): - """ - Get num elements defined by a shape - """ - num_elements = 0 - if isinstance(dims, int): - num_elements = dims - elif isinstance(dims, (list, tuple)): - num_elements = 1 - for dim in dims: - num_elements *= dim - else: - raise ValueError("Unexpected argument") - return num_elements - - -def isVlen(dt): - """ - Return True if the type contains variable length elements - """ - is_vlen = False - if len(dt) > 1: - names = dt.names - for name in names: - if isVlen(dt[name]): - is_vlen = True - break - else: - if dt.metadata and "vlen" in dt.metadata: - is_vlen = True - return is_vlen - - -def jsonToArray(data_shape, data_dtype, data_json): - """ - Return numpy array from the given json array. - """ - def fillVlenArray(rank, data, arr, index): - for i in range(len(data)): - if rank > 1: - index = fillVlenArray(rank - 1, data[i], arr, index) - else: - arr[index] = data[i] - index += 1 - return index - - if data_json is None: - return np.array([]).astype(data_dtype) - - if isinstance(data_json, (list, tuple)): - if None in data_json: - return np.array([]).astype(data_dtype) - - # need some special conversion for compound types -- - # each element must be a tuple, but the JSON decoder - # gives us a list instead. - if len(data_dtype) > 1 and not isinstance(data_json, (list, tuple)): - raise TypeError("expected list data for compound data type") - npoints = getNumElements(data_shape) - np_shape_rank = len(data_shape) - - if type(data_json) in (list, tuple): - converted_data = [] - if npoints == 1 and len(data_json) == len(data_dtype): - converted_data.append(toTuple(0, data_json)) - else: - converted_data = toTuple(np_shape_rank, data_json) - data_json = converted_data - else: - if isinstance(data_json, str): - data_json = data_json.encode("utf8") - data_json = [data_json,] # listify - - if isVlen(data_dtype): - arr = np.zeros((npoints,), dtype=data_dtype) - fillVlenArray(np_shape_rank, data_json, arr, 0) - else: - try: - arr = np.array(data_json, dtype=data_dtype) - except UnicodeEncodeError as ude: - msg = "Unable to encode data" - raise ValueError(msg) from ude - # raise an exception of the array shape doesn't match the selection shape - # allow if the array is a scalar and the selection shape is one element, - # numpy is ok with this - if arr.size != npoints: - msg = "Input data doesn't match selection number of elements" - msg += f" Expected {npoints}, but received: {arr.size}" - raise ValueError(msg) - if arr.shape != data_shape: - arr = arr.reshape(data_shape) # reshape to match selection - - return arr - - -def getElementSize(e, dt): - """ - Get number of byte needed to given element as a bytestream - """ - # print(f"getElementSize - e: {e} dt: {dt} metadata: {dt.metadata}") - if len(dt) > 1: - count = 0 - for name in dt.names: - field_dt = dt[name] - field_val = e[name] - count += getElementSize(field_val, field_dt) - elif not dt.metadata or "vlen" not in dt.metadata: - count = dt.itemsize # fixed size element - else: - # variable length element - vlen = dt.metadata["vlen"] - if isinstance(e, int): - if e == 0: - count = 4 # non-initialized element - else: - raise ValueError("Unexpected value: {}".format(e)) - elif isinstance(e, bytes): - count = len(e) + 4 - elif isinstance(e, str): - count = len(e.encode("utf-8")) + 4 - elif isinstance(e, np.ndarray): - nElements = math.prod(e.shape) - if e.dtype.kind != "O": - count = e.dtype.itemsize * nElements - else: - arr1d = e.reshape((nElements,)) - count = 0 - for item in arr1d: - count += getElementSize(item, dt) - count += 4 # byte count - elif isinstance(e, list) or isinstance(e, tuple): - if not e: - # empty list, just add byte count - count = 4 - else: - # not sure how to deal with this - count = len(e) * vlen.itemsize + 4 # +4 for byte count - else: - raise TypeError("unexpected type: {}".format(type(e))) - return count - - -def getByteArraySize(arr): - """ - Get number of bytes needed to store given numpy array as a bytestream - """ - if not isVlen(arr.dtype): - return arr.itemsize * math.prod(arr.shape) - nElements = math.prod(arr.shape) - # reshape to 1d for easier iteration - arr1d = arr.reshape((nElements,)) - dt = arr1d.dtype - count = 0 - for e in arr1d: - count += getElementSize(e, dt) - return count - - -def copyBuffer(src, des, offset): - """ - Copy to buffer at given offset - """ - # print(f"copyBuffer - src: {src} offset: {offset}") - # TBD: just do: des[offset:] = src[:] ? - for i in range(len(src)): - des[i + offset] = src[i] - - # print("returning:", offset + len(src)) - return offset + len(src) - - -def copyElement(e, dt, buffer, offset): - """ - Copy element to bytearray - """ - # print(f"copyElement - dt: {dt} offset: {offset}") - if len(dt) > 1: - for name in dt.names: - field_dt = dt[name] - field_val = e[name] - offset = copyElement(field_val, field_dt, buffer, offset) - elif not dt.metadata or "vlen" not in dt.metadata: - # print(f"e vlen: {e} type: {type(e)} itemsize: {dt.itemsize}") - e_buf = e.tobytes() - # print("tobytes:", e_buf) - if len(e_buf) < dt.itemsize: - # extend the buffer for fixed size strings - # print("extending buffer") - e_buf_ex = bytearray(dt.itemsize) - for i in range(len(e_buf)): - e_buf_ex[i] = e_buf[i] - e_buf = bytes(e_buf_ex) - - # print("length:", len(e_buf)) - offset = copyBuffer(e_buf, buffer, offset) - else: - # variable length element - vlen = dt.metadata["vlen"] - # print("copyBuffer vlen:", vlen) - if isinstance(e, int): - # print("copyBuffer int") - if e == 0: - # write 4-byte integer 0 to buffer - offset = copyBuffer(b"\x00\x00\x00\x00", buffer, offset) - else: - raise ValueError("Unexpected value: {}".format(e)) - elif isinstance(e, bytes): - # print("copyBuffer bytes") - count = np.int32(len(e)) - if count > MAX_VLEN_ELEMENT: - raise ValueError("vlen element too large") - offset = copyBuffer(count.tobytes(), buffer, offset) - offset = copyBuffer(e, buffer, offset) - elif isinstance(e, str): - # print("copyBuffer, str") - text = e.encode("utf-8") - count = np.int32(len(text)) - if count > MAX_VLEN_ELEMENT: - raise ValueError("vlen element too large") - offset = copyBuffer(count.tobytes(), buffer, offset) - offset = copyBuffer(text, buffer, offset) - - elif isinstance(e, np.ndarray): - nElements = math.prod(e.shape) - # print("copyBuffer ndarray, nElements:", nElements) - - if e.dtype.kind != "O": - count = np.int32(e.dtype.itemsize * nElements) - # print("copyBuffeer got vlen count:", count) - # print("copyBuffer e:", e) - if count > MAX_VLEN_ELEMENT: - raise ValueError("vlen element too large") - offset = copyBuffer(count.tobytes(), buffer, offset) - # print("copyBuffer write new count, offset:", offset) - offset = copyBuffer(e.tobytes(), buffer, offset) - # print("copyBuffer write data, offset:", offset) - else: - arr1d = e.reshape((nElements,)) - for item in arr1d: - offset = copyElement(item, dt, buffer, offset) - - elif isinstance(e, list) or isinstance(e, tuple): - # print("cooyBuffer list/tuple vlen:", vlen, "e:", e) - count = np.int32(len(e) * vlen.itemsize) - offset = copyBuffer(count.tobytes(), buffer, offset) - if isinstance(e, np.ndarray): - arr = e - else: - arr = np.asarray(e, dtype=vlen) - offset = copyBuffer(arr.tobytes(), buffer, offset) - - else: - raise TypeError("unexpected type: {}".format(type(e))) - # print("buffer: {}".format(buffer)) - return offset - - -def getElementCount(buffer, offset=0): - """ - Get the count value from persisted vlen array - """ - - n = offset - m = offset + 4 - count_bytes = bytes(buffer[n:m]) - - try: - count = int(np.frombuffer(count_bytes, dtype=" MAX_VLEN_ELEMENT: - # expect variable length element to be between 0 and 1mb - raise ValueError("varlen element size expected to be less than 1MB") - return count - - -def readElement(buffer, offset, arr, index, dt): - """ - Read a single element from buffer into array. - - Parameters: - buffer (bytearray): Byte array to read an element from. - offset (int): Starting offset in the buffer. - arr (numpy.ndarray): Array to store the element. - index (int): Index in 'arr' at which to store the element. - dt (numpy.dtype): Numpy datatype of the element. - - Note: If the provided datatype is a variable-length sequence, - this function will read the byte count from the first 4 bytes - of the buffer, and then read the entire sequence. - - Returns: - int: The updated offset value after reading the element. - """ - if len(dt) > 1: - e = arr[index] - for name in dt.names: - field_dt = dt[name] - offset = readElement(buffer, offset, e, name, field_dt) - elif not dt.metadata or "vlen" not in dt.metadata: - count = dt.itemsize - n = offset - m = offset + count - e_buffer = buffer[n:m] - offset += count - try: - e = np.frombuffer(bytes(e_buffer), dtype=dt) - arr[index] = e[0] - except ValueError: - print(f"ERROR: ValueError setting {e_buffer} and dtype: {dt}") - raise - else: - # variable length element - vlenBaseType = dt.metadata["vlen"] - e = arr[index] - - if isinstance(e, np.ndarray): - nelements = math.prod(dt.shape) - e.reshape((nelements,)) - for i in range(nelements): - offset = readElement(buffer, offset, e, i, dt) - e.reshape(dt.shape) - else: - # total number of bytes in the vlen sequence/variable-length string - count = getElementCount(buffer, offset=offset) - offset += 4 - n = offset - m = offset + count - if count > 0: - e_buffer = buffer[n:m] - offset += count - - if vlenBaseType is bytes: - arr[index] = bytes(e_buffer) - elif vlenBaseType is str: - s = e_buffer.decode("utf-8") - arr[index] = s - else: - try: - e = np.frombuffer(bytes(e_buffer), dtype=vlenBaseType) - except ValueError: - msg = f"Failed to parse vlen data: {e_buffer} with dtype: {vlenBaseType}" - raise ValueError(msg) - arr[index] = e - return offset - - -def encodeData(data, encoding="base64"): - """ Encode given data """ - if encoding != "base64": - raise ValueError("only base64 encoding is supported") - try: - if isinstance(data, str): - data = data.encode("utf8") - except UnicodeEncodeError: - raise ValueError("can not encode string value") - if not isinstance(data, bytes): - msg = "Expected str or bytes type to encodeData, " - msg += f"but got: {type(data)}" - raise TypeError(msg) - try: - encoded_data = base64.b64encode(data) - except Exception as e: - # TBD: what exceptions can be raised? - raise ValueError(f"Unable to encode: {e}") - return encoded_data - - -def decodeData(data, encoding="base64"): - if encoding != "base64": - raise ValueError("only base64 decoding is supported") - try: - decoded_data = base64.b64decode(data) - except Exception as e: - # TBD: catch actual exception - raise ValueError(f"Unable to decode: {e}") - return decoded_data - - -def arrayToBytes(arr, encoding=None): - """ - Return byte representation of numpy array - """ - if isVlen(arr.dtype): - nSize = getByteArraySize(arr) - buffer = bytearray(nSize) - offset = 0 - nElements = math.prod(arr.shape) - arr1d = arr.reshape((nElements,)) - for e in arr1d: - # print("arrayToBytes:", e) - offset = copyElement(e, arr1d.dtype, buffer, offset) - data = bytes(buffer) - else: - # fixed length type - data = arr.tobytes() - - if encoding: - data = encodeData(data) - return data - - -def bytesToArray(data, dt, shape, encoding=None): - """ - Create numpy array based on byte representation - """ - if encoding: - # decode the data - # will raise ValueError if non-decodeable - data = decodeData(data) - if not isVlen(dt): - # regular numpy from string - arr = np.frombuffer(data, dtype=dt) - else: - nelements = getNumElements(shape) - - arr = np.zeros((nelements,), dtype=dt) - offset = 0 - for index in range(nelements): - offset = readElement(data, offset, arr, index, dt) - if shape is not None: - arr = arr.reshape(shape) - # check that we can update the array if needed - # Note: this seems to have been required starting with numpuy v 1.17 - # Setting the flag directly is not recommended. - # cf: https://github.com/numpy/numpy/issues/9440 - - if not arr.flags["WRITEABLE"]: - arr_copy = arr.copy() - arr = arr_copy - - return arr - - -def getNumpyValue(value, dt=None, encoding=None): - """ - Return value as numpy type for given dtype and encoding - Encoding is expected to be one of None or "base64" - """ - # create a scalar numpy array - arr = np.zeros((), dtype=dt) - - if encoding and not isinstance(value, str): - msg = "Expected value to be string to use encoding" - raise ValueError(msg) - - if encoding == "base64": - try: - data = base64.decodebytes(value.encode("utf-8")) - except binascii.Error: - msg = "Unable to decode base64 string: {value}" - # log.warn(msg) - raise ValueError(msg) - arr = bytesToArray(data, dt, dt.shape) - else: - if isinstance(value, list): - # convert to tuple - value = tuple(value) - elif dt.kind == "f" and isinstance(value, str) and value == "nan": - value = np.nan - else: - # use as is - pass - arr = np.asarray(value, dtype=dt.base) - return arr[()] - - -def squeezeArray(data): - """ - Reduce dimensions by removing any 1-extent dimensions. - Just return input if no 1-extent dimensions - - Note: only works with ndarrays (for now at least) - """ - if not isinstance(data, np.ndarray): - raise TypeError("expected ndarray") - if len(data.shape) <= 1: - return data - can_reduce = True - for extent in data.shape: - if extent == 1: - can_reduce = True - break - if can_reduce: - data = data.squeeze() - return data - - -class IndexIterator(object): - """ - Class to iterate through list of chunks of a given dataset - """ - - def __init__(self, shape, sel=None): - self._shape = shape - self._rank = len(self._shape) - self._stop = False - - if self._rank < 1: - raise ValueError("IndexIterator can not be used on arrays of zero rank") - - if sel is None: - # select over entire dataset - slices = [] - for dim in range(self._rank): - slices.append(slice(0, self._shape[dim])) - self._sel = tuple(slices) - else: - if isinstance(sel, slice): - self._sel = (sel,) - else: - self._sel = sel - if len(self._sel) != self._rank: - raise ValueError("Invalid selection - selection region must have same rank as shape") - self._index = [] - for dim in range(self._rank): - s = self._sel[dim] - if s.start < 0 or s.stop > self._shape[dim] or s.stop <= s.start: - raise ValueError( - "Invalid selection - selection region must be within dataset space" - ) - self._index.append(s.start) - - def __iter__(self): - return self - - def __next__(self): - if self._stop: - raise StopIteration() - # bump up the last index and carry forward if we run outside the selection - dim = self._rank - 1 - ret_index = self._index.copy() - while True: - s = self._sel[dim] - if s.step: - step = s.step - else: - step = 1 - self._index[dim] += step - - if self._index[dim] < s.stop: - # we still have room to extend along this dimensions - break - - # reset to the start and continue iterating with higher dimension - self._index[dim] = s.start - dim -= 1 - if dim < 0: - # ran past last index, stop iteration on next run - self._stop = True - - return tuple(ret_index) - - -def ndarray_compare(arr1, arr2): - # compare two numpy arrays. - # return true if the same (exclusive of null vs. empty array) - # false otherwise - # TBD: this is slow for multi-megabyte vlen arrays, needs to be optimized - if not isinstance(arr1, np.ndarray) and not isinstance(arr2, np.ndarray): - if not isinstance(arr1, np.void) and not isinstance(arr2, np.void): - return arr1 == arr2 - if isinstance(arr1, np.void) and not isinstance(arr2, np.void): - if arr1.size == 0 and not arr2: - return True - else: - return False - if not isinstance(arr1, np.void) and isinstance(arr2, np.void): - if not arr1 and arr2.size == 0: - return True - else: - return False - # both np.voids - if arr1.size != arr2.size: - return False - - if len(arr1) != len(arr2): - return False - - for i in range(len(arr1)): - if not ndarray_compare(arr1[i], arr2[i]): - return False - return True - - if isinstance(arr1, np.ndarray) and not isinstance(arr2, np.ndarray): - # same only if arr1 is empty and arr2 is 0 - if arr1.size == 0 and not arr2: - return True - else: - return False - if not isinstance(arr1, np.ndarray) and isinstance(arr2, np.ndarray): - # same only if arr1 is empty and arr2 size is 0 - if not arr1 and arr2.size == 0: - return True - else: - return False - - # two ndarrays... - if arr1.shape != arr2.shape: - return False - if arr2.dtype != arr2.dtype: - return False - - if isVlen(arr1.dtype): - # need to compare element by element - - nElements = np.prod(arr1.shape) - arr1 = arr1.reshape((nElements,)) - arr2 = arr2.reshape((nElements,)) - for i in range(nElements): - if not ndarray_compare(arr1[i], arr2[i]): - return False - return True - else: - # can just us np array_compare - return np.array_equal(arr1, arr2) - - -def getBroadcastShape(mshape, element_count): - # if element_count is less than the number of elements - # defined by mshape, return a numpy compatible broadcast - # shape that contains element_count elements. - # If non exists return None - - if np.prod(mshape) == element_count: - return None - - if element_count == 1: - # this always works - return [1,] - - bcshape = [] - rank = len(mshape) - for n in range(rank - 1): - bcshape.insert(0, mshape[rank - n - 1]) - if element_count == np.prod(bcshape): - return bcshape # have a match - - return None # no broadcast found diff --git a/hsds/util/authUtil.py b/hsds/util/authUtil.py index eca78a02..24d80e56 100755 --- a/hsds/util/authUtil.py +++ b/hsds/util/authUtil.py @@ -71,7 +71,7 @@ def getDynamoDBClient(app): if aws_iam_role and not aws_secret_access_key: # TODO - refactor with similar code in s3Util - log.info("getted EC2 IAM role credentials") + log.info("got EC2 IAM role credentials") # Use EC2 IAM role to get credentials # See: https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ # iam-roles-for-amazon-ec2.html?icmpid=docs_ec2_console @@ -190,7 +190,7 @@ def initUserDB(app): """ log.info("initUserDB") if "user_db" in app: - msg = "user_db already initilized" + msg = "user_db already initialized" log.warn(msg) return @@ -461,7 +461,7 @@ def initGroupDB(app): """ log.info("initgroupDB") if "group_db" in app: - msg = "group_db already initilized" + msg = "group_db already initialized" log.warn(msg) return diff --git a/hsds/util/awsLambdaClient.py b/hsds/util/awsLambdaClient.py deleted file mode 100644 index fe801380..00000000 --- a/hsds/util/awsLambdaClient.py +++ /dev/null @@ -1,195 +0,0 @@ -from aiobotocore.session import get_session -from asyncio import CancelledError - -import datetime -import subprocess -import json -import time -from aiobotocore.config import AioConfig -from aiohttp.web_exceptions import HTTPInternalServerError -from aiohttp.client_exceptions import ClientError - -from .. import config -from .. import hsds_logger as log - -""" -get aiobotocore lambda client -""" - - -def getLambdaClient(app, session): - # first time setup of s3 client or limited time token has expired - - aws_region = None - aws_secret_access_key = None - aws_access_key_id = None - aws_iam_role = None - max_pool_connections = 64 - aws_session_token = None - try: - aws_iam_role = config.get("aws_iam_role") - except KeyError: - pass - try: - aws_secret_access_key = config.get("aws_secret_access_key") - except KeyError: - pass - try: - aws_access_key_id = config.get("aws_access_key_id") - except KeyError: - pass - try: - aws_region = config.get("aws_region") - except KeyError: - pass - try: - max_pool_connections = config.get("aio_max_pool_connections") - except KeyError: - pass - log.info(f"Lambda client init - aws_region {aws_region}") - - lambda_gateway = config.get("aws_lambda_gateway") - if not lambda_gateway: - msg = "Invalid aws lambda gateway" - log.error(msg) - raise ValueError(msg) - log.info(f"Using AWS Lambda Gateway: {lambda_gateway}") - - use_ssl = False - if lambda_gateway.startswith("https"): - use_ssl = True - - if not aws_secret_access_key or aws_secret_access_key == "xxx": - log.info("aws secret access key not set") - aws_secret_access_key = None - if not aws_access_key_id or aws_access_key_id == "xxx": - log.info("aws access key id not set") - aws_access_key_id = None - - if aws_iam_role and not aws_secret_access_key: - log.info(f"using iam role: {aws_iam_role}") - log.info("getting EC2 IAM role credentials") - # Use EC2 IAM role to get credentials - # See: "https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ \ - # iam-roles-for-amazon-ec2.html?icmpid=docs_ec2_console - req = "http://169.254.169.254/" - req += f"latest/meta-data/iam/security-credentials/{aws_iam_role}" - curl_cmd = ["curl", req] - p = subprocess.run(curl_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - if p.returncode != 0: - msg = f"Error getting IAM role credentials: {p.stderr}" - log.error(msg) - else: - stdout = p.stdout.decode("utf-8") - try: - cred = json.loads(stdout) - aws_secret_access_key = cred["SecretAccessKey"] - aws_access_key_id = cred["AccessKeyId"] - aws_cred_expiration = cred["Expiration"] - aws_session_token = cred["Token"] - log.info(f"Got Expiration of: {aws_cred_expiration}") - # trim off 'Z' and add 'UTC' - expiration_str = aws_cred_expiration[:-1] + "UTC" - # save the expiration - app["lambda_token_expiration"] = datetime.datetime.strptime( - expiration_str, "%Y-%m-%dT%H:%M:%S%Z" - ) - except json.JSONDecodeError: - msg = "Unexpected error decoding EC2 meta-data response" - log.error(msg) - except KeyError: - msg = "Missing expected key from EC2 meta-data response" - log.error(msg) - aws_region = config.get("aws_region") - if not aws_region: - aws_region = "us-east-1" - - max_pool_connections = config.get("aio_max_pool_connections") - aio_config = AioConfig(max_pool_connections=max_pool_connections) - kwargs = { - "region_name": aws_region, - "aws_secret_access_key": aws_secret_access_key, - "aws_access_key_id": aws_access_key_id, - "aws_session_token": aws_session_token, - "use_ssl": use_ssl, - "config": aio_config, - } - lambda_client = session.create_client("lambda", **kwargs) - - # TBD - we are getting errors if we try to reuse lambda client - # app["lambda"] = lambda_client - return lambda_client - - -""" -Async invoke for lambda function -""" - - -class lambdaInvoke: - def __init__(self, app, params, timeout=10): - self.app = app - self.params = params - self.timeout = timeout - self.lambdaFunction = config.get("aws_lambda_chunkread_function") - self.client = None - if "session" not in app: - app["session"] = get_session() - - self.session = app["session"] - - if "lambda_stats" not in app: - app["lambda_stats"] = {} - lambda_stats = app["lambda_stats"] - if self.lambdaFunction not in lambda_stats: - lambda_stats[self.lambdaFunction] = {"cnt": 0, "inflight": 0, "failed": 0} - self.funcStats = lambda_stats[self.lambdaFunction] - - async def __aenter__(self): - start_time = time.time() - payload = json.dumps(self.params) - msg = f"invoking lambda function {self.lambdaFunction} " - msg += "with payload: {self.params} start: {start_time}" - log.info(msg) - log.debug(f"Lambda function count: {self.funcStats['cnt']}") - self.funcStats["cnt"] += 1 - self.funcStats["inflight"] += 1 - - self.client = getLambdaClient(self.app, self.session) - - try: - kwargs = {"FunctionName": self.lambdaFunction, "Payload": payload} - lambda_rsp = await self.client.invoke(**kwargs) - finish_time = time.time() - msg = f"lambda.invoke({self.lambdaFunction} " - msg += f"start={start_time:.4f} " - msg += f"finish={finish_time:.4f} " - msg += f"elapsed={finish_time - start_time:.4f}" - log.info(msg) - self.funcStats["inflight"] -= 1 - msg = f"lambda.invoke - {self.funcStats['inflight']} " - msg += "inflight requests" - log.info(msg) - return lambda_rsp - except ClientError as ce: - log.error(f"Error for lambda invoke: {ce} ") - self.funcStats["inflight"] -= 1 - self.funcStats["failed"] += 1 - raise HTTPInternalServerError() - except CancelledError as cle: - log.warn(f"CancelledError for lambda invoke: {cle}") - self.funcStats["inflight"] -= 1 - self.funcStats["failed"] += 1 - raise HTTPInternalServerError() - except Exception as e: - msg = f"Unexpected exception for lamdea invoke: {e}, " - msg += "type: {type(e)}" - log.error(msg) - self.funcStats["inflight"] -= 1 - self.funcStats["failed"] += 1 - raise HTTPInternalServerError() - - async def __aexit__(self, exc_type, exc, tb): - log.debug("lambdaInvoke - aexit") - if self.client: - await self.client.close() diff --git a/hsds/util/boolparser.py b/hsds/util/boolparser.py deleted file mode 100644 index 2645fc96..00000000 --- a/hsds/util/boolparser.py +++ /dev/null @@ -1,293 +0,0 @@ -""" -Grammer: -Expression --> AndTerm { OR AndTerm}+ -AndTerm --> Condition { AND Condition}+ -Condition --> Terminal (>,<,>=,<=,==) Terminal | (Expression) -Terminal --> Number or String or Variable -Usage: - from boolparser import * - p = BooleanParser('') - p.evaluate(variable_dict) # variable_dict is a dictionary providing values - # for variables that appear in -""" - - -class TokenType: - NUM, STR, BYTE, VAR, GT, GTE, LT, LTE, EQ, NEQ, LP, RP, AND, OR = range(14) - - -class TreeNode: - tokenType = None - value = None - left = None - right = None - - def __init__(self, tokenType): - self.tokenType = tokenType - - -class Tokenizer: - expression = None - tokens = None - tokenTypes = None - i = 0 - - def __init__(self, exp): - self.expression = exp - - def next(self): - self.i += 1 - return self.tokens[self.i - 1] - - def peek(self): - return self.tokens[self.i] - - def hasNext(self): - return self.i < len(self.tokens) - - def nextTokenType(self): - return self.tokenTypes[self.i] - - def nextTokenTypeIsOperator(self): - t = self.tokenTypes[self.i] - return t in ( - TokenType.GT, - TokenType.GTE, - TokenType.LT, - TokenType.LTE, - TokenType.EQ, - TokenType.NEQ, - ) - - def tokenize(self): - import re - - reg = re.compile(r"(\bAND\b|\bOR\b|!=|==|<=|>=|<|>|\(|\)|\&|\|)") - SINGLE_QUOTE = "'" - DOUBLE_QUOTE = '"' - self.tokens = reg.split(self.expression) - self.tokens = [t.strip() for t in self.tokens if t.strip() != ""] - - self.tokenTypes = [] - for i in range(len(self.tokens)): - t = self.tokens[i] - if t in ("AND", "&"): - self.tokenTypes.append(TokenType.AND) - elif t in ("OR", "|"): - self.tokenTypes.append(TokenType.OR) - elif t == "(": - self.tokenTypes.append(TokenType.LP) - elif t == ")": - self.tokenTypes.append(TokenType.RP) - elif t == "<": - self.tokenTypes.append(TokenType.LT) - elif t == "<=": - self.tokenTypes.append(TokenType.LTE) - elif t == ">": - self.tokenTypes.append(TokenType.GT) - elif t == ">=": - self.tokenTypes.append(TokenType.GTE) - elif t == "==": - self.tokenTypes.append(TokenType.EQ) - elif t == "!=": - self.tokenTypes.append(TokenType.NEQ) - else: - # number of string or variable - if t[0] == SINGLE_QUOTE and t[-1] == SINGLE_QUOTE: - self.tokenTypes.append(TokenType.STR) - self.tokens[i] = t[1:-1] # strip quotes - elif t[0] == DOUBLE_QUOTE and t[-1] == DOUBLE_QUOTE: - self.tokenTypes.append(TokenType.STR) - self.tokens[i] = t[1:-1] # strip quotes - elif len(t) > 3 and all( - (t[0] == "b", t[1] == SINGLE_QUOTE, t[-1] == SINGLE_QUOTE) - ): - # binary string - self.tokenTypes.append(TokenType.BYTE) - self.tokens[i] = t[2:-1] # strip quotes and 'b' - else: - try: - float(t) - self.tokenTypes.append(TokenType.NUM) - except Exception: - self.tokenTypes.append(TokenType.VAR) - - def getVariables(self): - s = set() - for i in range(len(self.tokens)): - if self.tokenTypes[i] == TokenType.VAR: - s.add(self.tokens[i]) - variables = list(s) - variables.sort() - return variables - - -class BooleanParser: - tokenizer = None - root = None - - def __init__(self, exp): - # tbd - BooleanParser doesn't know about where clauses. - # For now just strip it off the query. - if exp.startswith("where"): - self.tokenizer = None - else: - n = exp.find("where") - if n > 0: - exp = exp[:n] - - self.tokenizer = Tokenizer(exp) - self.tokenizer.tokenize() - self.parse() - - def parse(self): - self.root = self.parseExpression() - - def getVariables(self): - if self.tokenizer is None: - return [] - - return self.tokenizer.getVariables() - - def parseExpression(self): - andTerm1 = self.parseAndTerm() - while ( - self.tokenizer.hasNext() and self.tokenizer.nextTokenType() == TokenType.OR - ): - self.tokenizer.next() - andTermX = self.parseAndTerm() - andTerm = TreeNode(TokenType.OR) - andTerm.left = andTerm1 - andTerm.right = andTermX - andTerm1 = andTerm - return andTerm1 - - def parseAndTerm(self): - condition1 = self.parseCondition() - while ( - self.tokenizer.hasNext() and self.tokenizer.nextTokenType() == TokenType.AND - ): - self.tokenizer.next() - conditionX = self.parseCondition() - condition = TreeNode(TokenType.AND) - condition.left = condition1 - condition.right = conditionX - condition1 = condition - return condition1 - - def parseCondition(self): - if self.tokenizer.hasNext() and self.tokenizer.nextTokenType() == TokenType.LP: - self.tokenizer.next() - expression = self.parseExpression() - next_token = None - if self.tokenizer.hasNext(): - next_token = self.tokenizer.nextTokenType() - if next_token == TokenType.RP: - self.tokenizer.next() - return expression - else: - msg = "Closing ) expected, but got " + self.tokenizer.next() - raise Exception(msg) - - terminal1 = self.parseTerminal() - if self.tokenizer.hasNext() and self.tokenizer.nextTokenTypeIsOperator(): - condition = TreeNode(self.tokenizer.nextTokenType()) - self.tokenizer.next() - terminal2 = self.parseTerminal() - condition.left = terminal1 - condition.right = terminal2 - return condition - else: - msg = "Operator expected, but got " + self.tokenizer.next() - raise Exception(msg) - - def parseTerminal(self): - if self.tokenizer.hasNext(): - tokenType = self.tokenizer.nextTokenType() - if tokenType == TokenType.NUM: - n = TreeNode(tokenType) - n.value = float(self.tokenizer.next()) - return n - elif tokenType in (TokenType.STR, TokenType.BYTE, TokenType.VAR): - n = TreeNode(tokenType) - n.value = self.tokenizer.next() - return n - else: - msg = "NUM, STR, or VAR expected, but got " - msg += self.tokenizer.next() - raise Exception(msg) - else: - msg = "NUM, STR, or VAR expected, but got " + self.tokenizer.next() - raise Exception(msg) - - def evaluate(self, variable_dict): - return self.evaluateRecursive(self.root, variable_dict) - - def evaluateRecursive(self, treeNode, variable_dict): - if treeNode.tokenType in (TokenType.NUM, TokenType.STR, TokenType.BYTE): - return treeNode.value - - if treeNode.tokenType == TokenType.VAR: - return variable_dict.get(treeNode.value) - - left = self.evaluateRecursive(treeNode.left, variable_dict) - right = self.evaluateRecursive(treeNode.right, variable_dict) - - if treeNode.tokenType == TokenType.GT: - return left > right - elif treeNode.tokenType == TokenType.GTE: - return left >= right - elif treeNode.tokenType == TokenType.LT: - return left < right - elif treeNode.tokenType == TokenType.LTE: - return left <= right - elif treeNode.tokenType == TokenType.EQ: - return left == right - elif treeNode.tokenType == TokenType.NEQ: - return left != right - elif treeNode.tokenType == TokenType.AND: - return left and right - elif treeNode.tokenType == TokenType.OR: - return left or right - else: - raise Exception("Unexpected type " + str(treeNode.tokenType)) - - def getEvalRecursive(self, treeNode): - if treeNode.tokenType == TokenType.NUM: - return treeNode.value - if treeNode.tokenType == TokenType.STR: - # add quotes for evaluation - return f"'{treeNode.value}'" - if treeNode.tokenType == TokenType.BYTE: - return f"b'{treeNode.value}'" - if treeNode.tokenType == TokenType.VAR: - return treeNode.value - - left = self.getEvalRecursive(treeNode.left) - if isinstance(left, str) and " " in left: - left = f"({left})" - right = self.getEvalRecursive(treeNode.right) - if isinstance(right, str) and " " in right: - right = f"({right})" - - if treeNode.tokenType == TokenType.GT: - return f"{left} > {right}" - elif treeNode.tokenType == TokenType.GTE: - return f"{left} >= {right}" - elif treeNode.tokenType == TokenType.LT: - return f"{left} < {right}" - elif treeNode.tokenType == TokenType.LTE: - return f"{left} <= {right}" - elif treeNode.tokenType == TokenType.EQ: - return f"{left} == {right}" - elif treeNode.tokenType == TokenType.NEQ: - return f"{left} != {right}" - elif treeNode.tokenType == TokenType.AND: - return f"{left} & {right}" - elif treeNode.tokenType == TokenType.OR: - return f"{left} | {right}" - else: - raise Exception("Unexpected type " + str(treeNode.tokenType)) - - def getEvalStr(self): - return self.getEvalRecursive(self.root) diff --git a/hsds/util/chunkUtil.py b/hsds/util/chunkUtil.py index dc03cc89..bf0aa51d 100644 --- a/hsds/util/chunkUtil.py +++ b/hsds/util/chunkUtil.py @@ -1,6 +1,10 @@ import numpy as np + +from h5json.array_util import ndarray_compare +from h5json.dset_util import getDatasetLayout +from h5json import selections + from .. import hsds_logger as log -from .arrayUtil import ndarray_compare CHUNK_BASE = 16 * 1024 # Multiplier by which chunks are adjusted CHUNK_MIN = 512 * 1024 # Soft lower limit (512k) @@ -9,230 +13,6 @@ PRIMES = [29, 31, 37, 41, 43, 47, 53, 59, 61, 67] # for chunk partitioning -def getChunkSize(layout, type_size): - """Return chunk size given layout. - i.e. just the product of the values in the list. - """ - if type_size == "H5T_VARIABLE": - type_size = DEFAULT_TYPE_SIZE - - chunk_size = type_size - for n in layout: - if n <= 0: - raise ValueError("Invalid chunk layout") - chunk_size *= n - return chunk_size - - -def get_dset_size(shape_json, typesize): - """Return the size of the dataspace. For - any unlimited dimensions, assume a value of 1. - (so the return size will be the absolute minimum) - """ - if shape_json is None or shape_json["class"] == "H5S_NULL": - return None - if shape_json["class"] == "H5S_SCALAR": - return typesize # just return size for one item - if typesize == "H5T_VARIABLE": - typesize = DEFAULT_TYPE_SIZE # just take a guess at the item size - dset_size = typesize - shape = shape_json["dims"] - rank = len(shape) - - for n in range(rank): - if shape[n] == 0: - # extendable extent with value of 0 - continue # assume this is one - dset_size *= shape[n] - return dset_size - - -def expandChunk( - layout, typesize, shape_json, chunk_min=CHUNK_MIN, layout_class="H5D_CHUNKED" -): - """Compute an increased chunk shape with a size in bytes greater than chunk_min.""" - if shape_json is None or shape_json["class"] == "H5S_NULL": - return None - if shape_json["class"] == "H5S_SCALAR": - return (1,) # just enough to store one item - - layout = list(layout) - log.debug(f"expandChunk layout: {layout} typesize: {typesize}") - dims = shape_json["dims"] - rank = len(dims) - extendable_dims = 0 # number of dimensions that are extenable - maxdims = None - if "maxdims" in shape_json: - maxdims = shape_json["maxdims"] - for n in range(rank): - if maxdims[n] == 0 or maxdims[n] > dims[n]: - extendable_dims += 1 - - dset_size = get_dset_size(shape_json, typesize) - if dset_size <= chunk_min and extendable_dims == 0: - # just use the entire dataspace shape as one big chunk - return tuple(dims) - - chunk_size = getChunkSize(layout, typesize) - if chunk_size >= chunk_min: - return tuple(layout) # good already - while chunk_size < chunk_min: - # just adjust along extendable dimensions first - old_chunk_size = chunk_size - for n in range(rank): - dim = rank - n - 1 # start from last dim - - if extendable_dims > 0: - if maxdims[dim] == 0: - # infinitely extendable dimensions - layout[dim] *= 2 - chunk_size = getChunkSize(layout, typesize) - if chunk_size > chunk_min: - break - elif maxdims[dim] > layout[dim]: - # can only be extended so much - layout[dim] *= 2 - if layout[dim] >= dims[dim]: - layout[dim] = maxdims[dim] # trim back - extendable_dims -= 1 # one less extenable dimension - - chunk_size = getChunkSize(layout, typesize) - if chunk_size > chunk_min: - break - else: - pass # ignore non-extensible for now - else: - # no extendable dimensions - if dims[dim] > layout[dim]: - # can expand chunk along this dimension - layout[dim] *= 2 - if layout[dim] > dims[dim]: - layout[dim] = dims[dim] # trim back - chunk_size = getChunkSize(layout, typesize) - if chunk_size > chunk_min: - break - else: - pass # can't extend chunk along this dimension - if chunk_size <= old_chunk_size: - # stop iteration if we haven't increased the chunk size - log.debug("stopping expandChunk iteration") - break - elif chunk_size > chunk_min: - break # we're good - else: - pass # do another round - return tuple(layout) - - -def shrinkChunk(layout, typesize, chunk_max=CHUNK_MAX, layout_class="H5D_CHUNKED"): - """Compute a reduced chunk shape with a size in bytes less than chunk_max.""" - layout = list(layout) - chunk_size = getChunkSize(layout, typesize) - if chunk_size <= chunk_max: - return tuple(layout) # good already - log.debug(f"shrinkChunk layout: {layout} typesize: {typesize}") - rank = len(layout) - - while chunk_size > chunk_max: - # just adjust along extendable dimensions first - old_chunk_size = chunk_size - for dim in range(rank): - if layout[dim] > 1: - # tricky way to do x // 2 with ceil - layout[dim] = -(-layout[dim] // 2) - chunk_size = getChunkSize(layout, typesize) - if chunk_size <= chunk_max: - break - else: - pass # can't shrink chunk along this dimension - if chunk_size >= old_chunk_size: - # reality check to see if we'll ever break out of the while loop - log.warning("Unexpected error in shrink_chunk") - break - elif chunk_size <= chunk_max: - break # we're good - else: - pass # do another round - return tuple(layout) - - -def guessChunk(shape_json, typesize): - """Guess an appropriate chunk layout for a dataset, given its shape and - the size of each element in bytes. Will allocate chunks only as large - as MAX_SIZE. Chunks are generally close to some power-of-2 fraction of - each axis, slightly favoring bigger values for the last index. - - Undocumented and subject to change without warning. - """ - if shape_json is None or shape_json["class"] == "H5S_NULL": - return None - if shape_json["class"] == "H5S_SCALAR": - return (1,) # just enough to store one item - - if "maxdims" in shape_json: - shape = shape_json["maxdims"] - else: - shape = shape_json["dims"] - - if typesize == "H5T_VARIABLE": - typesize = 128 # just take a guess at the item size - - # For unlimited dimensions we have to guess. use 1024 - shape = tuple((x if x != 0 else 1024) for i, x in enumerate(shape)) - - return shape - - -def getContiguousLayout( - shape_json, item_size, chunk_min=1000 * 1000, chunk_max=4 * 1000 * 1000 -): - """ - create a chunklayout for datasets use continguous storage. - """ - if not isinstance(item_size, int): - msg = "ContiguousLayout can only be used with fixed-length types" - raise ValueError(msg) - if chunk_max < chunk_min: - raise ValueError("chunk_max cannot be less than chunk_min") - if shape_json is None or shape_json["class"] == "H5S_NULL": - return None - if shape_json["class"] == "H5S_SCALAR": - return (1,) # just enough to store one item - dims = shape_json["dims"] - rank = len(dims) - if rank == 0: - raise ValueError("rank must be positive for Contiguous Layout") - for dim in dims: - if dim < 0: - raise ValueError("extents must be positive for Contiguous Layout") - if dim == 0: - # datashape with no elements, just return dims as layout - return dims - - nsize = item_size - layout = [ - 1, - ] * rank - - for i in range(rank): - dim = rank - i - 1 - extent = dims[dim] - if extent * nsize < chunk_max: - # just use the full extent as layout - layout[dim] = extent - nsize *= extent - else: - n = extent - while n > 1: - n = -(-n // 2) # use negatives so we round up on odds - if n * nsize < chunk_max: - break - layout[dim] = n - break # just use 1's for the rest of the layout - - return layout - - def frac(x, d): """ Utility func -- Works like fractional div, but returns ceiling @@ -253,31 +33,54 @@ def slice_stop(s): return s.start + w +def _toArraySlice(s): + """h5json's Selection.slices already returns slices with a true + coordinate stop (start + count * step), suitable for numpy indexing + or coordinate-range arithmetic. Just ensure a default step of 1. + """ + step = s.step if s.step is not None else 1 + return slice(s.start, s.stop, step) + + +def toNumpyIndex(selection): + """Convert a selections.Selection into a tuple of slices/coordinate-lists + usable for direct numpy array indexing (arr[toNumpyIndex(selection)]). + """ + return tuple(_toArraySlice(s) if isinstance(s, slice) else s for s in selection.slices) + + def getNumChunks(selection, layout): """ Get the number of chunks potentially required. If selection is provided (a list of slices), return the number of chunks that intersect with the selection. """ + + if not isinstance(selection, selections.Selection): + msg = "Expected selection.Selection type" + log.warning(msg) + raise ValueError(msg) + + if selection.nselect == 0: + # zero length selection + return 0 + + if selection.nselect == 1: + # single point selection + return 1 + rank = len(layout) - if len(selection) != rank: - msg = f"selection list has {len(selection)} items, but rank is {rank}" + if rank == 1 and layout[0] == 1: + # scalar dataset + return 1 + if len(selection.shape) != rank: + msg = f"selection list has {len(selection.shape)} items, but rank is {rank}" raise ValueError(msg) - # do a quick check that we don't have a null selection space' - # TBD: this needs to be revise to do the right think with stride > 1 - for s in selection: - if isinstance(s, slice): - if s.stop <= s.start: - log.debug("null selection") - return 0 - else: - # coordinate list - if len(s) == 0: - return 0 + # first, get the number of chunks needed for any coordinate selection chunk_indices = [] - for i in range(len(selection)): - s = selection[i] + for i in range(rank): + s = selection.slices[i] c = layout[i] if isinstance(s, slice): continue @@ -303,20 +106,21 @@ def getNumChunks(selection, layout): num_chunks = 1 # now deal with any slices in the selection - for i in range(len(selection)): - s = selection[i] + for i in range(rank): + s = selection.slices[i] c = layout[i] # chunk size if not isinstance(s, slice): # ignore coordinate lists since we dealt with them above continue - if s.step is None: - s = slice(s.start, s.stop, 1) - if s.step > 1: - num_points = frac((s.stop - s.start), s.step) - w = num_points * s.step - (s.step - 1) + step = s.step if s.step is not None else 1 + # s.stop is a true coordinate stop (start + num_points * step), + # so divide out the step to recover the number of selected points + num_points = (s.stop - s.start) // step + if step > 1: + w = num_points * step - (step - 1) else: - w = s.stop - s.start # selection width (>0) + w = num_points # selection width (>0) lc = frac(s.start, c) * c @@ -326,10 +130,10 @@ def getNumChunks(selection, layout): rc = ((s.start + w) // c) * c m = rc - lc - if c > s.step: + if c > step: count = m // c else: - count = m // s.step + count = m // step if s.start < lc: count += 1 # hit one chunk on the left if s.start + w > rc: @@ -343,10 +147,11 @@ def getChunkId(dset_id, point, layout): chunk_id = "c-" + dset_id[2:] + "_" rank = len(layout) + log.debug(f"getChunkId - dset_id: {dset_id}, point: {point}, layout: {layout}") for dim in range(rank): coord = None - if rank == 1: + if rank == 1 and not isinstance(point, (list, tuple, np.ndarray)): coord = point # integer for 1d dataset else: coord = point[dim] @@ -354,7 +159,7 @@ def getChunkId(dset_id, point, layout): chunk_index = int(coord) // c chunk_id += str(chunk_index) if dim + 1 < rank: - chunk_id += "_" # seperate dimensions with underscores + chunk_id += "_" # separate dimensions with underscores return chunk_id @@ -429,11 +234,8 @@ def getPartitionKey(chunk_id, partition_count): def getChunkIdForPartition(chunk_id, dset_json): """Return the partition specific chunk id for given chunk""" - if "layout" not in dset_json: - msg = "No layout found in dset_json" - log.error(msg) - raise KeyError(msg) - layout_json = dset_json["layout"] + + layout_json = getDatasetLayout(dset_json) if "partition_count" in layout_json: partition_count = layout_json["partition_count"] partition = getChunkPartition(chunk_id) @@ -472,7 +274,16 @@ def chunk_id_to_index(chunk_id): indices.append(x) return indices + log.debug(f"getChunkIds - dset_id: {dset_id}, selection: {selection}, layout: {layout}") + if prefix: + log.debug(f"prefix: {prefix}") + if not isinstance(selection, selections.Selection): + msg = "Expected selection.Selection type" + log.warning(msg) + raise ValueError(msg) + num_chunks = getNumChunks(selection, layout) + log.debug(f"getChunkIds - num_chunks: {num_chunks}") if num_chunks == 0: return [] # empty list if prefix is None: @@ -482,12 +293,16 @@ def chunk_id_to_index(chunk_id): log.warning(msg) raise ValueError(msg) prefix = "c-" + dset_id[2:] + "_" - rank = len(selection) + + if selection.shape == (): + # scalar dataset - single chunk, index 0 for each layout dimension + return [prefix + chunk_index_to_id([0] * len(layout))] + rank = len(selection.shape) # initialize chunk_ids based on coordinate index, if any num_coordinates = None chunk_items = set() - for s in selection: + for s in selection.slices: if isinstance(s, slice): continue elif num_coordinates is None: @@ -503,7 +318,7 @@ def chunk_id_to_index(chunk_id): for i in range(num_coordinates): chunk_idx = [] for dim in range(rank): - s = selection[dim] + s = selection.slices[dim] c = layout[dim] if isinstance(s, slice): chunk_index = None @@ -521,7 +336,7 @@ def chunk_id_to_index(chunk_id): # log.debug(f"getChunkIds - selection: {selection}") for dim in range(rank): - s = selection[dim] + s = selection.slices[dim] c = layout[dim] if not isinstance(s, slice): @@ -531,20 +346,23 @@ def chunk_id_to_index(chunk_id): if s.step is None: s = slice(s.start, s.stop, 1) + # s.stop is a true coordinate stop (start + num_points * step), + # so divide out the step to recover the number of selected points + num_points = (s.stop - s.start) // s.step chunk_indices = [] if s.step > c: # chunks may not be contiguous, skip along the selection and add # whatever chunks we land in - for i in range(s.start, s.stop, s.step): + for k in range(num_points): + i = s.start + k * s.step chunk_index = i // c chunk_indices.append(chunk_index) else: # get a contiguous set of chunks along the selection if s.step > 1: - num_points = frac((s.stop - s.start), s.step) w = num_points * s.step - (s.step - 1) else: - w = s.stop - s.start # selection width (>0) + w = num_points # selection width (>0) chunk_index_start = s.start // c chunk_index_end = frac((s.start + w), c) @@ -592,7 +410,7 @@ def getChunkCoordinate(chunk_id, layout): return coord -def getChunkSelection(chunk_id, slices, layout): +def getChunkSelection(chunk_id, selection, layout): """ Return the intersection of the chunk with the given slices selection of the array. @@ -603,6 +421,7 @@ def getChunkSelection(chunk_id, slices, layout): coord_mask = None # compute a boolean mask for the coordinates that apply to the given chunk_id + slices = selection.slices for dim in range(rank): s = slices[dim] c = layout[dim] @@ -625,8 +444,7 @@ def getChunkSelection(chunk_id, slices, layout): c = layout[dim] n = chunk_index[dim] * c if isinstance(s, slice): - if s.step is None: - s = slice(s.start, s.stop, 1) + s = _toArraySlice(s) if s.start >= n + c: return None # null intersection if s.stop < n: @@ -652,35 +470,36 @@ def getChunkSelection(chunk_id, slices, layout): coords.append(s[i]) sel.append(coords) - return sel + return selections.select(selection.shape, tuple(sel)) -def getChunkCoverage(chunk_id, slices, layout): +def getChunkCoverage(chunk_id, selection, layout): """ Get chunk-relative selection of the given chunk and selection. """ chunk_index = getChunkIndex(chunk_id) - chunk_sel = getChunkSelection(chunk_id, slices, layout) + chunk_sel = getChunkSelection(chunk_id, selection, layout) if not chunk_sel: - log.warn(f"slices: {slices} does intersect chunk: {chunk_id}") + log.warn(f"selection: {selection} does intersect chunk: {chunk_id}") return None rank = len(layout) - if len(slices) != rank: + if len(selection.shape) != rank: raise ValueError(f"invalid slices value for dataset of rank: {rank}") sel = [] for dim in range(rank): - s = chunk_sel[dim] + s = chunk_sel.slices[dim] w = layout[dim] offset = chunk_index[dim] * w if isinstance(s, slice): + s = _toArraySlice(s) start = s.start - offset if start < 0: msg = "Unexpected chunk selection" log.error(msg) raise ValueError(msg) - stop = s.stop - offset + stop = slice_stop(s) - offset if stop > w: msg = "Unexpected chunk selection" log.error(msg) @@ -691,24 +510,26 @@ def getChunkCoverage(chunk_id, slices, layout): coord = [] for i in range(len(s)): coord.append(s[i] - offset) - sel.append(tuple(coord)) + sel.append(coord) - return sel + return selections.select(tuple(layout), tuple(sel)) -def getDataCoverage(chunk_id, slices, layout): +def getDataCoverage(chunk_id, selection, layout): """ Get data-relative selection of the given chunk and selection. """ - chunk_sel = getChunkSelection(chunk_id, slices, layout) + chunk_sel = getChunkSelection(chunk_id, selection, layout) + chunk_slices = chunk_sel.slices rank = len(layout) sel = [] points = None coordinate_extent = None + slices = selection.slices for dim in range(rank): - c = chunk_sel[dim] + c = chunk_slices[dim] s = slices[dim] if isinstance(s, slice): continue @@ -734,11 +555,12 @@ def getDataCoverage(chunk_id, slices, layout): data_pts = None for dim in range(rank): - c = chunk_sel[dim] + c = chunk_slices[dim] s = slices[dim] if isinstance(s, slice): - if s.step is None: - s = slice(s.start, s.stop, 1) + s = _toArraySlice(s) + c = _toArraySlice(c) + c = slice(c.start, slice_stop(c), c.step) if c.step != s.step: msg = "expecting step for chunk selection to be the same as data selection" raise ValueError(msg) @@ -774,7 +596,7 @@ def getDataCoverage(chunk_id, slices, layout): if include_pt: data_pts.append(i) - return tuple(sel) + return selections.select(selection.mshape, tuple(sel)) def getChunkRelativePoint(chunkCoord, point): @@ -823,13 +645,19 @@ class ChunkIterator: def __init__(self, dset_id, selection, layout): self._prefix = "c-" + dset_id[2:] self._layout = layout + if not isinstance(selection, selections.Selection): + msg = "Expected selection.Selection type" + log.warning(msg) + raise ValueError(msg) + if selection.select_type != selections.H5S_SEL_HYPERSLABS: + msg = "Expected hyperslab selection" + log.warning(msg) + raise ValueError(msg) self._selection = selection - self._rank = len(selection) - self._chunk_index = [ - 0, - ] * self._rank + self._rank = len(selection.shape) + self._chunk_index = [0,] * self._rank for i in range(self._rank): - s = selection[i] + s = selection.slices[i] c = layout[i] self._chunk_index[i] = s.start // c @@ -837,7 +665,8 @@ def __iter__(self): return self def next(self): - if self._chunk_index[0] * self._layout[0] >= self._selection[0].stop: + slices = self._selection.slices + if self._chunk_index[0] * self._layout[0] >= slices[0].stop: # ran past the last chunk, end iteration raise StopIteration() chunk_id = self._prefix @@ -850,7 +679,7 @@ def next(self): dim = self._rank - 1 while dim >= 0: c = self._layout[dim] - s = self._selection[dim] + s = slices[dim] self._chunk_index[dim] += 1 chunk_end = self._chunk_index[dim] * c @@ -866,12 +695,19 @@ def next(self): return chunk_id -def chunkReadSelection(chunk_arr, slices=None, select_dt=None): +def chunkReadSelection(chunk_arr, selection=None, select_dt=None): """ Return data from requested chunk and selection """ log.debug("chunkReadSelection") + if selection is None: + selection = selections.select(chunk_arr.shape, ...) + if not isinstance(selection, selections.Selection): + msg = "Expected selection.Selection type" + log.warning(msg) + raise ValueError(msg) + dims = chunk_arr.shape log.debug(f"got chunk dims: {dims}") rank = len(dims) @@ -879,14 +715,20 @@ def chunkReadSelection(chunk_arr, slices=None, select_dt=None): msg = "No dimension passed to chunkReadSelection" raise ValueError(msg) - log.debug(f"got selection: {slices}") - slices = tuple(slices) + log.debug(f"got selection: {selection}") + slices = toNumpyIndex(selection) if select_dt is None: # no field selection select_dt = chunk_arr.dtype - if len(slices) != rank: + # rank (dimension-count) check, not a shape-value check - see the + # matching comment in chunkWriteSelection() below. For a dataset whose + # own top-level type is an array/subarray dtype, chunk_arr's rank + # exceeds slices' rank by exactly the array type's own (absorbed) + # dims; chunk_arr[slices] below already handles that correctly via + # numpy's partial-indexing rules. + if len(slices) > rank: msg = "Selection rank does not match shape rank" raise ValueError(msg) @@ -904,24 +746,58 @@ def chunkReadSelection(chunk_arr, slices=None, select_dt=None): log.debug(f"select_dtype: {len(select_dt)} from {len(dt)} fields") # create an array with just the given fields arr = np.zeros(output_arr.shape, select_dt) - # slot in each of the given fields + # slot in each of the given fields - assign per-field (rather than + # arr[...] = output_arr[fields[0]] as a single-field shortcut) so + # this works uniformly whether the field is a scalar type or an + # array type (an array-typed field's own shape doesn't fit into + # arr's outer shape, which a single arr[...] assignment assumes) fields = select_dt.names - if len(fields) > 1: - for field in fields: - arr[field] = output_arr[field] - else: - arr[...] = output_arr[fields[0]] + for field in fields: + arr[field] = output_arr[field] output_arr = arr # return this return output_arr -def chunkWriteSelection(chunk_arr=None, slices=None, data=None): +def chunkWriteSelection(chunk_arr=None, selection=None, data=None): """ Write data for requested chunk and selection """ - log.debug(f"chunkWriteSelection for slices: {slices}") + if selection is None: + selection = selections.select(chunk_arr.shape, ...) + if not isinstance(selection, selections.Selection): + msg = "Expected selection.Selection type" + log.warning(msg) + raise ValueError(msg) + # These are rank (dimension-count) checks, not shape-value checks - + # selection.shape is the full underlying chunk/dataset shape + # regardless of what sub-range is actually selected (that's + # selection.mshape), so only rank is meaningful to compare here; the + # actual sub-range size is handled later via chunk_arr[slices]. + # + # For a dataset whose own top-level type is an array/subarray dtype + # (H5T_ARRAY, e.g. numpy's "3i1"), chunk_arr's (and data's) shape + # absorbs the array type's own trailing dims (chunk_arr.shape == + # dataset_shape + array_dims, e.g. (2, 3) for a 2-element dataset of + # 3-int8 arrays), while selection is built against the dataset's + # logical shape only (e.g. (2,)) - so chunk_arr's rank can legitimately + # exceed selection's rank by exactly that many trailing (absorbed) + # dims, and data's rank must match chunk_arr's for the same reason. + # The indexing below (chunk_arr[slices]) already handles those + # trailing dims correctly via numpy's own partial-indexing rules (a + # selection tuple shorter than the array's rank selects the full + # trailing dimensions), so no other change is needed. Covered in + # tests/unit/chunk_util_test.py's testChunkWriteSelectionArrayDtype + # and tests/integ/value_test.py's testCreateBareArrayDataset; see + # also h5pyd's TestSubarray.test_write_array/test_write_list. + sel_rank = len(selection.shape) + extra_rank = len(chunk_arr.shape) - sel_rank # absorbed array-dtype dims, if any + if extra_rank < 0: + msg = "Selection rank does not match dataset rank" + log.error(msg) + raise ValueError(msg) + log.debug(f"chunkWriteSelection for selection: {selection}") dims = chunk_arr.shape rank = len(dims) @@ -930,11 +806,8 @@ def chunkWriteSelection(chunk_arr=None, slices=None, data=None): msg = "No dimension passed to chunkWriteSelection" log.error(msg) raise ValueError(msg) - if len(slices) != rank: - msg = "Selection rank does not match dataset rank" - log.error(msg) - raise ValueError(msg) - if len(data.shape) != rank: + + if len(data.shape) != sel_rank + extra_rank: msg = "Input arr does not match dataset rank" log.error(msg) raise ValueError(msg) @@ -948,6 +821,7 @@ def chunkWriteSelection(chunk_arr=None, slices=None, data=None): log.debug("ChunkWriteSelection for all fields") updated = False + slices = toNumpyIndex(selection) try: if field_update: arr = chunk_arr[slices] @@ -1124,433 +998,12 @@ def chunkWritePoints(chunk_id=None, if len(select_dt) < len(dset_dtype): # get the element from the chunk chunk_val = list(chunk_arr[coord]) - # and just update the relevant fields - index = 0 - for (x, field) in zip(val, dset_dtype.names): - if field in select_dt.names: - chunk_val[index] = x - index += 1 + # and just update the relevant fields - zip against select_dt's + # own (narrowed) field names, not dset_dtype's full list, since + # val only has as many elements as select_dt has fields + for (x, field) in zip(val, select_dt.names): + index = dset_dtype.names.index(field) + chunk_val[index] = x val = tuple(chunk_val) # this will get written back chunk_arr[coord] = val # update the point - - -def _getWhereFieldName(query): - """ - Get the field name for a where clause. - Returns None if no where statement - """ - if query.startswith("where "): - i = len("where ") - else: - i = query.find(" where ") - if i > 0: - i += len(" where ") - if i < 0: - # no where statement - return None - - field_name = "" - end_quote_char = None - while i < len(query): - ch = query[i] - i += 1 - if end_quote_char and ch == end_quote_char: - # end of variable - end_quote_char = None - break - elif ch in ("'", '"'): - end_quote_char = ch - continue - if field_name and not ch.isalnum() and not ch == "_" and not end_quote_char: - # end of variable - break - if end_quote_char or ch.isalnum() or ch == "_": - field_name += ch - if not field_name: - # got a where keyword, but no field name - raise ValueError("query where with no fieldname") - if end_quote_char: - raise ValueError("unclosed quote") - - return field_name - - -def _getWhereElements(query): - """ - Get the values from a where clause - """ - - n = query.find(" in ") - if n < 0: - raise ValueError("where query with no 'in' keyword") - n += 4 # advance past " in " - elements = [] - i = query[n:].find("(") - if i < 0: - raise ValueError("where in query with no '(' character)") - i += n + 1 # advance past '(' - - end_quote_char = None - s = None - - while i < len(query): - ch = query[i] - i += 1 - if end_quote_char and ch == end_quote_char: - # end of variable - end_quote_char = None - if s is None: - s = "" - elements.append(s) - s = None - continue - if ch in ("'", '"'): - end_quote_char = ch - if s == "b": - # use bytes not str - s = b'' - else: - s = "" - continue - if ch == ",": - if s is not None: - elements.append(s) - s = None - continue - if ch == ")": - if end_quote_char: - raise ValueError("unclosed quote in 'where in' list") - if s is not None: - elements.append(s) - break - if ch.isspace(): - if end_quote_char: - if isinstance(s, bytes): - ch = ch.encode('utf8') - s += ch - continue - # anything else, just add to our variable - if isinstance(s, bytes): - ch = ch.encode('utf8') - if s is None: - s = ch - else: - s += ch - - if end_quote_char: - raise ValueError("unclosed quote") - - return elements - - -def _getEvalStr(query, arr_name, field_names): - """ - _getEvalStr: Get eval string for given query - Gets Eval string to use with numpy where method. - """ - i = 0 - eval_str = "" - var_name = None - end_quote_char = None - var_count = 0 - paren_count = 0 - black_list = ("import",) # field names that are not allowed - for item in black_list: - if item in field_names: - msg = "invalid field name" - log.warn(f"Bad query: {msg}") - raise ValueError(msg) - - if query.startswith("where "): - # no eval, return None - return None - # strip off any where clause after the query - n = query.find(" where ") - - where_field = None - if n > 0: - where_field = _getWhereFieldName(query) - log.debug(f"where field: [{where_field}]") - log.debug(f"query orig: {query}") - query = query[:n] - log.debug(f"query adjusted: {query}") - - while i < len(query): - ch = query[i] - if (i + 1) < len(query): - ch_next = query[i + 1] - else: - ch_next = None - if var_name and not ch.isalnum() and not ch == "_": - # end of variable - if var_name not in field_names: - # invalid - msg = f"query variable: {var_name}" - log.warn("Bad query: " + msg) - raise ValueError(msg) - eval_str += arr_name + "['" + var_name + "']" - var_name = None - var_count += 1 - - if end_quote_char: - if ch == end_quote_char: - # end of literal - end_quote_char = None - eval_str += ch - elif ch in ("'", '"'): - end_quote_char = ch - eval_str += ch - elif ch.isalnum() or ch == "_": - if ch == "b" and ch_next in ("'", '"'): - eval_str += "b" # start of a byte string literal - elif var_name is None: - if ch.isalpha(): - var_name = ch # start of a variable - else: - eval_str += ch # assume a numeric value - else: - var_name += ch - elif ch == "(" and end_quote_char is None: - paren_count += 1 - eval_str += ch - elif ch == ")" and end_quote_char is None: - paren_count -= 1 - if paren_count < 0: - msg = "Mismatched paren" - log.warn("Bad query: " + msg) - raise ValueError(msg) - eval_str += ch - else: - # just add to eval_str - eval_str += ch - i = i + 1 - if end_quote_char: - msg = "no matching quote character" - log.warn("Bad Query: " + msg) - raise ValueError(msg) - if var_count == 0: - msg = "No field value" - log.warn("Bad query: " + msg) - raise ValueError(msg) - if paren_count != 0: - msg = "Mismatched paren" - log.warn("Bad query: " + msg) - raise ValueError(msg) - return eval_str - - -def getQueryDtype(dt): - """make a dtype for query response""" - field_names = dt.names - # make up a index field name that doesn't conflict with existing names - index_name = "index" - for i in range(len(field_names)): - if index_name in field_names: - index_name = "_" + index_name - else: - break - - dt_fields = [(index_name, "uint64"), ] - for i in range(len(dt)): - dt_fields.append((dt.names[i], dt[i])) - query_dt = np.dtype(dt_fields) - - return query_dt - - -def chunkQuery( - chunk_id=None, - chunk_layout=None, - chunk_arr=None, - slices=None, - query=None, - query_update=None, - select_dt=None, - limit=0, -): - """ - Run query on chunk and selection - """ - msg = f"chunkQuery - chunk_id: {chunk_id} query: [{query}] slices: {slices}, limit: {limit}" - log.debug(msg) - - if not isinstance(chunk_arr, np.ndarray): - raise TypeError("unexpected array type") - - dims = chunk_arr.shape - - rank = len(dims) - - dset_dt = chunk_arr.dtype - if select_dt is None: - select_dt = dset_dt - - if rank != 1: - msg = "Query operations only supported on one-dimensional datasets" - log.error(msg) - raise ValueError(msg) - - if not slices: - slices = [slice(0, dims[0], 1), ] - log.debug(f"chunkQuery slices: {slices}") - if len(slices) != rank: - msg = "Selection rank does not match shape rank" - log.error(msg) - raise ValueError(msg) - slices = tuple(slices) - chunk_sel = chunk_arr[slices] - - chunk_coord = getChunkCoordinate(chunk_id, chunk_layout) - - # do query selection - field_names = dset_dt.names - - # get the eval str - eval_str = _getEvalStr(query, "chunk_sel", field_names) - if eval_str: - log.debug(f"eval_str: {eval_str}") - else: - log.debug("no eval_str") - - # check for a where in statement - where_field = _getWhereFieldName(query) - if where_field: - log.debug(f"where_field: {where_field}") - if where_field not in field_names: - msg = f"where field {where_field} is not a member of dataset type" - raise ValueError(msg) - where_elements = _getWhereElements(query) - if not where_elements: - msg = "query: where key word with no elements" - raise ValueError(msg) - # convert to ndarray, checking that we can convert to our dtype along the way - try: - where_elements_arr = np.array(where_elements, dtype=dset_dt[where_field]) - except ValueError: - msg = "where elements are not compatible with field datatype" - raise ValueError(msg) - isin_mask = np.isin(chunk_sel[where_field], where_elements_arr) - - if not np.any(isin_mask): - # all false - log.debug("query - no rows found for where elements") - return None - - isin_indices = np.where(isin_mask) - if not isinstance(isin_indices, tuple): - log.warn(f"expected where_indices of tuple but got: {type(isin_indices)}") - return None - if len(isin_indices) == 0: - log.warn("chunkQuery - got empty tuple where in result") - return None - - isin_indices = isin_indices[0] - if not isinstance(isin_indices, np.ndarray): - log.warn(f"expected isin_indices of ndarray but got: {type(isin_indices)}") - return None - nrows = isin_indices.shape[0] - elif eval_str: - log.debug("no where keyword") - isin_indices = None - else: - log.warn("query - no eval and no where in, returning None") - return None - - if query_update: - if where_field: - msg = "query update is not supported with where in" - raise ValueError(msg) - replace_mask = [None,] * len(field_names) - for i in range(len(field_names)): - field_name = field_names[i] - if field_name in query_update: - replace_mask[i] = query_update[field_name] - log.debug(f"chunkQuery - replace_mask: {replace_mask}") - replace_fields = [None, ] * len(field_names) - if replace_mask == replace_fields: - msg = "chunkQuery - no fields found in query_update" - raise ValueError(msg) - else: - replace_mask = None - - if eval_str: - where_indices = np.where(eval(eval_str)) - if not isinstance(where_indices, tuple): - log.warn(f"expected where_indices of tuple but got: {type(where_indices)}") - return None - if len(where_indices) == 0: - log.warn("chunkQuery - got empty tuple where result") - return None - - where_indices = where_indices[0] - if not isinstance(where_indices, np.ndarray): - log.warn(f"expected where_indices of ndarray but got: {type(where_indices)}") - return None - nrows = where_indices.shape[0] - log.debug(f"chunkQuery - {nrows} where rows found") - else: - where_indices = None - - if isin_indices is None: - pass # skip intersection - else: - if where_indices is None: - # just use the isin_indices - where_indices = isin_indices - else: - # interest the two sets of indices - intersect = np.intersect1d(where_indices, isin_indices) - - nrows = intersect.shape[0] - if nrows == 0: - log.debug("chunkQuery - no rows found after intersect with is in") - return None - else: - log.debug(f"chunkQuery - intersection, {nrows} found") - # use the intsection as our new where index - where_indices = intersect - - if limit > 0 and nrows > limit: - # truncate to limit rows - log.debug(f"limiting response to {limit} rows") - where_indices = where_indices[:limit] - nrows = limit - - where_result = chunk_sel[where_indices].copy() - - if replace_mask and nrows > 0: - log.debug(f"apply replace_mask: {replace_mask}") - for i in range(len(field_names)): - field = field_names[i] - if replace_mask[i] is not None: - where_result[field] = replace_mask[i] - # update source array - for i in range(nrows): - index = where_indices[i] - row = where_result[i] - chunk_arr[index] = row - - # adjust the index to correspond with the dataset - s = slices[0] - if s.step is None: - s = slice(s.start, s.stop, 1) - start = s.start + chunk_coord[0] - if start > 0: - # can just increment every value by same amount - where_indices += start - if s.step and s.step > 1: - for i in range(nrows): - where_indices[i] = where_indices[i] + (s.step - 1) * i - - dt_rsp = getQueryDtype(select_dt) - # construct response array - rsp_arr = np.zeros((nrows,), dtype=dt_rsp) - field_names = select_dt.names - for field in field_names: - rsp_arr[field] = where_result[field] - index_name = dt_rsp.names[0] - rsp_arr[index_name] = where_indices - log.debug(f"chunkQuery returning {len(rsp_arr)} rows") - - return rsp_arr diff --git a/hsds/util/domainUtil.py b/hsds/util/domainUtil.py index 8d713739..21c9a4e5 100644 --- a/hsds/util/domainUtil.py +++ b/hsds/util/domainUtil.py @@ -268,7 +268,6 @@ def validateDomainKey(domain_key): def getDomainFromRequest(request, validate=True): - # print(f"getDomainFromRequest: {request}, validate={validate}") app = request.app domain = None bucket = None @@ -299,7 +298,7 @@ def getDomainFromRequest(request, validate=True): if bucket: if validate and not isValidBucketName(bucket): raise ValueError(f"bucket name: {bucket} is not valid") - if domain[0] == "/": + if len(domain) > 0 and domain[0] == "/": domain = bucket + domain return domain diff --git a/hsds/util/dsetUtil.py b/hsds/util/dsetUtil.py index 044127f0..71cec8dd 100644 --- a/hsds/util/dsetUtil.py +++ b/hsds/util/dsetUtil.py @@ -10,428 +10,74 @@ # request a copy from help@hdfgroup.org. # ############################################################################## -from aiohttp.web_exceptions import HTTPBadRequest, HTTPInternalServerError +from aiohttp.web_exceptions import HTTPBadRequest import math +import re -from .. import hsds_logger as log - -""" -Filters that are known to HSDS. -Format is: - FILTER_CODE, FILTER_ID, Name - - H5Z_FILTER_FLETCHER32, H5Z_FILTER_SZIP, H5Z_FILTER_NBIT, - and H5Z_FILTER_SCALEOFFSET, are not currently supported. - - Non-supported filters metadata will be stored, but are - not (currently) used for compression/decompression. -""" - -FILTER_DEFS = ( - ("H5Z_FILTER_NONE", 0, "none"), - ("H5Z_FILTER_DEFLATE", 1, "gzip"), # aka as "zlib" for blosc - ("H5Z_FILTER_SHUFFLE", 2, "shuffle"), - ("H5Z_FILTER_FLETCHER32", 3, "fletcher32"), - ("H5Z_FILTER_SZIP", 4, "szip"), - ("H5Z_FILTER_NBIT", 5, "nbit"), - ("H5Z_FILTER_SCALEOFFSET", 6, "scaleoffset"), - ("H5Z_FILTER_LZF", 32000, "lzf"), - ("H5Z_FILTER_BLOSC", 32001, "blosclz"), - ("H5Z_FILTER_SNAPPY", 32003, "snappy"), - ("H5Z_FILTER_LZ4", 32004, "lz4"), - ("H5Z_FILTER_LZ4HC", 32005, "lz4hc"), - ("H5Z_FILTER_BITSHUFFLE", 32008, "bitshuffle"), - ("H5Z_FILTER_ZSTD", 32015, "zstd"), -) - -COMPRESSION_FILTER_IDS = ( - "H5Z_FILTER_DEFLATE", - "H5Z_FILTER_SZIP", - "H5Z_FILTER_SCALEOFFSET", - "H5Z_FILTER_LZF", - "H5Z_FILTER_BLOSC", - "H5Z_FILTER_SNAPPY", - "H5Z_FILTER_LZ4", - "H5Z_FILTER_LZ4HC", - "H5Z_FILTER_ZSTD", -) - -COMPRESSION_FILTER_NAMES = ( - "gzip", - "szip", - "lzf", - "blosclz", - "snappy", - "lz4", - "lz4hc", - "zstd", -) - -CHUNK_LAYOUT_CLASSES = ( - "H5D_CHUNKED", - "H5D_CHUNKED_REF", - "H5D_CHUNKED_REF_INDIRECT", - "H5D_CONTIGUOUS_REF", -) - - -# copied from arrayUtil.py -def isVlen(dt): - """ - Return True if the type contains variable length elements - """ - is_vlen = False - if len(dt) > 1: - names = dt.names - for name in names: - if isVlen(dt[name]): - is_vlen = True - break - else: - if dt.metadata and "vlen" in dt.metadata: - is_vlen = True - return is_vlen - - -def getFilterItem(key): - """ - Return filter code, id, and name, based on an id, a name or a code. - """ - - if key == "deflate": - key = "gzip" # use gzip as equivalent - for item in FILTER_DEFS: - for i in range(3): - if key == item[i]: - return {"class": item[0], "id": item[1], "name": item[2]} - return None # not found - - -def getFilters(dset_json): - """Return list of filters, or empty list""" - if "creationProperties" not in dset_json: - return [] - creationProperties = dset_json["creationProperties"] - if "filters" not in creationProperties: - return [] - filters = creationProperties["filters"] - return filters - - -def getCompressionFilter(filters): - """Return compression filter from filters, or None""" - for filter in filters: - if "class" not in filter: - msg = f"filter option: {filter} with no class key" - log.warn(msg) - continue - filter_class = filter["class"] - if filter_class in COMPRESSION_FILTER_IDS: - return filter - if all( - ( - filter_class == "H5Z_FILTER_USER", - "name" in filter, - filter["name"] in COMPRESSION_FILTER_NAMES, - ) - ): - return filter - return None - - -def getShuffleFilter(filters): - """Return shuffle filter, or None""" - FILTER_CLASSES = ("H5Z_FILTER_SHUFFLE", "H5Z_FILTER_BITSHUFFLE") - for filter in filters: - log.debug(f"filter: {filter}") - if "class" not in filter: - log.warn(f"filter option: {filter} with no class key") - continue - filter_class = filter["class"] - if filter_class in FILTER_CLASSES: - log.debug(f"found filter: {filter}") - return filter - - log.debug("Shuffle filter not used") - return None - - -def getFilterOps(app, dset_id, filters, dtype=None, chunk_shape=None): - """Get list of filter operations to be used for this dataset""" - filter_map = app["filter_map"] - - try: - if dset_id in filter_map: - log.debug(f"returning filter from filter_map for dset: {dset_id}") - return filter_map[dset_id] - except TypeError: - log.error(f"getFilterOps TypeError - dset_id: {dset_id} filter_map: {filter_map}") - raise - - compressionFilter = getCompressionFilter(filters) - log.debug(f"got compressionFilter: {compressionFilter}") - - filter_ops = {} - - shuffleFilter = getShuffleFilter(filters) - - if shuffleFilter and not isVlen(dtype): - shuffle_name = shuffleFilter["name"] - if shuffle_name == "shuffle": - filter_ops["shuffle"] = 1 # use regular shuffle - elif shuffle_name == "bitshuffle": - filter_ops["shuffle"] = 2 # use bitshuffle - else: - log.warn(f"unexpected shuffleFilter: {shuffle_name}") - filter_ops["shuffle"] = 0 # no shuffle - else: - filter_ops["shuffle"] = 0 # no shuffle - - if compressionFilter: - if compressionFilter["class"] == "H5Z_FILTER_DEFLATE": - filter_ops["compressor"] = "zlib" # blosc compressor - else: - if "name" in compressionFilter: - filter_ops["compressor"] = compressionFilter["name"] - else: - filter_ops["compressor"] = "lz4" # default to lz4 - if "level" not in compressionFilter: - filter_ops["level"] = 5 # medium level - else: - filter_ops["level"] = int(compressionFilter["level"]) - - if filter_ops: - # save the chunk shape and dtype - filter_ops["chunk_shape"] = chunk_shape - filter_ops["dtype"] = dtype - log.debug(f"save filter ops for {dset_id}") - filter_map[dset_id] = filter_ops # save +from h5json.shape_util import getShapeDims +from h5json.objid import isValidUuid +from h5json.array_util import jsonToArray, bytesArrayToList +from h5json import selections - return filter_ops - else: - return None - - -def getDsetRank(dset_json): - """Get rank returning 0 for sclar or NULL datashapes""" - datashape = dset_json["shape"] - if datashape["class"] == "H5S_NULL": - return 0 - if datashape["class"] == "H5S_SCALAR": - return 0 - if "dims" not in datashape: - log.warn(f"expected to find dims key in shape_json: {datashape}") - return 0 - dims = datashape["dims"] - rank = len(dims) - return rank +from .. import hsds_logger as log +from .chunkUtil import _toArraySlice, slice_stop, toNumpyIndex -def isNullSpace(dset_json): - """Return true if this dataset is a null dataspace""" - datashape = dset_json["shape"] - if datashape["class"] == "H5S_NULL": - return True - else: +def isSelectAll(selection, dims): + """ return True if the selection covers the entire dataspace """ + if len(selection.shape) != len(dims): + raise ValueError("isSelectAll - dimensions don't match") + if selection.select_type not in (selections.H5S_SEL_ALL, selections.H5S_SEL_HYPERSLABS): return False + for dim in range(len(dims)): + if selection.step[dim] not in (None, 1): + return False + if selection.start[dim] != 0: + return False + if selection.count[dim] != dims[dim]: + return False + return True + + +def isSelect(params, body=None): + """ return True if a select param is set in query params or in the + request body """ + if "select" in params and params["select"]: + return True - -def isScalarSpace(dset_json): - """ return true if this is a scalar dataset """ - datashape = dset_json["shape"] - is_scalar = False - if datashape["class"] == "H5S_NULL": - is_scalar = False - elif datashape["class"] == "H5S_SCALAR": - is_scalar = True - else: - if "dims" not in datashape: - log.warn(f"expected to find dims key in shape_json: {datashape}") - is_scalar = False - else: - dims = datashape["dims"] - if len(dims) == 0: - # guess this properly be a H5S_SCALAR class - # but treat this as equivalent - is_scalar = True - return is_scalar - - -def getHyperslabSelection(dsetshape, start=None, stop=None, step=None): - """ - Get slices given lists of start, stop, step values - - TBD: for step>1, adjust the slice to not extend beyond last - data point returned - """ - rank = len(dsetshape) - if start: - if not isinstance(start, (list, tuple)): - start = [start] - if len(start) != rank: - msg = "Bad Request: start array length not equal to dataset rank" - log.warn(msg) - raise HTTPBadRequest(reason=msg) - for dim in range(rank): - if start[dim] < 0 or start[dim] >= dsetshape[dim]: - msg = "Bad Request: start index invalid for dim: " + str(dim) - log.warn(msg) - raise HTTPBadRequest(reason=msg) - else: - start = [] - for dim in range(rank): - start.append(0) - - if stop: - if not isinstance(stop, (list, tuple)): - stop = [stop] - if len(stop) != rank: - msg = "Bad Request: stop array length not equal to dataset rank" - log.warn(msg) - raise HTTPBadRequest(reason=msg) - for dim in range(rank): - if stop[dim] <= start[dim] or stop[dim] > dsetshape[dim]: - msg = "Bad Request: stop index invalid for dim: " + str(dim) - log.warn(msg) - raise HTTPBadRequest(reason=msg) - else: - stop = [] - for dim in range(rank): - stop.append(dsetshape[dim]) - - if step: - if not isinstance(step, (list, tuple)): - step = [step] - if len(step) != rank: - msg = "Bad Request: step array length not equal to dataset rank" - log.warn(msg) - raise HTTPBadRequest(reason=msg) - for dim in range(rank): - if step[dim] <= 0 or step[dim] > dsetshape[dim]: - msg = "Bad Request: step index invalid for dim: " + str(dim) - log.warn(msg) - raise HTTPBadRequest(reason=msg) - else: - step = [] - for dim in range(rank): - step.append(1) - - slices = [] - - for dim in range(rank): - - try: - s = slice(int(start[dim]), int(stop[dim]), int(step[dim])) - except ValueError: - msg = "Bad Request: invalid start/stop/step value" - log.warn(msg) - raise HTTPBadRequest(reason=msg) - slices.append(s) - return tuple(slices) - - -def getSelectionShape(selection): - """Return the shape of the given selection. - Examples (selection -> returned shape): - [(3,7,1)] -> [4] - [(3, 7, 3)] -> [1] - [(44, 52, 1), (48,52,1)] -> [8, 4] - [[1,2,7]] -> - """ - shape = [] - rank = len(selection) - coordinate_extent = None - for i in range(rank): - s = selection[i] - if isinstance(s, slice): - extent = 0 - if s.step and s.step > 1: - step = s.step - else: - step = 1 - if s.stop > s.start: - extent = s.stop - s.start - if step > 1 and extent > 0: - extent = extent // step - if (s.stop - s.start) % step != 0: - extent += 1 - shape.append(extent) - else: - # coordinate list - extent = len(s) - if coordinate_extent is None: - coordinate_extent = extent - shape.append(extent) - elif coordinate_extent != extent: - msg = "shape mismatch: indexing arrays could not be broadcast together " - msg += f"with shapes ({coordinate_extent},) ({extent},)" - log.warn(msg) - raise HTTPBadRequest(reason=msg) - else: - pass - - return shape - - -def getShapeDims(shape): - """ - Get dims from a given shape json. Return [1,] for Scalar datasets, - None for null dataspaces - """ - dims = None - if isinstance(shape, int): - dims = [shape, ] - elif isinstance(shape, list) or isinstance(shape, tuple): - dims = shape # can use as is - elif isinstance(shape, str): - # only valid string value is H5S_NULL - if shape != "H5S_NULL": - raise ValueError("Invalid value for shape") - dims = None - elif isinstance(shape, dict): - if "class" not in shape: - raise ValueError("'class' key not found in shape") - if shape["class"] == "H5S_NULL": - dims = None - elif shape["class"] == "H5S_SCALAR": - dims = [1,] - elif shape["class"] == "H5S_SIMPLE": - if "dims" not in shape: - raise ValueError("'dims' key expected for shape") - dims = shape["dims"] - else: - raise ValueError("Unknown shape class: {}".format(shape["class"])) - else: - raise ValueError(f"Unexpected shape class: {type(shape)}") - - return dims + if isinstance(body, dict): + if "select" in body and body["select"]: + return True + for key in ("start", "stop", "step"): + if key in body and body[key]: + return True + return False -def isSelectAll(slices, dims): - """ return True if the selection covers the entire dataspace """ - if len(slices) != len(dims): - raise ValueError("isSelectAll - dimensions don't match") - is_all = True - for (s, dim) in zip(slices, dims): - if s.step is not None and s.step != 1: - is_all = False - break - if s.start != 0: - is_all = False - break - if s.stop != dim: - is_all = False - break - return is_all +def getSelectParam(params, body=None): + """ return the select value (a query string or dict), if any, from + request query params or JSON body. Raises ValueError if select is + given in both. Returns None if no selection is specified. """ + select = None + if body and isinstance(body, dict): + if "select" in body and body["select"]: + select = body.get("select") + elif "start" in body and "stop" in body: + select = body + if "select" in params and params["select"]: + if select is not None: + raise ValueError("select defined in both request body and query parameters") + select = params.get("select") + return select def getQueryParameter(request, query_name, body=None, default=None): """ - Herlper function, get query parameter value from request. + Helper function, get query parameter value from request. If body is provided (as a JSON object) look in JSON and if not found look for query param. Return default value (or None) if not found """ - # as a convience, look up different capitilizations of query name + # as a convenience, look up different capitalizations of query name params = request.rel_url.query query_names = [] query_names.append(query_name.lower()) @@ -566,11 +212,7 @@ def getSelectionList(select, dims): if select is None or len(select) == 0: """Return set of slices covering data space""" - slices = [] - for extent in dims: - s = slice(0, extent, 1) - slices.append(s) - return tuple(slices) + return selections.select(tuple(dims), ...) # convert selection to list by dimension elements = _getSelectElements(select) @@ -657,7 +299,16 @@ def getSelectionList(select, dims): select_list.append(s) # end dimension loop log.debug(f"select_list: {select_list}") - return tuple(select_list) + return selections.select(tuple(dims), tuple(select_list)) + + +def getSelect(params, dims, body=None): + """ return the requested selection region, if any, as a + selections.Selection, given the extents to select against. + If no selection is specified in query params or JSON body, + returns a selection covering the entire dims. """ + select = getSelectParam(params, body=body) + return getSelectionList(select, dims) def get_slices(select, dset_json): @@ -669,31 +320,36 @@ def get_slices(select, dset_json): dset_id = dset_json["id"] datashape = dset_json["shape"] - if datashape["class"] == "H5S_NULL": + shape_class = datashape["class"] + if shape_class == "H5S_NULL": msg = "Null space datasets can not be used as target for GET value" log.warn(msg) raise HTTPBadRequest(reason=msg) - dims = getShapeDims(datashape) # throws 400 for HS_NULL dsets + if shape_class == "H5S_SCALAR": + # treat as a synthetic rank-1, extent-1 space + slices = selections.select((1,), (slice(0, 1, 1),)) + else: + dims = getShapeDims(datashape) # throws 400 for HS_NULL dsets - try: - slices = getSelectionList(select, dims) - except ValueError: - msg = f"Invalid selection: {select} on dims: {dims} " - msg += f"for dataset: {dset_id}" - log.warn(msg) - raise + try: + slices = getSelectionList(select, dims) + except ValueError: + msg = f"Invalid selection: {select} on dims: {dims} " + msg += f"for dataset: {dset_id}" + log.warn(msg) + raise return slices def getSelectionPagination(select, dims, itemsize, max_request_size): """ - Paginate a select tupe into multiple selects where each - select requires less than max_request_size bytes""" + Paginate a selections.Selection into multiple Selections where each + requires less than max_request_size bytes""" msg = f"getSelectionPagination - select: {select}, dims: {dims}, " msg += f"itemsize: {itemsize}, max_request_size: {max_request_size}" log.debug(msg) - select_shape = getSelectionShape(select) + select_shape = select.mshape log.debug(f"getSelectionPagination - select_shape: {select_shape}") select_size = math.prod(select_shape) * itemsize log.debug(f"getSelectionPagination - select_size: {select_size}") @@ -702,18 +358,18 @@ def getSelectionPagination(select, dims, itemsize, max_request_size): log.debug("getSelectionPagination - not needed") return (select,) + slices = select.slices + # get pagination dimension - first dimension with > 1 extent rank = len(dims) paginate_dim = None paginate_extent = None for i in range(rank): - s = select[i] + s = slices[i] if isinstance(s, slice): + s = _toArraySlice(s) + s = slice(s.start, slice_stop(s), s.step) paginate_extent = 0 - if s.step and s.stop > 1: - step = s.step - else: - step = 1 if s.stop > s.start: paginate_extent = s.stop - s.start else: @@ -735,11 +391,10 @@ def getSelectionPagination(select, dims, itemsize, max_request_size): page_size = select_size // page_count log.debug(f"getSelectionPagination - page_size: {page_size}") - s = select[paginate_dim] - # log.debug(f"pagination dim: {paginate_dim} select: {s} paginate_extent: {paginate_extent}") - # page_extent = -(-max_request_size // page_size) - # log.debug(f"getSelectionPagination - page_extent: {page_extent}") - # page_count = -(-paginate_extent // page_extent) + s = slices[paginate_dim] + if isinstance(s, slice): + s = _toArraySlice(s) + s = slice(s.start, slice_stop(s), s.step) if paginate_extent < page_count: msg = f"select pagination unable to paginate select dim: {paginate_dim} " msg += f"into {page_count} pages" @@ -752,10 +407,7 @@ def getSelectionPagination(select, dims, itemsize, max_request_size): paginate_slices = [] if isinstance(s, slice): start = s.start - if s.step and s.stop > 1: - step = s.step - else: - step = 1 + step = s.step if s.step else 1 while start < s.stop: stop = start + page_extent @@ -778,7 +430,7 @@ def getSelectionPagination(select, dims, itemsize, max_request_size): log.debug(f"page_coord s[{start}:{stop}]") page_coord = s[start:stop] log.debug(f"page coords: {page_coord}") - paginate_slices.append(tuple(page_coord)) + paginate_slices.append(list(page_coord)) start = stop # adjust page count to number to actual pagination page_count = len(paginate_slices) @@ -788,21 +440,125 @@ def getSelectionPagination(select, dims, itemsize, max_request_size): # dimension, original selection for each other dimension pagination = [] for page in range(page_count): - s = [] + page_args = [] for i in range(rank): if i == paginate_dim: - s.append(paginate_slices[page]) + page_args.append(paginate_slices[page]) else: - s.append(select[i]) - pagination.append(tuple(s)) + page_args.append(slices[i]) + pagination.append(selections.select(select.shape, tuple(page_args))) pagination = tuple(pagination) # log.debug(f"returning pagination: {pagination}") return pagination +_REGIONREF_ATTR_RE = re.compile(r"^/(groups|datasets)/([^/]+)/attributes/([^/]+)$") +_REGIONREF_DSET_RE = re.compile(r"^/datasets/([^/]+)$") + + +def parseRegionRefParam(regionref): + """Parse a 'regionref' query param value into (collection, obj_id, attr_name). + + Accepts exactly these forms: + /groups//attributes/ + /datasets//attributes/ + /datasets/ + attr_name is None for the bare dataset form. Raises HTTPBadRequest for + any other form, or if the extracted id isn't a valid id for its + collection. + """ + m = _REGIONREF_ATTR_RE.match(regionref) + if m: + collection, obj_id, attr_name = m.group(1), m.group(2), m.group(3) + else: + m = _REGIONREF_DSET_RE.match(regionref) + if not m: + msg = f"Invalid regionref path: {regionref}" + log.warn(msg) + raise HTTPBadRequest(reason=msg) + collection, obj_id, attr_name = "datasets", m.group(1), None + + if not isValidUuid(obj_id, obj_class=collection): + msg = f"Invalid regionref object id: {obj_id}" + log.warn(msg) + raise HTTPBadRequest(reason=msg) + + return collection, obj_id, attr_name + + +def extractJsonArrayElement(shape, dtype, json_value, selection): + """Extract the single JSON-decoded element selected by selection (which + must select exactly one element) out of a JSON-encoded array value. + + shape/dtype describe json_value as a whole (e.g. an attribute's own + shape/type); selection is a selections.Selection over that same shape. + Caller is responsible for having already verified selection.nselect == 1. + """ + arr = jsonToArray(shape, dtype, json_value) + sub_arr = arr[toNumpyIndex(selection)] + return unwrapSingleElement(bytesArrayToList(sub_arr)) + + +def unwrapSingleElement(value): + """Peel away single-element list nesting down to the leaf value. + Used after extracting a selection known to select exactly one element, + where the surrounding shape may still be e.g. (1,) or (1, 1).""" + while isinstance(value, list): + if len(value) != 1: + raise ValueError("expected a single element") + value = value[0] + return value + + +def regionRefSelectionToTargetSelection(ref_json, target_dims): + """Given a decoded region-reference JSON value ({"id": ...} optionally + with "select_type"/"selection" or "selection_dict"), reconstruct its + selection and re-apply it against target_dims (the shape of the dataset + the caller actually wants to read from). + + Raises HTTPBadRequest if the region reference's selection rank doesn't + match len(target_dims), or if it selects outside target_dims' bounds. + A region reference with no selection info (just {"id": ...}) selects + the entire target dataset. + """ + if "selection_dict" in ref_json: + ref_sel = selections.from_dict(ref_json["selection_dict"]) + elif "select_type" in ref_json: + ref_sel = selections.from_region_json(ref_json) + else: + # no selection info - the whole (target) dataset is selected + return selections.select(tuple(target_dims), ...) + + ref_rank = len(ref_sel.shape) + if ref_rank != len(target_dims): + msg = f"regionref selection rank ({ref_rank}) does not match " + msg += f"dataset rank ({len(target_dims)})" + log.warn(msg) + raise HTTPBadRequest(reason=msg) + + mins, maxs = ref_sel.bbox + if mins is None: + msg = "regionref selection is empty" + log.warn(msg) + raise HTTPBadRequest(reason=msg) + for dim in range(ref_rank): + if maxs[dim] > target_dims[dim]: + msg = f"regionref selection for dim {dim} (extent {maxs[dim]}) " + msg += f"exceeds dataset extent ({target_dims[dim]})" + log.warn(msg) + raise HTTPBadRequest(reason=msg) + + try: + return selections.select(tuple(target_dims), ref_sel.slices) + except ValueError as ve: + msg = f"Invalid regionref selection: {ve}" + log.warn(msg) + raise HTTPBadRequest(reason=msg) + + def getSliceQueryParam(sel): """ - Helper method - set query parameter for given shape + selection + Helper method - set query parameter for given selections.Selection Query arg should be in the form: [, , ... , ] brackets are optional for one dimensional arrays. @@ -812,12 +568,18 @@ def getSliceQueryParam(sel): start, end, and stride: n:m:s """ # pass dimensions, and selection as query params - rank = len(sel) + slices = sel.slices + rank = len(slices) if rank > 0: sel_param = "[" for i in range(rank): - s = sel[i] + s = slices[i] if isinstance(s, slice): + # sel.slices encodes stop as start + count, not a real + # coordinate for step > 1 dims; convert to a real coordinate + # stop before writing it out as a query param + s = _toArraySlice(s) + s = slice(s.start, slice_stop(s), s.step) sel_param += str(s.start) sel_param += ":" sel_param += str(s.stop) @@ -861,58 +623,10 @@ def setChunkDimQueryParam(params, dims): extent = dims[i] dim_param += str(extent) dim_param += "]" - log.debug("dim query param: {}".format(dim_param)) + log.debug(f"dim query param: {dim_param}") params["dim"] = dim_param -def getDsetMaxDims(dset_json): - """ - Get maxdims from a given shape. Return [1,] for Scalar datasets - - Use with H5S_NULL datasets will throw a 400 error. - """ - if "shape" not in dset_json: - log.error("No shape found in dset_json") - raise HTTPInternalServerError() - shape_json = dset_json["shape"] - maxdims = None - if shape_json["class"] == "H5S_NULL": - msg = "Expected shape class other than H5S_NULL" - log.warn(msg) - raise HTTPBadRequest(reason=msg) - elif shape_json["class"] == "H5S_SCALAR": - maxdims = [ - 1, - ] - elif shape_json["class"] == "H5S_SIMPLE": - if "maxdims" in shape_json: - maxdims = shape_json["maxdims"] - else: - log.error("Unexpected shape class: {}".format(shape_json["class"])) - raise HTTPInternalServerError() - return maxdims - - -def getChunkLayout(dset_json): - """Get chunk layout. Throw 500 if used with non-H5D_CHUNKED layout""" - if "layout" not in dset_json: - log.error("No layout found in dset_json") - raise HTTPInternalServerError() - layout_json = dset_json["layout"] - if "class" not in layout_json: - log.error(f"Expected class key for layout: {layout_json}") - raise HTTPInternalServerError() - layout_class = layout_json["class"] - if layout_class not in CHUNK_LAYOUT_CLASSES: - log.error(f"Unexpected shape layout: {layout_class}") - raise HTTPInternalServerError() - if "dims" not in layout_json: - log.error(f"Expected dims key in layout: {layout_json}") - raise HTTPInternalServerError() - layout = layout_json["dims"] - return layout - - def getChunkInitializer(dset_json): """ get initializer application and arguments if set """ initializer = None @@ -962,102 +676,3 @@ def getPreviewQuery(dims): select += "0:1," select += "]" return select - - -def isExtensible(dims, maxdims): - """ - Determine if the dataset can be extended - """ - if maxdims is None or len(dims) == 0: - return False - rank = len(dims) - if len(maxdims) != rank: - raise ValueError("rank of maxdims does not match dataset") - for n in range(rank): - # TBD - shouldn't have H5S_UNLIMITED in any new files. - # Remove check once this is confirmed - if maxdims[n] in (0, "H5S_UNLIMITED") or maxdims[n] > dims[n]: - return True - return False - - -def getDatasetLayout(dset_json): - """ Return layout json from creation property list or layout json """ - layout = None - - if "creationProperties" in dset_json: - cp = dset_json["creationProperties"] - if "layout" in cp: - layout = cp["layout"] - if not layout and "layout" in dset_json: - layout = dset_json["layout"] - if not layout: - log.warn(f"no layout for {dset_json}") - return layout - - -def getDatasetLayoutClass(dset_json): - """ return layout class """ - layout = getDatasetLayout(dset_json) - if layout and "class" in layout: - layout_class = layout["class"] - else: - layout_class = None - return layout_class - - -def getChunkDims(dset_json): - """ get chunk shape for given dset_json """ - - layout = getDatasetLayout(dset_json) - if layout and "dims" in layout: - return layout["dims"] - else: - # H5D_COMPACT and H5D_CONTIGUOUS will not have a dims key - # Check the layout dict in dset_json to see if it's - # defined there - if "layout" in dset_json: - layout = dset_json["layout"] - if "dims" in layout: - return layout["dims"] - return None - - -class ItemIterator: - """ - Class to iterator through items in a selection - """ - - def __init__(self, selection): - self._selection = selection - self._rank = len(selection) - self._index = [0,] * self._rank - for i in range(self._rank): - s = self._selection[i] - self._index[i] = s.start - - def __iter__(self): - return self - - def next(self): - if self._index[0] >= self._selection[0].stop: - # ran past last item, end iteration - raise StopIteration() - dim = self._rank - 1 - - index = [0, ] * self._rank - for i in range(self._rank): - index[i] = self._index[i] - while dim >= 0: - s = self._selection[dim] - self._index[dim] += s.step - if self._index[dim] < s.stop: - if self._rank == 1: - index = index[0] - return index - if dim > 0: - self._index[dim] = s.start - dim -= 1 - if self._rank == 1: - index = index[0] # return int, not list - return index diff --git a/hsds/util/fileClient.py b/hsds/util/fileClient.py index 1bc5e786..0d7d88ba 100644 --- a/hsds/util/fileClient.py +++ b/hsds/util/fileClient.py @@ -88,7 +88,7 @@ def _getFileStats(self, filepath, data=None): return key_stats def _file_stats_increment(self, counter, inc=1): - """Incremenet the indicated connter""" + """Increment the indicated counter""" if "file_stats" not in self._app: # setup stats file_stats = {} @@ -173,8 +173,34 @@ async def get_object(self, key, bucket=None, offset=0, length=-1): msg = f"Unexpected Exception {type(e)} get get_object {key}: {e}" log.error(msg) raise HTTPInternalServerError() + + posix_delay = config.get("posix_delay", default=0.0) + if posix_delay > 0.0: + log.warn(f"posix_delay for get_object, sleep for: {posix_delay}") + await asyncio.sleep(posix_delay) + return data + def _mkdir(self, dirpath): + """ create the given directory if it doesn't already exist """ + try: + dirpath = pp.normpath(dirpath) + log.debug(f"normpath: {dirpath}") + + if not pp.isdir(dirpath): + log.debug(f"mkdir({dirpath})") + mkdir(dirpath) + else: + log.debug(f"isdir {dirpath} found") + except IOError as ioe: + if ioe.errno == 17: + # likely directory was created by another process since we checked + log.warn(f"mkdir failed, {dirpath} created outside this process") + else: + msg = f"fileClient: IOError on mkdir {dirpath}: {ioe}" + log.warn(msg) + raise HTTPInternalServerError() + async def put_object(self, key, data, bucket=None): """Write data to given key. Returns client specific dict on success @@ -202,15 +228,7 @@ async def put_object(self, key, data, bucket=None): for key_dir in key_dirs: dirpath = pp.join(dirpath, key_dir) log.debug(f"pp.join({key_dir}) => {dirpath}") - - dirpath = pp.normpath(dirpath) - log.debug(f"normpath: {dirpath}") - - if not pp.isdir(dirpath): - log.debug(f"mkdir({dirpath})") - mkdir(dirpath) - else: - log.debug(f"isdir {dirpath} found") + self._mkdir(dirpath) log.debug(f"open({filepath}, 'wb')") async with aiofiles.open(filepath, loop=loop, mode="wb") as f: await f.write(data) @@ -242,6 +260,12 @@ async def put_object(self, key, data, bucket=None): msg = f"fileClient.put_object {key} complete, " msg += f"write_rsp: {write_rsp}" log.debug(msg) + + posix_delay = config.get("posix_delay", default=0.0) + if posix_delay > 0.0: + log.warn(f"posix_delay for put_object, sleep for: {posix_delay}") + await asyncio.sleep(posix_delay) + return write_rsp async def delete_object(self, key, bucket=None): @@ -282,7 +306,11 @@ async def delete_object(self, key, bucket=None): msg = f"Unexpected Exception {type(e)} deleting file obj {key}: {e}" log.error(msg) raise HTTPInternalServerError() - await asyncio.sleep(0) # for async compat + + posix_delay = config.get("posix_delay", default=0.0) + if posix_delay > 0.0: + log.warn(f"posix_delay for delete_object , sleep for: {posix_delay}") + await asyncio.sleep(posix_delay) # for async compat async def is_object(self, key, bucket=None): self._validateBucket(bucket) @@ -417,6 +445,11 @@ async def list_keys( msg == f"got {len(key_names)}" log.warning(msg) + posix_delay = config.get("posix_delay", default=0.0) + if posix_delay > 0.0: + log.warn(f"posix_delay for list_keys, sleep for: {posix_delay}") + await asyncio.sleep(posix_delay) + return key_names async def releaseClient(self): diff --git a/hsds/util/hdf5dtype.py b/hsds/util/hdf5dtype.py deleted file mode 100644 index 3d7d1d2f..00000000 --- a/hsds/util/hdf5dtype.py +++ /dev/null @@ -1,876 +0,0 @@ -############################################################################## -# Copyright by The HDF Group. # -# All rights reserved. # -# # -# This file is part of HSDS (HDF5 Scalable Data Service), Libraries and # -# Utilities. The full HSDS copyright notice, including # -# terms governing use, modification, and redistribution, is contained in # -# the file COPYING, which can be found at the root of the source code # -# distribution tree. If you do not have access to this file, you may # -# request a copy from help@hdfgroup.org. # -############################################################################## - -import weakref -import numpy as np - - -class Reference: - """ - Represents an HDF5 object reference - """ - - @property - def id(self): - """Low-level identifier appropriate for this object""" - return self._id - - @property - def objref(self): - """Weak reference to object""" - return self._objref # return weak ref to ref'd object - - def __init__(self, bind): - """Create a new reference by binding to - a group/dataset/committed type - """ - self._id = bind._id - self._objref = weakref.ref(bind) - - def __repr__(self): - # TBD: this is not consistent with hsds or h5py... - if not isinstance(self._id.id, str): - raise TypeError("Expected string id") - item = None - - collection_type = self._id.collection_type - item = f"{collection_type}/{self._id.id}" - return item - - def tolist(self): - if type(self._id.id) is not str: - raise TypeError("Expected string id") - if self._id.objtype_code == "d": - return [ - ("datasets/" + self._id.id), - ] - elif self._id.objtype_code == "g": - return [ - ("groups/" + self._id.id), - ] - elif self._id.objtype_code == "t": - return [ - ("datatypes/" + self._id.id), - ] - else: - raise TypeError("Unexpected id type") - - -class RegionReference: - """ - Represents an HDF5 region reference - """ - - @property - def id(self): - """Low-level identifier appropriate for this object""" - return self._id - - @property - def objref(self): - """Weak reference to object""" - return self._objref # return weak ref to ref'd object - - def __init__(self, bind): - """Create a new reference by binding to - a group/dataset/committed type - """ - self._id = bind._id - self._objref = weakref.ref(bind) - - def __repr__(self): - return "" - - -def special_dtype(**kwds): - """Create a new h5py "special" type. Only one keyword may be given. - - Legal keywords are: - - vlen = basetype - Base type for HDF5 variable-length datatype. This can be Python - str type or instance of np.dtype. - Example: special_dtype( vlen=str ) - - enum = (basetype, values_dict) - Create a NumPy representation of an HDF5 enumerated type. Provide - a 2-tuple containing an (integer) base dtype and a dict mapping - string names to integer values. - - ref = Reference | RegionReference - Create a NumPy representation of an HDF5 object or region reference - type.""" - - if len(kwds) != 1: - raise TypeError("Exactly one keyword may be provided") - - name, val = kwds.popitem() - - if name == "vlen": - - return np.dtype("O", metadata={"vlen": val}) - - if name == "enum": - - try: - dt, enum_vals = val - except TypeError: - msg = "Enums must be created from a 2-tuple " - msg += "(basetype, values_dict)" - raise TypeError(msg) - - dt = np.dtype(dt) - if dt.kind not in "iu": - raise TypeError("Only integer types can be used as enums") - - return np.dtype(dt, metadata={"enum": enum_vals}) - - if name == "ref": - dt = None - if val is Reference: - dt = np.dtype("S48", metadata={"ref": Reference}) - elif val is RegionReference: - dt = np.dtype("S48", metadata={"ref": RegionReference}) - else: - raise ValueError("Ref class must be Reference or RegionReference") - - return dt - - raise TypeError(f'Unknown special type "{name}"') - - -def check_dtype(**kwds): - """Check a dtype for h5py special type "hint" information. Only one - keyword may be given. - - vlen = dtype - If the dtype represents an HDF5 vlen, returns the Python base class. - Currently only builting string vlens (str) are supported. Returns - None if the dtype does not represent an HDF5 vlen. - - enum = dtype - If the dtype represents an HDF5 enumerated type, returns the dictionary - mapping string names to integer values. Returns None if the dtype does - not represent an HDF5 enumerated type. - - ref = dtype - If the dtype represents an HDF5 reference type, returns the reference - class (either Reference or RegionReference). Returns None if the dtype - does not represent an HDF5 reference type. - """ - - if len(kwds) != 1: - raise TypeError("Exactly one keyword may be provided") - - name, dt = kwds.popitem() - - if name not in ("vlen", "enum", "ref"): - raise TypeError('Unknown special type "%s"' % name) - - try: - return dt.metadata[name] - except TypeError: - return None - except KeyError: - return None - - -def getTypeResponse(typeItem): - """ - Convert the given type item to a predefined type string for - predefined integer and floating point types ("H5T_STD_I64LE", et. al). - For compound types, recursively iterate through the typeItem and do - same conversion for fields of the compound type.""" - response = None - if "uuid" in typeItem: - # committed type, just return uuid - response = "datatypes/" + typeItem["uuid"] - elif typeItem["class"] in ("H5T_INTEGER", "H5T_FLOAT"): - # just return the class and base for pre-defined types - response = {} - response["class"] = typeItem["class"] - response["base"] = typeItem["base"] - elif typeItem["class"] == "H5T_OPAQUE": - response = {} - response["class"] = "H5T_OPAQUE" - response["size"] = typeItem["size"] - elif typeItem["class"] == "H5T_REFERENCE": - response = {} - response["class"] = "H5T_REFERENCE" - response["base"] = typeItem["base"] - elif typeItem["class"] == "H5T_COMPOUND": - response = {} - response["class"] = "H5T_COMPOUND" - fieldList = [] - for field in typeItem["fields"]: - fieldItem = {} - fieldItem["name"] = field["name"] - fieldItem["type"] = getTypeResponse(field["type"]) # recurse call - fieldList.append(fieldItem) - response["fields"] = fieldList - else: - response = {} # otherwise, return full type - for k in typeItem.keys(): - if k == "base": - if isinstance(typeItem[k], dict): - response[k] = getTypeResponse(typeItem[k]) # recurse call - else: - response[k] = typeItem[k] # predefined type - elif k not in ("size", "base_size"): - response[k] = typeItem[k] - return response - - -def getTypeItem(dt, metadata=None): - """ - Return type info. - For primitive types, return string with typename - For compound types return array of dictionary items - """ - predefined_int_types = { - "int8": "H5T_STD_I8", - "uint8": "H5T_STD_U8", - "int16": "H5T_STD_I16", - "uint16": "H5T_STD_U16", - "int32": "H5T_STD_I32", - "uint32": "H5T_STD_U32", - "int64": "H5T_STD_I64", - "uint64": "H5T_STD_U64", - } - predefined_float_types = { - "float16": "H5T_IEEE_F16", - "float32": "H5T_IEEE_F32", - "float64": "H5T_IEEE_F64", - } - # print(">getTypeItem:", dt.str) - if not metadata and dt.metadata: - metadata = dt.metadata - # if metadata: - # print("> metadata:", metadata) - # if dt.shape: - # print("> shape:", dt.shape) - # if len(dt) > 1: - # print("> len:", len(dt)) - - type_info = {} - if len(dt) > 1: - # compound type - names = dt.names - type_info["class"] = "H5T_COMPOUND" - fields = [] - for name in names: - field = {"name": name} - field["type"] = getTypeItem(dt[name]) - fields.append(field) - type_info["fields"] = fields - elif dt.shape: - # array type - if dt.base == dt: - raise TypeError("Expected base type to be different than parent") - # array type - type_info["dims"] = dt.shape - type_info["class"] = "H5T_ARRAY" - # print("> array type, metadata:", metadata) - type_info["base"] = getTypeItem(dt.base, metadata=metadata) - elif dt.kind == "O": - # vlen string or data - # - # check for h5py variable length extension - - if metadata and "vlen" in metadata: - vlen_check = metadata["vlen"] - if vlen_check is not None and not isinstance(vlen_check, np.dtype): - vlen_check = np.dtype(vlen_check) - - if metadata and "ref" in metadata: - ref_check = metadata["ref"] - else: - ref_check = check_dtype(ref=dt.base) - if vlen_check == bytes: - type_info["class"] = "H5T_STRING" - type_info["length"] = "H5T_VARIABLE" - type_info["charSet"] = "H5T_CSET_ASCII" - type_info["strPad"] = "H5T_STR_NULLTERM" - elif vlen_check == str: - type_info["class"] = "H5T_STRING" - type_info["length"] = "H5T_VARIABLE" - type_info["charSet"] = "H5T_CSET_UTF8" - type_info["strPad"] = "H5T_STR_NULLTERM" - elif isinstance(vlen_check, np.dtype): - # vlen data - type_info["class"] = "H5T_VLEN" - type_info["size"] = "H5T_VARIABLE" - type_info["base"] = getTypeItem(vlen_check) - elif vlen_check is not None: - # unknown vlen type - raise TypeError("Unknown h5py vlen type: " + str(vlen_check)) - elif ref_check is not None: - # a reference type - type_info["class"] = "H5T_REFERENCE" - - if ref_check is Reference: - type_info["base"] = "H5T_STD_REF_OBJ" # objref - elif ref_check is RegionReference: - type_info["base"] = "H5T_STD_REF_DSETREG" # region ref - else: - raise TypeError("unexpected reference type") - else: - raise TypeError("unknown object type") - elif dt.kind == "V": - # void type - type_info["class"] = "H5T_OPAQUE" - type_info["size"] = dt.itemsize - type_info["tag"] = "" # todo - determine tag - elif dt.base.kind == "S": - # check for object reference - ref_check = check_dtype(ref=dt.base) - if ref_check is not None: - # a reference type - type_info["class"] = "H5T_REFERENCE" - - if ref_check is Reference: - type_info["base"] = "H5T_STD_REF_OBJ" # objref - elif ref_check is RegionReference: - type_info["base"] = "H5T_STD_REF_DSETREG" # region ref - else: - raise TypeError("unexpected reference type") - else: - # Fixed length string type - type_info["class"] = "H5T_STRING" - type_info["length"] = dt.itemsize - type_info["charSet"] = "H5T_CSET_ASCII" - type_info["strPad"] = "H5T_STR_NULLPAD" - elif dt.base.kind == "U": - # Fixed length unicode type - ref_check = check_dtype(ref=dt.base) - if ref_check is not None: - raise TypeError("unexpected reference type") - - # Fixed length string type with unicode support - type_info["class"] = "H5T_STRING" - - # this can be problematic if the encoding of the string is not valid, - # or reqires too many bytes. Use variable length strings to handle all - # UTF8 strings correctly - type_info["charSet"] = "H5T_CSET_UTF8" - # convert from UTF32 length to a fixed length - type_info["length"] = dt.itemsize - type_info["strPad"] = "H5T_STR_NULLPAD" - - elif dt.kind == "b": - # boolean type - h5py stores as enum - # assume LE unless the numpy byteorder is '>' - byteorder = "LE" - if dt.base.byteorder == ">": - byteorder = "BE" - # this mapping is an h5py convention for boolean support - mapping = {"FALSE": 0, "TRUE": 1} - type_info["class"] = "H5T_ENUM" - type_info["mapping"] = mapping - base_info = {"class": "H5T_INTEGER"} - base_info["base"] = "H5T_STD_I8" + byteorder - type_info["base"] = base_info - elif dt.kind == "f": - # floating point type - type_info["class"] = "H5T_FLOAT" - byteorder = "LE" - if dt.byteorder == ">": - byteorder = "BE" - if dt.name in predefined_float_types: - # maps to one of the HDF5 predefined types - float_type = predefined_float_types[dt.base.name] - type_info["base"] = float_type + byteorder - else: - raise TypeError("Unexpected floating point type: " + dt.name) - elif dt.kind == "i" or dt.kind == "u": - # integer type - - # assume LE unless the numpy byteorder is '>' - byteorder = "LE" - if dt.base.byteorder == ">": - byteorder = "BE" - - # numpy integer type - but check to see if this is the hypy - # enum extension - if metadata and "enum" in metadata: - # yes, this is an enum! - mapping = metadata["enum"] - type_info["class"] = "H5T_ENUM" - type_info["mapping"] = mapping - if dt.name not in predefined_int_types: - raise TypeError("Unexpected integer type: " + dt.name) - # maps to one of the HDF5 predefined types - base_info = {"class": "H5T_INTEGER"} - base_info["base"] = predefined_int_types[dt.name] + byteorder - type_info["base"] = base_info - else: - type_info["class"] = "H5T_INTEGER" - base_name = dt.name - - if dt.name not in predefined_int_types: - raise TypeError("Unexpected integer type: " + dt.name) - - type_info["base"] = predefined_int_types[base_name] + byteorder - - else: - # unexpected kind - raise TypeError(f"unexpected dtype kind: {dt.kind}") - - return type_info - - -def getItemSize(typeItem): - """ - Get size of an item in bytes. - For variable length types (e.g. variable length strings), - return the string "H5T_VARIABLE" - """ - # handle the case where we are passed a primitive type first - if isinstance(typeItem, str) or isinstance(typeItem, bytes): - for type_prefix in ("H5T_STD_I", "H5T_STD_U", "H5T_IEEE_F"): - if typeItem.startswith(type_prefix): - nlen = len(type_prefix) - num_bits = typeItem[nlen:] - if num_bits[-2:] in ("LE", "BE"): - num_bits = num_bits[:-2] - try: - return int(num_bits) // 8 - except ValueError: - raise TypeError("Invalid Type") - # none of the expect primative types mathched - raise TypeError("Invalid Type") - if not isinstance(typeItem, dict): - raise TypeError("invalid type") - - item_size = 0 - if "class" not in typeItem: - raise KeyError("'class' not provided") - typeClass = typeItem["class"] - - if typeClass == "H5T_INTEGER": - if "base" not in typeItem: - raise KeyError("'base' not provided") - item_size = getItemSize(typeItem["base"]) - - elif typeClass == "H5T_FLOAT": - if "base" not in typeItem: - raise KeyError("'base' not provided") - item_size = getItemSize(typeItem["base"]) - - elif typeClass == "H5T_STRING": - if "length" not in typeItem: - raise KeyError("'length' not provided") - item_size = typeItem["length"] - - elif typeClass == "H5T_VLEN": - item_size = "H5T_VARIABLE" - elif typeClass == "H5T_OPAQUE": - if "size" not in typeItem: - raise KeyError("'size' not provided") - item_size = int(typeItem["size"]) - - elif typeClass == "H5T_ARRAY": - if "dims" not in typeItem: - raise KeyError("'dims' must be provided for array types") - if "base" not in typeItem: - raise KeyError("'base' not provided") - item_size = getItemSize(typeItem["base"]) - - elif typeClass == "H5T_ENUM": - if "base" not in typeItem: - raise KeyError("'base' must be provided for enum types") - item_size = getItemSize(typeItem["base"]) - - elif typeClass == "H5T_REFERENCE": - if "length" in typeItem: - item_size = typeItem["length"] - elif "base" in typeItem and typeItem["base"] == "H5T_STD_REF_OBJ": - # obj ref values are in the form: "groups/" or - # "datasets/" or "datatypes/" - item_size = 48 - else: - raise KeyError("Unable to determine item size for reference type") - elif typeClass == "H5T_COMPOUND": - if "fields" not in typeItem: - raise KeyError("'fields' not provided for compound type") - fields = typeItem["fields"] - if not isinstance(fields, list): - raise TypeError("Type Error: expected list type for 'fields'") - if not fields: - raise KeyError("no 'field' elements provided") - # add up the size of each sub-field - for field in fields: - if not isinstance(field, dict): - raise TypeError("Expected dictionary type for field") - if "type" not in field: - raise KeyError("'type' missing from field") - subtype_size = getItemSize(field["type"]) # recursive call - if subtype_size == "H5T_VARIABLE": - item_size = "H5T_VARIABLE" - break # don't need to look at the rest - - item_size += subtype_size - else: - raise TypeError("Invalid type class") - - # calculate array type - if "dims" in typeItem and isinstance(item_size, int): - dims = typeItem["dims"] - for dim in dims: - item_size *= dim - - return item_size - - -def getDtypeItemSize(dtype): - """ Return size of dtype in bytes - For variable length types (e.g. variable length strings), - return the string "H5T_VARIABLE - """ - item_size = 0 - if len(dtype) > 0: - # compound dtype - for i in range(len(dtype)): - sub_dt = dtype[i] - sub_dt_size = getDtypeItemSize(sub_dt) - if sub_dt_size == "H5T_VARIABLE": - item_size = "H5T_VARIABLE" # return variable if any component is variable - break - item_size += sub_dt_size - else: - # primitive type - if dtype.metadata and "vlen" in dtype.metadata: - item_size = "H5T_VARIABLE" - else: - item_size = dtype.itemsize - return item_size - - -def getNumpyTypename(hdf5TypeName, typeClass=None): - predefined_int_types = { - "H5T_STD_I8": "i1", - "H5T_STD_U8": "u1", - "H5T_STD_I16": "i2", - "H5T_STD_U16": "u2", - "H5T_STD_I32": "i4", - "H5T_STD_U32": "u4", - "H5T_STD_I64": "i8", - "H5T_STD_U64": "u8", - } - predefined_float_types = { - "H5T_IEEE_F16": "f2", - "H5T_IEEE_F32": "f4", - "H5T_IEEE_F64": "f8", - } - - if len(hdf5TypeName) < 3: - raise Exception("Type Error: invalid typename: ") - endian = "<" # default endian - key = hdf5TypeName - if hdf5TypeName.endswith("LE"): - key = hdf5TypeName[:-2] - elif hdf5TypeName.endswith("BE"): - key = hdf5TypeName[:-2] - endian = ">" - - if key in predefined_int_types and ( - typeClass is None or typeClass == "H5T_INTEGER" - ): - return endian + predefined_int_types[key] - if key in predefined_float_types and ( - typeClass is None or typeClass == "H5T_FLOAT" - ): - return endian + predefined_float_types[key] - raise TypeError("Type Error: invalid type") - - -def createBaseDataType(typeItem): - dtRet = None - if isinstance(typeItem, str): - # should be one of the predefined types - dtName = getNumpyTypename(typeItem) - dtRet = np.dtype(dtName) - return dtRet # return predefined type - - if not isinstance(typeItem, dict): - raise TypeError("Type Error: invalid type") - - if "class" not in typeItem: - raise KeyError("'class' not provided") - typeClass = typeItem["class"] - - dims = "" - if "dims" in typeItem: - if typeClass != "H5T_ARRAY": - raise TypeError("'dims' only supported for integer types") - - dims = None - if isinstance(typeItem["dims"], int): - dims = typeItem["dims"] # make into a tuple - elif not isinstance(typeItem["dims"], list) and not isinstance( - typeItem["dims"], tuple - ): - raise TypeError("expected list or integer for dims") - else: - dims = typeItem["dims"] - dims = str(tuple(dims)) - - if typeClass == "H5T_INTEGER": - if "base" not in typeItem: - raise KeyError("'base' not provided") - baseType = getNumpyTypename(typeItem["base"], typeClass="H5T_INTEGER") - dtRet = np.dtype(dims + baseType) - elif typeClass == "H5T_FLOAT": - if "base" not in typeItem: - raise KeyError("'base' not provided") - baseType = getNumpyTypename(typeItem["base"], typeClass="H5T_FLOAT") - dtRet = np.dtype(dims + baseType) - elif typeClass == "H5T_STRING": - if "length" not in typeItem: - raise KeyError("'length' not provided") - if "charSet" not in typeItem: - raise KeyError("'charSet' not provided") - - if typeItem["length"] == "H5T_VARIABLE": - if dims: - msg = "ArrayType is not supported for variable len types" - raise TypeError(msg) - if typeItem["charSet"] == "H5T_CSET_ASCII": - dtRet = special_dtype(vlen=bytes) - elif typeItem["charSet"] == "H5T_CSET_UTF8": - dtRet = special_dtype(vlen=str) - else: - raise TypeError("unexpected 'charSet' value") - else: - nStrSize = typeItem["length"] - if not isinstance(nStrSize, int): - raise TypeError("expecting integer value for 'length'") - type_code = None - if typeItem["charSet"] == "H5T_CSET_ASCII": - type_code = "S" - elif typeItem["charSet"] == "H5T_CSET_UTF8": - # use the same type_code as ascii strings - # (othewise, numpy will reserve bytes for UTF32 representation) - type_code = "S" - else: - raise TypeError("unexpected 'charSet' value") - # a fixed size string - dtRet = np.dtype(dims + type_code + str(nStrSize)) - elif typeClass == "H5T_VLEN": - if dims: - msg = "ArrayType is not supported for variable len types" - raise TypeError(msg) - if "base" not in typeItem: - raise KeyError("'base' not provided") - baseType = createBaseDataType(typeItem["base"]) - dtRet = special_dtype(vlen=np.dtype(baseType)) - elif typeClass == "H5T_OPAQUE": - if dims: - msg = "Opaque Type is not supported for variable len types" - raise TypeError(msg) - if "size" not in typeItem: - raise KeyError("'size' not provided") - nSize = int(typeItem["size"]) - if nSize <= 0: - raise TypeError("'size' must be non-negative") - dtRet = np.dtype("V" + str(nSize)) - elif typeClass == "H5T_ARRAY": - if not dims: - raise KeyError("'dims' must be provided for array types") - if "base" not in typeItem: - raise KeyError("'base' not provided") - arrayBaseType = typeItem["base"] - if isinstance(arrayBaseType, dict): - if "class" not in arrayBaseType: - raise KeyError("'class' not provided for array base type") - type_classes = ("H5T_INTEGER", "H5T_FLOAT", "H5T_STRING", "H5T_ARRAY") - if arrayBaseType["class"] not in type_classes: - msg = "Array Type base type must be integer, float, string, or array" - raise TypeError(msg) - baseType = createDataType(arrayBaseType) - metadata = None - if baseType.metadata: - metadata = dict(baseType.metadata) - dtRet = np.dtype(dims + baseType.str, metadata=metadata) - else: - dtRet = np.dtype(dims + baseType.str) - return dtRet # return predefined type - elif typeClass == "H5T_REFERENCE": - if "base" not in typeItem: - raise KeyError("'base' not provided") - if typeItem["base"] == "H5T_STD_REF_OBJ": - dtRet = special_dtype(ref=Reference) - elif typeItem["base"] == "H5T_STD_REF_DSETREG": - dtRet = special_dtype(ref=RegionReference) - else: - raise TypeError("Invalid base type for reference type") - - elif typeClass == "H5T_ENUM": - if "base" not in typeItem: - raise KeyError("Expected 'base' to be provided for enum type") - base_json = typeItem["base"] - if "class" not in base_json: - raise KeyError("Expected class field in base type") - if base_json["class"] != "H5T_INTEGER": - msg = "Only integer base types can be used with enum type" - raise TypeError(msg) - if "mapping" not in typeItem: - raise KeyError("'mapping' not provided for enum type") - mapping = typeItem["mapping"] - if len(mapping) == 0: - raise KeyError("empty enum map") - - dt = createBaseDataType(base_json) - if all( - ( - dt.kind == "i", - dt.name == "int8", - len(mapping) == 2, - "TRUE" in mapping, - "FALSE" in mapping, - ) - ): - # convert to numpy boolean type - dtRet = np.dtype("bool") - else: - # not a boolean enum, use h5py special dtype - dtRet = special_dtype(enum=(dt, mapping)) - - else: - raise TypeError("Invalid type class") - - return dtRet - - -def createDataType(typeItem): - """ - Create a numpy datatype given a json type - """ - dtRet = None - if type(typeItem) in (str, bytes): - # should be one of the predefined types - dtName = getNumpyTypename(typeItem) - dtRet = np.dtype(dtName) - return dtRet # return predefined type - - if not isinstance(typeItem, dict): - raise TypeError("invalid type") - - if "class" not in typeItem: - raise KeyError("'class' not provided") - typeClass = typeItem["class"] - - if typeClass == "H5T_COMPOUND": - if "fields" not in typeItem: - raise KeyError("'fields' not provided for compound type") - fields = typeItem["fields"] - if type(fields) is not list: - raise TypeError("Type Error: expected list type for 'fields'") - if not fields: - raise KeyError("no 'field' elements provided") - subtypes = [] - for field in fields: - - if not isinstance(field, dict): - raise TypeError("Expected dictionary type for field") - if "name" not in field: - raise KeyError("'name' missing from field") - if "type" not in field: - raise KeyError("'type' missing from field") - field_name = field["name"] - if not isinstance(field_name, str): - raise TypeError("field names must be strings") - # verify the field name is ascii - try: - field_name.encode("ascii") - except UnicodeEncodeError: - raise TypeError("non-ascii field name not allowed") - - dt = createDataType(field["type"]) # recursive call - if dt is None: - raise Exception("unexpected error") - subtypes.append((field["name"], dt)) # append tuple - - dtRet = np.dtype(subtypes) - else: - dtRet = createBaseDataType(typeItem) # create non-compound dt - return dtRet - - -def validateTypeItem(typeItem): - """ - Validate a json type - call createDataType and if no exception, - it's valid - """ - createDataType(typeItem) - # throws KeyError, TypeError, or ValueError - - -def getBaseTypeJson(type_name): - """ - Return JSON representation of a predefined type string - """ - predefined_int_types = ( - "H5T_STD_I8", - "H5T_STD_U8", - "H5T_STD_I16", - "H5T_STD_U16", - "H5T_STD_I32", - "H5T_STD_U32", - "H5T_STD_I64", - "H5T_STD_U64", - ) - predefined_float_types = ("H5T_IEEE_F16", "H5T_IEEE_F32", "H5T_IEEE_F64") - type_json = {} - # predefined typenames start with 'H5T' and end with "LE" or "BE" - if all( - ( - type_name.startswith("H5T_"), - type_name[-1] == "E", - type_name[-2] in ("L", "B"), - ) - ): - # trime of the "BE/"LE" - type_prefix = type_name[:-2] - if type_prefix in predefined_int_types: - type_json["class"] = "H5T_INTEGER" - type_json["base"] = type_name - elif type_prefix in predefined_float_types: - type_json["class"] = "H5T_FLOAT" - type_json["base"] = type_name - else: - raise TypeError("Invalid type name") - else: - raise TypeError("Invalid type name") - return type_json - - -def getSubType(dt_parent, fields): - """ Return a dtype that is a compound type composed of - the fields given in the field_names list - """ - if len(dt_parent) == 0: - raise TypeError("getSubType - parent must be compound type") - if not fields: - raise TypeError("null field specification") - if isinstance(fields, str): - fields = [fields,] # convert to a list - - field_names = set(dt_parent.names) - dt_items = [] - for field in fields: - if field not in field_names: - raise TypeError(f"field: {field} is not defined in parent type") - dt_items.append((field, dt_parent[field])) - dt = np.dtype(dt_items) - - return dt diff --git a/hsds/util/httpUtil.py b/hsds/util/httpUtil.py index 0d43ae4a..a101b760 100644 --- a/hsds/util/httpUtil.py +++ b/hsds/util/httpUtil.py @@ -25,7 +25,8 @@ from aiohttp.web_exceptions import HTTPRequestEntityTooLarge from aiohttp.web_exceptions import HTTPServiceUnavailable, HTTPBadRequest from aiohttp.client_exceptions import ClientError -from hsds.util.idUtil import isValidUuid + +from h5json.objid import isValidUuid from .. import hsds_logger as log from .. import config @@ -693,17 +694,6 @@ def getAcceptType(request): return accept_type -def isAWSLambda(request): - """ - Return true if this is a lambda request - """ - is_lambda = False - if "User-Agent" in request.headers: - if request.headers["User-Agent"] == "AWSLambda": - is_lambda = True - return is_lambda - - def getContentType(request): """ Get the content type from request headers. diff --git a/hsds/util/idUtil.py b/hsds/util/idUtil.py deleted file mode 100644 index fe21bbb0..00000000 --- a/hsds/util/idUtil.py +++ /dev/null @@ -1,540 +0,0 @@ -############################################################################## -# Copyright by The HDF Group. # -# All rights reserved. # -# # -# This file is part of HSDS (HDF5 Scalable Data Service), Libraries and # -# Utilities. The full HSDS copyright notice, including # -# terms governing use, modification, and redistribution, is contained in # -# the file COPYING, which can be found at the root of the source code # -# distribution tree. If you do not have access to this file, you may # -# request a copy from help@hdfgroup.org. # -############################################################################## -# -# idUtil: -# id (uuid) related functions -# - -import os.path -import hashlib -import uuid -from aiohttp.web_exceptions import HTTPServiceUnavailable -from .. import hsds_logger as log - - -S3_URI = "s3://" -FILE_URI = "file://" -AZURE_URI = "blob.core.windows.net/" # preceded with "https://" - - -def _getStorageProtocol(uri): - """ returns 's3://', 'file://', or 'https://...net/' prefix if present. - If the prefix is in the form: https://myaccount.blob.core.windows.net/mycontainer - (references Azure blob storage), return: https://myaccount.blob.core.windows.net/ - otherwise None """ - - if not uri: - protocol = None - elif uri.startswith(S3_URI): - protocol = S3_URI - elif uri.startswith(FILE_URI): - protocol = FILE_URI - elif uri.startswith("https://") and uri.find(AZURE_URI) > 0: - n = uri.find(AZURE_URI) + len(AZURE_URI) - protocol = uri[:n] - elif uri.find("://") >= 0: - raise ValueError(f"storage uri: {uri} not supported") - else: - protocol = None - return protocol - - -def _getBaseName(uri): - """ Return the part of the URI after the storage protocol (if any) """ - - protocol = _getStorageProtocol(uri) - if not protocol: - return uri - else: - return uri[len(protocol):] - - -def getIdHash(id): - """Return md5 prefix based on id value""" - m = hashlib.new("md5") - m.update(id.encode("utf8")) - hexdigest = m.hexdigest() - return hexdigest[:5] - - -def isSchema2Id(id): - """return true if this is a v2 id""" - # v1 ids are in the standard UUID format: 8-4-4-4-12 - # v2 ids are in the non-standard: 8-8-4-6-6 - parts = id.split("-") - if len(parts) != 6: - raise ValueError(f"Unexpected id formation for uuid: {id}") - if len(parts[2]) == 8: - return True - else: - return False - - -def getIdHexChars(id): - """get the hex chars of the given id""" - if id[0] == "c": - # don't include chunk index - index = id.index("_") - parts = id[0:index].split("-") - else: - parts = id.split("-") - if len(parts) != 6: - raise ValueError(f"Unexpected id format for uuid: {id}") - return "".join(parts[1:]) - - -def hexRot(ch): - """rotate hex character by 8""" - return format((int(ch, base=16) + 8) % 16, "x") - - -def isRootObjId(id): - """returns true if this is a root id (only for v2 schema)""" - if not isSchema2Id(id): - raise ValueError("isRootObjId can only be used with v2 ids") - validateUuid(id) # will throw ValueError exception if not a objid - if id[0] != "g": - return False # not a group - token = getIdHexChars(id) - # root ids will have last 16 chars rotated version of the first 16 - is_root = True - for i in range(16): - if token[i] != hexRot(token[i + 16]): - is_root = False - break - return is_root - - -def getRootObjId(id): - """returns root id for this objid if this is a root id - (only for v2 schema) - """ - if isRootObjId(id): - return id # this is the root id - token = list(getIdHexChars(id)) - # root ids will have last 16 chars rotated version of the first 16 - for i in range(16): - token[i + 16] = hexRot(token[i]) - token = "".join(token) - root_id = "g-" + token[0:8] + "-" + token[8:16] + "-" + token[16:20] - root_id += "-" + token[20:26] + "-" + token[26:32] - - return root_id - - -def createObjId(obj_type, rootid=None): - if obj_type not in ("groups", "datasets", "datatypes", "chunks", "roots"): - raise ValueError("unexpected obj_type") - - prefix = None - if obj_type == "datatypes": - prefix = "t" # don't collide with datasets - elif obj_type == "roots": - prefix = "g" # root obj is a group - else: - prefix = obj_type[0] - if not rootid and obj_type != "roots": - # v1 schema - folder - objid = prefix + "-" + str(uuid.uuid1()) - elif rootid and not isSchema2Id(rootid): - # v1 schema - domain - objid = prefix + "-" + str(uuid.uuid1()) - else: - # schema v2 - salt = uuid.uuid4().hex - # take a hash to randomize the uuid - token = list(hashlib.sha256(salt.encode()).hexdigest()) - - if rootid: - # replace first 16 chars of token with first 16 chars of root id - root_hex = getIdHexChars(rootid) - token[0:16] = root_hex[0:16] - else: - # obj_type == "roots" - # use only 16 chars, but make it look a 32 char id - for i in range(16): - token[16 + i] = hexRot(token[i]) - # format as a string - token = "".join(token) - objid = prefix + "-" + token[0:8] + "-" + token[8:16] + "-" - objid += token[16:20] + "-" + token[20:26] + "-" + token[26:32] - - return objid - - -def getS3Key(id): - """Return s3 key for given id. - - For schema v1: - A md5 prefix is added to the front of the returned key to better - distribute S3 objects. - For schema v2: - The id is converted to the pattern: "db/{rootid[0:16]}" for rootids and - "db/id[0:16]/{prefix}/id[16-32]" for other ids - Chunk ids have the chunk index added after the slash: - "db/id[0:16]/d/id[16:32]/x_y_z - - For domain id's: - Return a key with the .domain suffix and no preceding slash. - For non-default buckets, use the format: /s3_key - If the id has a storage specifier ("s3://", "file://", etc.) - include that along with the bucket name. e.g.: "s3://mybucket/a_folder/a_file.h5" - """ - - base_id = _getBaseName(id) # strip any s3://, etc. - if base_id.find("/") > 0: - # a domain id - domain_suffix = ".domain.json" - index = base_id.find("/") + 1 - key = base_id[index:] - if not key.endswith(domain_suffix): - if key[-1] != "/": - key += "/" - key += domain_suffix - else: - if isSchema2Id(id): - # schema v2 id - hexid = getIdHexChars(id) - prefix = id[0] # one of g, d, t, c - if prefix not in ("g", "d", "t", "c"): - raise ValueError(f"Unexpected id: {id}") - - if isRootObjId(id): - key = f"db/{hexid[0:8]}-{hexid[8:16]}" - else: - partition = "" - if prefix == "c": - # use 'g' so that chunks will show up under their dataset - s3col = "d" - n = id.find("-") - if n > 1: - # extract the partition index if present - partition = "p" + id[1:n] - else: - s3col = prefix - key = f"db/{hexid[0:8]}-{hexid[8:16]}/{s3col}/{hexid[16:20]}" - key += f"-{hexid[20:26]}-{hexid[26:32]}" - if prefix == "c": - if partition: - key += "/" - key += partition - # add the chunk coordinate - index = id.index("_") # will raise ValueError if not found - n = index + 1 - coord = id[n:] - key += "/" - key += coord - elif prefix == "g": - # add key suffix for group - key += "/.group.json" - elif prefix == "d": - # add key suffix for dataset - key += "/.dataset.json" - else: - # add key suffix for datatype - key += "/.datatype.json" - else: - # v1 id - # schema v1 id - idhash = getIdHash(id) - key = f"{idhash}-{id}" - - return key - - -def getObjId(s3key): - """Return object id given valid s3key""" - if all( - ( - len(s3key) >= 44 and s3key[0:5].isalnum(), - len(s3key) >= 44 and s3key[5] == "-", - len(s3key) >= 44 and s3key[6] in ("g", "d", "c", "t"), - ) - ): - # v1 obj keys - objid = s3key[6:] - elif s3key.endswith("/.domain.json"): - objid = "/" + s3key[: -(len("/.domain.json"))] - elif s3key.startswith("db/"): - # schema v2 object key - parts = s3key.split("/") - chunk_coord = "" # used only for chunk ids - partition = "" # likewise - token = [] - for ch in parts[1]: - if ch != "-": - token.append(ch) - - if len(parts) == 3: - # root id - # last part should be ".group.json" - if parts[2] != ".group.json": - raise ValueError(f"unexpected S3Key: {s3key}") - # add 16 more chars using rotated version of first 16 - for i in range(16): - token.append(hexRot(token[i])) - prefix = "g" - elif len(parts) == 5: - # group, dataset, or datatype or chunk - for ch in parts[3]: - if ch != "-": - token.append(ch) - - if parts[2] == "g" and parts[4] == ".group.json": - prefix = "g" # group json - elif parts[2] == "t" and parts[4] == ".datatype.json": - prefix = "t" # datatype json - elif parts[2] == "d": - if parts[4] == ".dataset.json": - prefix = "d" # dataset json - else: - # chunk object - prefix = "c" - chunk_coord = "_" + parts[4] - else: - raise ValueError(f"unexpected S3Key: {s3key}") - elif len(parts) == 6: - # chunk key with partitioning - for ch in parts[3]: - if ch != "-": - token.append(ch) - if parts[2][0] != "d": - raise ValueError(f"unexpected S3Key: {s3key}") - prefix = "c" - partition = parts[4] - if partition[0] != "p": - raise ValueError(f"unexpected S3Key: {s3key}") - partition = partition[1:] # strip off the p - chunk_coord = "_" + parts[5] - else: - raise ValueError(f"unexpected S3Key: {s3key}") - - token = "".join(token) - objid = prefix + partition + "-" + token[0:8] + "-" + token[8:16] - objid += "-" + token[16:20] + "-" + token[20:26] + "-" - objid += token[26:32] + chunk_coord - else: - msg = f"unexpected S3Key: {s3key}" - log.warn(msg) - raise ValueError(msg) - return objid - - -def isS3ObjKey(s3key): - valid = False - try: - objid = getObjId(s3key) - if objid: - valid = True - except KeyError: - pass # ignore - except ValueError: - pass # ignore - return valid - - -def createNodeId(prefix, node_number=None): - """Create a random id used to identify nodes""" - node_id = "" # nothing too bad happens if this doesn't get set - if node_number is not None: - # just make an id based on the node_number - hash_key = f"{node_number + 1:03d}" - else: - # use the container id if we are running inside docker - hash_key = getIdHash(str(uuid.uuid1())) - proc_file = "/proc/self/cgroup" - if os.path.isfile(proc_file): - with open(proc_file) as f: - first_line = f.readline() - if first_line: - fields = first_line.split(":") - if len(fields) >= 3: - field = fields[2] - if field.startswith("/docker/"): - docker_len = len("/docker/") - - if len(field) > docker_len + 12: - n = docker_len - m = n + 12 - node_id = field[n:m] - - if node_id: - key = f"{prefix}-{node_id}-{hash_key}" - else: - key = f"{prefix}-{hash_key}" - return key - - -def getCollectionForId(obj_id): - """return groups/datasets/datatypes based on id""" - if not isinstance(obj_id, str): - raise ValueError("invalid object id") - collection = None - if obj_id.startswith("g-"): - collection = "groups" - elif obj_id.startswith("d-"): - collection = "datasets" - elif obj_id.startswith("t-"): - collection = "datatypes" - else: - raise ValueError("not a collection id") - return collection - - -def validateUuid(id, obj_class=None): - if not isinstance(id, str): - raise ValueError("Expected string type") - if len(id) < 38: - # id should be prefix (e.g. "g-") and uuid value - raise ValueError("Unexpected id length") - if id[0] not in ("g", "d", "t", "c"): - raise ValueError("Unexpected prefix") - if id[0] != "c" and id[1] != "-": - # chunk ids may have a partition index following the c - raise ValueError("Unexpected prefix") - if obj_class is not None: - obj_class = obj_class.lower() - prefix = obj_class[0] - if obj_class.startswith("datatype"): - prefix = "t" - if id[0] != prefix: - raise ValueError(f"Unexpected prefix for class: {obj_class}") - if id[0] == "c": - # trim the type char and any partition id - n = id.find("-") - if n == -1: - raise ValueError("Invalid chunk id") - - # trim the chunk index for chunk ids - m = id.find("_") - if m == -1: - raise ValueError("Invalid chunk id") - n += 1 - id = "c-" + id[n:m] - if len(id) != 38: - # id should be 36 now - raise ValueError("Unexpected id length") - - for ch in id: - if ch.isalnum(): - continue - if ch == "-": - continue - raise ValueError(f"Unexpected character in uuid: {ch}") - - -def isValidUuid(id, obj_class=None): - try: - validateUuid(id, obj_class) - return True - except ValueError: - return False - - -def isValidChunkId(id): - if not isValidUuid(id): - return False - if id[0] != "c": - return False - return True - - -def getClassForObjId(id): - """return domains/chunks/groups/datasets/datatypes based on id""" - if not isinstance(id, str): - raise ValueError("Expected string type") - if len(id) == 0: - raise ValueError("Empty string") - if id[0] == "/": - return "domains" - if isValidChunkId(id): - return "chunks" - else: - return getCollectionForId(id) - - -def isObjId(id): - """return true if uuid or domain""" - if not isinstance(id, str) or len(id) == 0: - return False - if id.find("/") > 0: - # domain id is any string in the form / - return True - return isValidUuid(id) - - -def getUuidFromId(id): - """strip off the type prefix ('g-' or 'd-', or 't-') - and return the uuid part""" - return id[2:] - - -def getObjPartition(id, count): - """Get the id of the dn node that should be handling the given obj id""" - hash_code = getIdHash(id) - hash_value = int(hash_code, 16) - number = hash_value % count - return number - - -def getNodeNumber(app): - if app["node_type"] == "sn": - log.error("node number if only for DN nodes") - raise ValueError() - - dn_ids = app["dn_ids"] - log.debug(f"getNodeNumber(from dn_ids: {dn_ids})") - for i in range(len(dn_ids)): - dn_id = dn_ids[i] - if dn_id == app["id"]: - log.debug(f"returning nodeNumber: {i}") - return i - log.error("getNodeNumber, no matching id") - return -1 - - -def getNodeCount(app): - dn_urls = app["dn_urls"] - log.debug(f"getNodeCount for dn_urls: {dn_urls}") - dn_node_count = len(dn_urls) - return dn_node_count - - -def validateInPartition(app, obj_id): - node_number = getNodeNumber(app) - node_count = getNodeCount(app) - msg = f"obj_id: {obj_id}, node_count: {node_count}, " - msg += f"node_number: {node_number}" - log.debug(msg) - partition_number = getObjPartition(obj_id, node_count) - if partition_number != node_number: - # The request shouldn't have come to this node' - msg = f"wrong node for 'id':{obj_id}, expected node {node_number} " - msg += f"got {partition_number}" - log.error(msg) - raise KeyError(msg) - - -def getDataNodeUrl(app, obj_id): - """Return host/port for datanode for given obj_id. - Throw exception if service is not ready""" - dn_urls = app["dn_urls"] - dn_node_count = getNodeCount(app) - node_state = app["node_state"] - if node_state != "READY" or dn_node_count <= 0: - msg = "Service not ready" - log.warn(msg) - raise HTTPServiceUnavailable() - dn_number = getObjPartition(obj_id, dn_node_count) - url = dn_urls[dn_number] - log.debug(f"got dn_url: {url} for obj_id: {obj_id}") - return url diff --git a/hsds/util/jwtUtil.py b/hsds/util/jwtUtil.py index 38a960c7..3c5ee737 100644 --- a/hsds/util/jwtUtil.py +++ b/hsds/util/jwtUtil.py @@ -198,7 +198,7 @@ def verifyBearerToken(app, token): log.debug(f"claim: {name} not found in bearer token") if not username: - log.warn("unable to retreive username from bearer token") + log.warn("unable to retrieve username from bearer token") raise HTTPUnauthorized() exp = None diff --git a/hsds/util/k8sClient.py b/hsds/util/k8sClient.py index b3e65c51..51b78f4c 100644 --- a/hsds/util/k8sClient.py +++ b/hsds/util/k8sClient.py @@ -78,21 +78,15 @@ def getIPKeys(metadata): if not isinstance(managedFields, list): log.warn(f"expected managedFields to be list but got: {type(managedFields)}") return pod_ips - # log.debug(f"mangagedFields - {len(managedFields)} items") for item in managedFields: if not isinstance(item, dict): log.warn(f"ignoring item type {type(item)}: {item}") continue for key in KEY_PATH: - # log.debug(f"using key: {key}") if key not in item: # key not found, move on to next managedField - # msg = f"getIPKeys - looking for {key} key but not present" - # log.debug(msg) break item = item[key] - # log.debug(f"got obj type: {type(item)}") - # log.debug(f"item: {item}") if not isinstance(item, dict): log.warn("not a dict") break diff --git a/hsds/util/linkUtil.py b/hsds/util/linkUtil.py index 3469a8a1..65939e7d 100644 --- a/hsds/util/linkUtil.py +++ b/hsds/util/linkUtil.py @@ -13,122 +13,71 @@ # linkdUtil: # link related functions # +from h5json.time_util import getNow +from h5json.link_util import validateLinkName, getLinkClass, getLinkPath, getLinkFilePath from .. import hsds_logger as log -def validateLinkName(name): - """ verify the link name is valid """ - if not isinstance(name, str): - msg = "Unexpected type for link name" - log.warn(msg) - raise ValueError(msg) - if name.find("/") >= 0: - msg = "link name contains slash" +def getRequestLink(title, link_json, predate_max_time=0.0): + """ return normalized link from request json + Throw value error if badly formatted """ + + if not isinstance(link_json, dict): + msg = f"expected dict for for links, but got: {type(link_json)}" log.warn(msg) raise ValueError(msg) + log.debug(f"getRequestLink title: {title} link_json: {link_json}") + link_item = {} # normalized link item to return -def getLinkClass(link_json): - """ verify this is a valid link - returns the link class """ - if "class" in link_json: - link_class = link_json["class"] - else: - link_class = None - if "h5path" in link_json and "id" in link_json: - msg = "link tgt_id and h5path both set" - log.warn(msg) - raise ValueError(msg) - if "id" in link_json: - tgt_id = link_json["id"] - if not isinstance(tgt_id, str) or len(tgt_id) < 38: - msg = f"link with invalid id: {tgt_id}" - log.warn(msg) - raise ValueError(msg) - if tgt_id[:2] not in ("g-", "t-", "d-"): - msg = "link tgt must be group, datatype or dataset uuid" + now = getNow() + + validateLinkName(title) # will raise ValueError is invalid + + link_class = getLinkClass(link_json) + + link_item = {"class": link_class} + + if link_class == "H5L_TYPE_HARD": + if "id" not in link_json: + msg = "expected id key for hard link" log.warn(msg) - raise ValueError(msg) - if link_class: - if link_class != "H5L_TYPE_HARD": - msg = f"expected link class to be H5L_TYPE_HARD but got: {link_class}" - log.warn(msg) - raise ValueError(msg) - else: - link_class = "H5L_TYPE_HARD" - elif "h5path" in link_json: - h5path = link_json["h5path"] - log.debug(f"link path: {h5path}") - if "h5domain" in link_json: - if link_class: - if link_class != "H5L_TYPE_EXTERNAL": - msg = f"expected link class to be H5L_TYPE_EXTERNAL but got: {link_class}" - log.warn(msg) - raise ValueError(msg) - else: - link_class = "H5L_TYPE_EXTERNAL" - else: - if link_class: - if link_class != "H5L_TYPE_SOFT": - msg = f"expected link class to be H5L_TYPE_SOFT but got: {link_class}" - log.warn(msg) - raise ValueError(msg) - else: - link_class = "H5L_TYPE_SOFT" + raise ValueError + link_item["id"] = link_json["id"] else: - msg = "link with no id or h5path" + if link_class in ("H5L_TYPE_SOFT", "H5L_TYPE_EXTERNAL"): + link_item["h5path"] = getLinkPath(link_json) + + if link_class == "H5L_TYPE_EXTERNAL": + link_item["file"] = getLinkFilePath(link_json) + + if "created" in link_json: + created = link_json["created"] + # allow "pre-dated" attributes if recent enough + if now - created < predate_max_time: + link_item["created"] = created + else: + log.warn("stale created timestamp for link, ignoring") + if "created" not in link_item: + link_item["created"] = now + + return link_item + + +def getRequestLinks(links_json, predate_max_time=0.0): + """ return list of normalized links from request json + Throw value error if any is badly formatted """ + + if not isinstance(links_json, dict): + msg = f"POST_Groups expected dict for for links, but got: {type(links_json)}" log.warn(msg) raise ValueError(msg) - return link_class + links = {} # normalized link items to return + kwargs = {"predate_max_time": predate_max_time} + for title in links_json: + links[title] = getRequestLink(title, links_json[title], **kwargs) -def isEqualLink(link1, link2): - """ Return True if the two links are the same """ - - for obj in (link1, link2): - if not isinstance(obj, dict): - raise TypeError(f"unexpected type: {type(obj)}") - if "class" not in obj: - raise TypeError("expected class key for link") - if link1["class"] != link2["class"]: - return False # different link types - link_class = link1["class"] - if link_class == "H5L_TYPE_HARD": - for obj in (link1, link2): - if "id" not in obj: - raise TypeError(f"expected id key for link: {obj}") - if link1["id"] != link2["id"]: - return False - elif link_class == "H5L_TYPE_SOFT": - for obj in (link1, link2): - if "h5path" not in obj: - raise TypeError(f"expected h5path key for link: {obj}") - if link1["h5path"] != link2["h5path"]: - return False - elif link_class == "H5L_TYPE_EXTERNAL": - for obj in (link1, link2): - for k in ("h5path", "h5domain"): - if k not in obj: - raise TypeError(f"expected {k} key for link: {obj}") - if link1["h5path"] != link2["h5path"]: - return False - if link1["h5domain"] != link2["h5domain"]: - return False - else: - raise TypeError(f"unexpected link class: {link_class}") - return True - - -def h5Join(path, paths): - h5path = path - if not paths: - return h5path - if isinstance(paths, str): - paths = (paths,) - for s in paths: - if h5path[-1] != "/": - h5path += "/" - h5path += s - return h5path + return links diff --git a/hsds/util/nodeUtil.py b/hsds/util/nodeUtil.py new file mode 100644 index 00000000..8f67f400 --- /dev/null +++ b/hsds/util/nodeUtil.py @@ -0,0 +1,124 @@ +############################################################################## +# Copyright by The HDF Group. # +# All rights reserved. # +# # +# This file is part of HSDS (HDF5 Scalable Data Service), Libraries and # +# Utilities. The full HSDS copyright notice, including # +# terms governing use, modification, and redistribution, is contained in # +# the file COPYING, which can be found at the root of the source code # +# distribution tree. If you do not have access to this file, you may # +# request a copy from help@hdfgroup.org. # +############################################################################## +# +# nodeUtil: +# node (SN/DN mapping) related functions +# +import hashlib +import os.path +import uuid + +from aiohttp.web_exceptions import HTTPServiceUnavailable + +from .. import hsds_logger as log + + +def _getIdHash(id): + """Return md5 prefix based on id value""" + m = hashlib.new("md5") + m.update(id.encode("utf8")) + hexdigest = m.hexdigest() + return hexdigest[:5] + + +def createNodeId(prefix, node_number=None): + """Create a random id used to identify nodes""" + node_id = "" # nothing too bad happens if this doesn't get set + if node_number is not None: + # just make an id based on the node_number + hash_key = f"{node_number + 1:03d}" + else: + # use the container id if we are running inside docker + hash_key = _getIdHash(str(uuid.uuid1())) + proc_file = "/proc/self/cgroup" + if os.path.isfile(proc_file): + with open(proc_file) as f: + first_line = f.readline() + if first_line: + fields = first_line.split(":") + if len(fields) >= 3: + field = fields[2] + if field.startswith("/docker/"): + docker_len = len("/docker/") + + if len(field) > docker_len + 12: + n = docker_len + m = n + 12 + node_id = field[n:m] + + if node_id: + key = f"{prefix}-{node_id}-{hash_key}" + else: + key = f"{prefix}-{hash_key}" + return key + + +def getObjPartition(id, count): + """Get the id of the dn node that should be handling the given obj id""" + hash_code = _getIdHash(id) + hash_value = int(hash_code, 16) + number = hash_value % count + return number + + +def getNodeNumber(app): + if app["node_type"] == "sn": + log.error("node number if only for DN nodes") + raise ValueError() + + dn_ids = app["dn_ids"] + log.debug(f"getNodeNumber(from dn_ids: {dn_ids})") + for i in range(len(dn_ids)): + dn_id = dn_ids[i] + if dn_id == app["id"]: + log.debug(f"returning nodeNumber: {i}") + return i + log.error("getNodeNumber, no matching id") + return -1 + + +def getNodeCount(app): + dn_urls = app["dn_urls"] + log.debug(f"getNodeCount for dn_urls: {dn_urls}") + dn_node_count = len(dn_urls) + return dn_node_count + + +def validateInPartition(app, obj_id): + node_number = getNodeNumber(app) + node_count = getNodeCount(app) + msg = f"obj_id: {obj_id}, node_count: {node_count}, " + msg += f"node_number: {node_number}" + log.debug(msg) + partition_number = getObjPartition(obj_id, node_count) + if partition_number != node_number: + # The request shouldn't have come to this node' + msg = f"wrong node for 'id':{obj_id}, expected node {node_number} " + msg += f"got {partition_number}" + log.error(msg) + raise KeyError(msg) + + +def getDataNodeUrl(app, obj_id): + """Return host/port for datanode for given obj_id. + Throw exception if service is not ready""" + dn_urls = app["dn_urls"] + dn_node_count = getNodeCount(app) + node_state = app["node_state"] + if node_state != "READY" or dn_node_count <= 0: + msg = "Service not ready" + log.warn(msg) + raise HTTPServiceUnavailable() + dn_number = getObjPartition(obj_id, dn_node_count) + url = dn_urls[dn_number] + log.debug(f"got dn_url: {url} for obj_id: {obj_id}") + return url diff --git a/hsds/util/rangegetUtil.py b/hsds/util/rangegetUtil.py index 3e3df563..9fb53d6a 100644 --- a/hsds/util/rangegetUtil.py +++ b/hsds/util/rangegetUtil.py @@ -82,7 +82,7 @@ def _chunk_dist(chunk_left, chunk_right): def _find_min_pair(h5chunks, max_gap=None): """ Given a list of chunk_map entries which are sorted by offset, - return the indicies of the two chunks nearest to each other in the file. + return the indices of the two chunks nearest to each other in the file. If max_gap is set, chunks must be within max_gap bytes """ num_chunks = len(h5chunks) diff --git a/hsds/util/storUtil.py b/hsds/util/storUtil.py index 3bbb073c..7b3b8a4e 100644 --- a/hsds/util/storUtil.py +++ b/hsds/util/storUtil.py @@ -69,7 +69,7 @@ def getCompressors(): def getSupportedFilters(include_compressors=True): """return list of other supported filters""" filters = [ - # "bitshuffle", + "bitshuffle", "shuffle", "fletcher32", "nbit", # No-op @@ -493,7 +493,7 @@ async def getStorBytes(app, chunk_bytes = [] for chunk_location in chunk_locations: - log.debug(f"getStoreBytes - processing chunk_location: {chunk_location}") + log.debug(f"getStorBytes - processing chunk_location: {chunk_location}") n = chunk_location.offset - offset if n < 0: log.warn(f"getStorBytes - unexpected offset for chunk_location: {chunk_location}") diff --git a/hsds/util/timeUtil.py b/hsds/util/timeUtil.py deleted file mode 100755 index e4ae9d3f..00000000 --- a/hsds/util/timeUtil.py +++ /dev/null @@ -1,83 +0,0 @@ -############################################################################## -# Copyright by The HDF Group. # -# All rights reserved. # -# # -# This file is part of HSDS (HDF5 Scalable Data Service), Libraries and # -# Utilities. The full HSDS copyright notice, including # -# terms governing use, modification, and redistribution, is contained in # -# the file COPYING, which can be found at the root of the source code # -# distribution tree. If you do not have access to this file, you may # -# request a copy from help@hdfgroup.org. # -############################################################################## -from datetime import datetime -import time -import os -import pytz - - -def unixTimeToUTC(timestamp): - """Convert unix timestamp (seconds since Jan 1, 1970, to ISO-8601 - compatible UTC time string. - - """ - utc = pytz.utc - dtTime = datetime.fromtimestamp(timestamp, utc) - iso_str = dtTime.isoformat() - # isoformat returns a string like this: - # '2014-10-30T04:25:21+00:00' - # strip off the '+00:00' and replace - # with 'Z' (both are ISO-8601 compatible) - npos = iso_str.rfind("+") - iso_z = iso_str[:npos] + "Z" - return iso_z - - -def elapsedTime(timestamp): - """Get Elapsed time from given timestamp""" - delta = int(time.time()) - timestamp - if delta < 0: - return "Invalid timestamp!" - day_length = 24 * 60 * 60 - days = 0 - hour_length = 60 * 60 - hours = 0 - minute_length = 60 - minutes = 0 - ret_str = "" - - if delta > day_length: - days = delta // day_length - delta = delta % day_length - ret_str += "{} days ".format(days) - if delta > hour_length or days > 0: - hours = delta // hour_length - delta = delta % hour_length - ret_str += "{} hours ".format(hours) - if delta > minute_length or days > 0 or hours > 0: - minutes = delta // minute_length - delta = delta % minute_length - ret_str += "{} minutes ".format(minutes) - ret_str += "{} seconds".format(delta) - return ret_str - - -def getNow(app): - """ - Get current time in unix timestamp - - Returns a precise timestamp even on platforms where - time.time() has low resolution (e.g. Windows) - """ - system = os.name - current_time = 0 - - if system == "nt": - # Windows - current_time = (time.perf_counter() - app["start_time_relative"]) + app["start_time"] - elif system == "posix": - # Unix - current_time = time.time() - else: - raise ValueError(f"Unsupported OS: {system}") - - return current_time diff --git a/lambda_build.sh b/lambda_build.sh deleted file mode 100755 index 58f4a35a..00000000 --- a/lambda_build.sh +++ /dev/null @@ -1 +0,0 @@ -docker build -f Dockerfile.lambda -t hslambda . diff --git a/lambda_function.py b/lambda_function.py deleted file mode 100755 index 8bacf02a..00000000 --- a/lambda_function.py +++ /dev/null @@ -1,275 +0,0 @@ -import multiprocessing -import os -import json -import time -import logging -import requests_unixsocket -import uuid -from hsds.hsds_app import HsdsApp - -# note: see https://aws.amazon.com/blogs/compute/parallel-processing-in-python-with-aws-lambda/ - - -def getEventMethod(event): - method = "GET" # default - if "method" in event: - method = event["method"] - else: - # scan for method in the api gateway 2.0 format - if "requestContext" in event: - reqContext = event["requestContext"] - if "http" in reqContext: - http = reqContext["http"] - if "method" in http: - method = http["method"] - return method - - -def getEventPath(event): - path = None - if "path" in event: - path = event["path"] - else: - # scan for path in the api gateway 2.0 format - if "requestContext" in event: - reqContext = event["requestContext"] - if "http" in reqContext: - http = reqContext["http"] - if "path" in http: - path = http["path"] - return path - - -def getEventHeaders(event): - headers = {} # default - if "headers" in event: - event_headers = event["headers"] - for k in event_headers: - v = event_headers[k] - headers[k] = v - # set User-Agent to let HSDS know that this is Lambda - headers["User-Agent"] = "AWSLambda" - return headers - - -def getEventParams(event): - params = {} # default - if "params" in event: - params = event["params"] - elif "queryStringParameters" in event: - params = event["queryStringParameters"] - return params - - -def getEventBody(event): - body = {} # default - if "body" in event: - body = event["body"] - return body - - -def invoke(hsds, method, path, params=None, headers=None, body=None): - # invoke given request - req = hsds.endpoint + path - print(f"invoke: {req}") - result = {} - with requests_unixsocket.Session() as s: - try: - if method == "GET": - rsp = s.get(req, params=params, headers=headers) - elif method == "POST": - rsp = s.post(req, params=params, headers=headers, data=body) - elif method == "PUT": - rsp = s.put(req, params=params, headers=headers, data=body) - elif method == "DELETE": - rsp = s.delete(req, params=params, headers=headers) - else: - err_msg = f"Unexpected request method: {method}" - print(err_msg) - return {"status_code": 400, "error": err_msg} - - print(f"got status_code: {rsp.status_code} from req: {req}") - - result["isBase64Encoded"] = False - result["statusCode"] = rsp.status_code - # convert case-insisitive headers to dict - result["headers"] = json.dumps(dict(rsp.headers)) - - if rsp.status_code in (200, 201): - if rsp.text is None or len(rsp.text) == 0: - pass # empty response text - elif rsp.headers.get("Content-Type") == "application/octet-stream": - # hexencode the response - result["body"] = rsp.content.hex() - result["isBase64Encoded"] = True - else: - # should be json - try: - rsp_json = json.loads(rsp.text) - result["body"] = rsp_json - except json.JSONDecodeError: - print(f"unexpected response: {rsp.text}") - result["statusCode"] = 500 - else: - body = {"statusCode": rsp.status_code} - result["body"] = json.dumps(body) - - except Exception as e: - print(f"got exception: {e}, quitting") - except KeyboardInterrupt: - print("got KeyboardInterrupt, quitting") - finally: - print("request done") - return result - - -def lambda_handler(event, context): - # setup logging - if "LOG_LEVEL" in os.environ: - log_level_cfg = os.environ["LOG_LEVEL"] - else: - log_level_cfg = "INFO" - if log_level_cfg == "DEBUG": - log_level = logging.DEBUG - elif log_level_cfg == "INFO": - log_level = logging.INFO - elif log_level_cfg in ("WARN", "WARNING"): - log_level = logging.WARN - elif log_level_cfg == "ERROR": - log_level = logging.ERROR - else: - print(f"unsupported log_level: {log_level_cfg}, using INFO instead") - log_level = logging.INFO - - logging.basicConfig(format="%(asctime)s %(message)s", level=log_level) - - if "AWS_S3_GATEWAY" not in os.environ: - err_msg = "AWS_S3_GATEWAY environment variable not set" - print(err_msg) - return {"status_code": 500, "error": err_msg} - - # process event data - function_name = context.function_name - if "AWS_ROLE_ARN" in os.environ: - print(f"using AWS_ROLE_ARN: {os.environ['AWS_ROLE_ARN']}") - if "AWS_SESSION_TOKEN" in os.environ: - print(f"using AWS_SESSION_TOKEN: {os.environ['AWS_SESSION_TOKEN']}") - method = getEventMethod(event) - if method not in ("GET", "POST", "PUT", "DELETE"): - err_msg = f"method: {method} is unsupported" - print(err_msg) - return {"status_code": 400, "error": err_msg} - - headers = getEventHeaders(event) - params = getEventParams(event) - req = getEventPath(event) - if not req: - err_msg = "no request path provided ('path' key not present?)" - print(err_msg) - return {"status_code": 400, "error": err_msg} - - # determine if this method will modify storage - # if not, we'll pass readonly to the dn nodes so they - # will not run s3sync task - if method == "GET": - readonly = True - elif method == "PUT": - readonly = False - elif method == "DELETE": - readonly = False - elif method == "POST": - # post is write unless we are doing a point selection - if req.startswith("/datasets") and req.endswith("value"): - readonly = True - else: - readonly = False - else: - print(f"unexpected method: {method}") - readonly = False - - if headers and not isinstance(headers, dict): - err_msg = f"expected headers to be a dict, but got: {type(headers)}" - print(err_msg) - return {"status_code": 400, "error": err_msg} - - if params and not isinstance(params, dict): - err_msg = f"expected params to be a dict, but got: {type(params)}" - print(err_msg) - return {"status_code": 400, "error": err_msg} - - body = getEventBody(event) - if body and method not in ("PUT", "POST"): - err_msg = "body only support with PUT and POST methods" - print(err_msg) - return {"status_code": 400, "error": err_msg} - - cpu_count = multiprocessing.cpu_count() - if "TARGET_DN_COUNT" in os.environ: - target_dn_count = int(os.environ["TARGET_DN_COUNT"]) - else: - # base dn count on half the VCPUs (rounded up) - target_dn_count = -(-cpu_count // 2) - - tmp_dir = "/tmp" - rand_name = uuid.uuid4().hex[:8] - socket_dir = f"{tmp_dir}/hs{rand_name}/" - - # instantiate hsdsapp object - hsds = HsdsApp( - username=function_name, - password="lambda", - islambda=True, - dn_count=target_dn_count, - readonly=readonly, - socket_dir=socket_dir, - ) - hsds.run() - - # wait for server to startup - waiting_on_ready = True - - while waiting_on_ready: - try: - time.sleep(0.1) - hsds.check_processes() - except Exception as e: - print(f"got exception: {e}") - break - if hsds.ready: - waiting_on_ready = False - print("READY! use endpoint:", hsds.endpoint) - - result = invoke(hsds, method, req, params=params, headers=headers, body=body) - hsds.check_processes() - hsds.stop() - - if "requestContext" in event: - # Invoked from API Gateway - we need to stringify the result - if "body" not in result: - return {"status_code": 500, "error": f"unexpected result: {result}"} - result = json.dumps(result["body"]) - return result - - -# -# main -# - -if __name__ == "__main__": - # request to use for testing - req = "/datasets/d-d38053ea-3418fe27-22d9-478e7b-913279/value" - params = {"domain": "/shared/tall.h5", "bucket": "hdflab2"} - - class Context: - @property - def function_name(self): - return "hslambda" - - # simplified event format - # see: - # https://docs.aws.amazon.com/apigateway/latest/developerguide/ \ - # http-api-develop-integrations-lambda.html - # for a description of the API Gateway 2.0 format which is also supported - event = {"method": "GET", "path": req, "params": params} - context = Context() - result = lambda_handler(event, context) diff --git a/openapi.yml b/openapi.yml new file mode 100644 index 00000000..86d2a66f --- /dev/null +++ b/openapi.yml @@ -0,0 +1,2973 @@ +openapi: 3.1.0 +info: + title: HSDS REST API + description: | + HSDS (Highly Scalable Data Service) is a REST-based web service for HDF5 data. + It implements the HDF5 REST API convention used by h5pyd and h5serv: domains + (roughly analogous to HDF5 files), groups, datasets, committed datatypes, + links, and attributes, plus dataset value read/write with hyperslab/point/query + selection. + + This spec documents the public Service-Node (SN) REST API - the interface + exposed to REST clients. It was generated from the HSDS source code + (hsds/domain_sn.py, group_sn.py, link_sn.py, attr_sn.py, ctype_sn.py, + dset_sn.py, chunk_sn.py, servicenode.py) rather than hand-written, so it + should closely track actual server behavior; however some undocumented + edge cases / internal inconsistencies noted in the source are called out + in operation descriptions rather than fully modeled. + + ### Addressing a domain + Every request (except `GET /domains`) operates on a "domain" (an HDF5 + file-like container). The domain is specified either via the `domain` + query parameter or the `X-Hdf-domain` request header. An optional `bucket` + query parameter (or `X-Hdf-bucket` header) selects a non-default storage + bucket. + + ### Authentication + Basic Auth (`Authorization: Basic `). Many read (GET) + operations also accept anonymous requests (no `Authorization` header) if + the server is configured with `allow_noauth` - in that case the request + is treated as user `"default"`. All write operations (PUT/POST/DELETE) + require a valid `Authorization` header. + + ### Error responses + HSDS does not use a JSON error envelope. Error responses use aiohttp's + default plain-text rendering: `Content-Type: text/plain`, body of the + form `": "` (e.g. `"400: Invalid domain name"`). + + ### Boolean query parameters + Many query-string boolean parameters (e.g. `verbose`, `include_attrs`, + `include_links`, `getalias`, `getdnids`, `flush`, `rescan`, `meta_only`, + `keep_root`) are, in several handlers, parsed with a naive + "is this key present and non-empty string" check rather than true + boolean parsing. In those handlers `?flag=0` or `?flag=false` are + **truthy** (they enable the flag, they do not disable it) - only + omitting the parameter entirely is guaranteed to mean "false". This is + called out per-parameter below where explicitly confirmed against the + source; treat any other boolean query parameter with the same + suspicion unless it's documented as using the shared `getBooleanParam` + helper (which does parse `"0"`/`"false"` correctly). + + ### Known service-level issues (not just documentation gaps) + - `PUT .../value` with a `query` parameter (query-based update) and + `Limit` set: when the matching rows span more than one chunk, `Limit` + is applied independently within each chunk rather than coordinated + across chunks, so more than `Limit` rows total can end up updated. + This is a known, deliberately-deferred bug - see the `query` + parameter description on `PUT /datasets/{id}/value`. + version: "1.0.0" + license: + name: Apache-2.0 + +servers: + - url: "{scheme}://{host}:{port}" + description: HSDS service node endpoint + variables: + scheme: + enum: [http, https] + default: http + host: + default: localhost + port: + default: "5101" + +security: + - basicAuth: [] + - {} # anonymous access is allowed for many read operations when the server has allow_noauth set + +tags: + - name: Status + description: Service/node status and health-check endpoints. Not domain-specific - no `domain` param is needed or used. + - name: Domains + description: Domain (HDF5-file-equivalent) and folder operations + - name: ACLs + description: Access control list operations + - name: Groups + description: Group operations + - name: Links + description: Link operations (hard/soft/external links between groups and objects) + - name: Attributes + description: Attribute operations, shared across groups/datasets/datatypes + - name: Datatypes + description: Committed (named) datatype operations + - name: Datasets + description: Dataset metadata, shape, and type operations + - name: Value + description: Dataset data read/write (hyperslab, point, and query selection) + +paths: + /about: + get: + operationId: GetAbout + tags: [Status] + summary: Get service status and version info + description: | + Returns general information about the service as a whole (not a + specific node): overall cluster state, version, and node count. + This is the endpoint `tools/status_check.py` polls to determine + when the service has finished starting up (`state == "READY"`). + + Authentication is optional. If `Authorization` is supplied it is + validated and the response reflects the authenticated user; + otherwise the response reports `username: "anonymous"`. + security: [] + responses: + '200': + description: Service status. + content: + application/json: + schema: + type: object + properties: + start_time: + type: integer + description: Unix timestamp the service node started. + state: + type: string + enum: [INITIALIZING, WAITING, READY, TERMINATING] + description: Overall cluster readiness state. + hsds_version: { type: string } + name: + type: string + description: Configured server name (`server_name` config value). + greeting: { type: string } + about: { type: string } + node_count: + type: integer + description: Number of DN nodes currently registered. + dn_urls: + type: array + items: { type: string } + dn_ids: + type: array + items: { type: string } + username: + type: string + description: The authenticated username, or "anonymous" if no credentials were supplied. + isadmin: { type: boolean } + '401': { $ref: '#/components/responses/Unauthorized' } + + /info: + get: + operationId: GetInfo + tags: [Status] + summary: Get this node's state and resource usage + description: | + Returns state and OS-level resource usage (CPU, disk I/O, network + I/O) for the specific node that handles the request - unlike + `/about`, this reflects one node's local state, not the cluster + as a whole. No authentication is required or checked. + security: [] + responses: + '200': + description: Node status. + content: + application/json: + schema: + type: object + properties: + node: + type: object + properties: + id: { type: string } + type: + type: string + description: Node type - "sn" for the service node handling this request. + enum: [sn, dn, head_node] + start_time: { type: integer } + state: + type: string + enum: [INITIALIZING, WAITING, READY, TERMINATING] + node_number: + type: integer + description: Only present for `dn` nodes. + node_count: { type: integer } + cpu: + type: object + properties: + percent: { type: number } + cores: { type: integer } + diskio: + type: object + properties: + read_count: { type: integer } + read_time: { type: integer } + read_bytes: { type: integer } + write_count: { type: integer } + write_time: { type: integer } + write_bytes: { type: integer } + netio: + type: object + properties: + bytes_sent: + type: integer + description: "Note: a source-level bug (hsds/basenode.py) assigns bytes_recv over this key immediately after setting it, so this actually reports received bytes, and sent-byte count is not exposed." + packets_sent: { type: integer } + packets_recv: { type: integer } + errin: { type: integer } + errout: { type: integer } + dropin: { type: integer } + dropout: { type: integer } + memory: + type: object + properties: + phys_total: { type: integer } + phys_available: { type: integer } + swap_total: { type: integer } + swap_used: { type: integer } + swap_free: { type: integer } + percent: { type: number } + disk: + type: object + properties: + total: { type: integer } + used: { type: integer } + free: { type: integer } + percent: { type: number } + log_stats: + type: object + description: Count of log messages emitted, by level. + req_count: + type: integer + description: Number of HTTP requests handled by this node so far. + s3_stats: + type: object + description: Present only when using S3 storage. + azure_stats: + type: object + description: Present only when using Azure Blob storage. + meta_cache_stats: + type: object + description: Metadata object cache stats. Populated only on DN nodes - empty on SN/head. + properties: + count: { type: integer } + dirty_count: { type: integer } + utililization_per: { type: number } + mem_used: { type: integer } + mem_target: { type: integer } + chunk_cache_stats: + type: object + description: Chunk data cache stats. Populated only on DN nodes - empty on SN/head. + properties: + count: { type: integer } + dirty_count: { type: integer } + utililization_per: { type: number } + mem_used: { type: integer } + mem_target: { type: integer } + domain_cache_stats: + type: object + description: Domain object cache stats. Populated only on DN nodes - empty on SN/head. + properties: + count: { type: integer } + dirty_count: { type: integer } + utililization_per: { type: number } + mem_used: { type: integer } + mem_target: { type: integer } + + /: + get: + operationId: GetDomain + tags: [Domains] + summary: Get domain (or resolve an object by h5path within it) + description: | + Returns JSON describing the domain identified by the `domain` query + param / `X-Hdf-domain` header. A `domain` (or `X-Hdf-domain`) is + required here - omitting it 400s. Use `GET /domains` to list + top-level domains instead. + + If `h5path` is given, resolves that path (relative to `parent_id`, or + the domain root if `parent_id` is absent) and returns the JSON for the + object found there (group/dataset/datatype) instead of the domain + itself. `getdnids`/`verbose`/`getobjs` only apply to the plain + domain-JSON response - they're silently ignored when `h5path` is + used. + parameters: + - $ref: '#/components/parameters/DomainParam' + - $ref: '#/components/parameters/DomainHeader' + - $ref: '#/components/parameters/BucketParam' + - name: parent_id + in: query + description: Base group id to resolve `h5path` under. Defaults to the domain root. + schema: { type: string } + - name: h5path + in: query + description: If present, resolve this path and return the target object's JSON instead of the domain JSON. + schema: { type: string } + - name: include_links + in: query + description: Include links when returning an `h5path`-resolved object. + schema: { type: boolean, default: false } + - name: include_attrs + in: query + description: Include attributes when returning an `h5path`-resolved object. + schema: { type: boolean, default: false } + - name: follow_soft_links + in: query + description: Allow following soft links while resolving `h5path`. + schema: { type: boolean, default: false } + - name: follow_external_links + in: query + description: Allow following external links while resolving `h5path`. + schema: { type: boolean, default: false } + - name: verbose + in: query + description: Include extended domain statistics (num_groups, num_datasets, total_size, md5_sum, etc.) + schema: { type: boolean, default: false } + - name: getobjs + in: query + description: Include a full `domain_objs` map of every object in the domain. + schema: { type: boolean, default: false } + - name: getdnids + in: query + description: Include the list of data-node ids serving this domain. + schema: { type: boolean, default: false } + responses: + '200': + description: Domain JSON, or (with `h5path`) resolved-object JSON - see description. + content: + application/json: + schema: + oneOf: + - $ref: '#/components/schemas/DomainResponse' + - $ref: '#/components/schemas/ObjectResponse' + '400': { $ref: '#/components/responses/BadRequest' } + '403': { $ref: '#/components/responses/Forbidden' } + '404': { $ref: '#/components/responses/NotFound' } + '410': { $ref: '#/components/responses/Gone' } + '503': { $ref: '#/components/responses/ServiceUnavailable' } + + put: + operationId: PutDomain + tags: [Domains] + summary: Create a domain or folder, flush pending writes, or trigger a rescan + description: | + Default behavior creates a new domain (or folder, if `folder: true`). + Passing `flush: true` (query param or body key) flushes the domain + instead. Passing `rescan: true` triggers a background rescan of the + domain's root group. `flush` and `rescan` may be combined; flush runs + first. + + `rescan` only makes sense for domains that have a `root` group + (ordinary, non-folder domains) - passing `rescan=true` for an + existing *folder* domain 400s rather than attempting a rescan. + + The `201` response does not actually include a `class` field (the + DN's creation handler never sets it, even though `DomainResponse` - + shared with `GET /` - declares one); fetch the domain again via + `GET /` if you need `class`. + parameters: + - $ref: '#/components/parameters/DomainParam' + - $ref: '#/components/parameters/DomainHeader' + - $ref: '#/components/parameters/BucketParam' + - name: getdnids + in: query + schema: { type: boolean, default: false } + description: Include `dn_ids` in the response (also settable via body). + - name: flush + in: query + schema: { type: boolean, default: false } + description: Flush the domain instead of creating it (also settable via body). + - name: rescan + in: query + schema: { type: boolean, default: false } + description: Rescan the domain root instead of creating it (also settable via body). + requestBody: + required: false + content: + application/json: + schema: + type: object + properties: + getdnids: { type: boolean } + flush: { type: boolean } + rescan: { type: boolean } + folder: + type: boolean + description: Create a folder domain (a container for other domains) rather than a data domain. + owner: + type: string + description: Domain owner. Setting this to a user other than the requester requires admin privileges. + linked_domain: + type: string + description: Existing domain whose root group this new domain should link to (creates an alias domain). Mutually exclusive with `folder`. + linked_bucket: + type: string + description: Storage bucket for `linked_domain`, if different from the request's bucket. + root_id: + type: string + description: Client-supplied root group id (schema2 uuid) for the new domain. + group: + type: object + description: Options for the new domain's root group. + properties: + creationProperties: + type: object + description: Creation properties copied onto the new root group. + responses: + '201': + description: Domain created. + content: + application/json: + schema: { $ref: '#/components/schemas/DomainResponse' } + '204': + description: Flush or rescan completed with nothing further to report (or `getdnids` requested a body with just `dn_ids`, in which case 200 is returned instead - see description). + '400': { $ref: '#/components/responses/BadRequest' } + '403': { $ref: '#/components/responses/Forbidden' } + '404': { $ref: '#/components/responses/NotFound' } + '409': { $ref: '#/components/responses/Conflict' } + '410': { $ref: '#/components/responses/Gone' } + '503': { $ref: '#/components/responses/ServiceUnavailable' } + + delete: + operationId: DeleteDomain + tags: [Domains] + summary: Delete a domain + description: | + Deletes the domain and (unless `keep_root` is set) its root group and + all reachable objects/chunks. Deleting a non-empty folder domain + (one that still has sub-domains/folders) returns 409. + + If `meta_only` is set, only the server's in-memory domain cache entry + is cleared - no authentication or ACL check is performed in this case + and no actual storage delete happens. + + If a JSON body is supplied, its `meta_only`/`keep_root` keys take + priority over the query parameters of the same name (the two are not + merged). + parameters: + - $ref: '#/components/parameters/DomainParam' + - $ref: '#/components/parameters/DomainHeader' + - $ref: '#/components/parameters/BucketParam' + - name: meta_only + in: query + schema: { type: boolean, default: false } + description: Only clear the cached domain entry; skip auth and the real delete. + - name: keep_root + in: query + schema: { type: boolean, default: false } + description: Don't delete the root group object when deleting the domain. + requestBody: + required: false + content: + application/json: + schema: + type: object + properties: + meta_only: { type: boolean } + keep_root: { type: boolean } + responses: + '200': + description: Domain deleted (or meta-cache cleared). + content: + application/json: + schema: { type: object } + '400': { $ref: '#/components/responses/BadRequest' } + '403': { $ref: '#/components/responses/Forbidden' } + '404': { $ref: '#/components/responses/NotFound' } + '409': { $ref: '#/components/responses/Conflict' } + '410': { $ref: '#/components/responses/Gone' } + + post: + operationId: PostDomain + tags: [Domains] + summary: Resolve multiple h5paths within a domain in one request + parameters: + - $ref: '#/components/parameters/DomainParam' + - $ref: '#/components/parameters/DomainHeader' + - $ref: '#/components/parameters/BucketParam' + - name: parent_id + in: query + schema: { type: string } + description: Base group id to resolve each h5path under. Defaults to the domain root. + - name: include_links + in: query + schema: { type: boolean, default: false } + - name: include_attrs + in: query + schema: { type: boolean, default: false } + - name: follow_soft_links + in: query + schema: { type: boolean, default: false } + - name: follow_external_links + in: query + schema: { type: boolean, default: false } + requestBody: + required: true + content: + application/json: + schema: + type: object + required: [h5paths] + properties: + h5paths: + type: array + items: { type: string } + description: List of h5paths to resolve. + responses: + '200': + description: Map of each requested h5path to its resolved object JSON. + content: + application/json: + schema: + type: object + properties: + h5paths: + type: object + additionalProperties: + $ref: '#/components/schemas/ObjectResponse' + '400': { $ref: '#/components/responses/BadRequest' } + '403': { $ref: '#/components/responses/Forbidden' } + '404': { $ref: '#/components/responses/NotFound' } + '410': { $ref: '#/components/responses/Gone' } + + /domains: + get: + operationId: GetDomains + tags: [Domains] + summary: List domains under a folder + description: | + Lists domains directly under the folder given by `domain`/`X-Hdf-domain` + (default `/`, i.e. top-level domains). Supports glob-pattern filtering, + a query filter over root-group attributes, and S3-style pagination. + + ### Query syntax (`query`) + Same engine and grammar as `GET /datasets/{id}/value`'s `query` + param (`h5json.query_util.arrayQuery`): `"temperature > 20"`, + `"city IN ('seattle', 'portland')"`, combined with + `AND`/`OR`/`NOT`/`XOR`. Each candidate domain's root-group scalar + primitive attributes (integer/float/string) are used as the query + variables - unlike a dataset (one fixed schema for every row), + each domain can have a completely different set of attributes, + so: + - A domain missing an attribute the query references (or having + it as a non-scalar or non-primitive type) simply doesn't match + - this is not an error, even if *no* domain anywhere has that + attribute. + - Folder domains (no root group) never match a query. + - Genuine syntax errors (unlike a missing attribute) still 400. + parameters: + - $ref: '#/components/parameters/DomainParam' + - $ref: '#/components/parameters/DomainHeader' + - $ref: '#/components/parameters/BucketParam' + - name: pattern + in: query + description: Glob pattern matched against each candidate domain's basename. + schema: { type: string } + - name: query + in: query + description: Query-filter string over root-group scalar attributes (see syntax above). + schema: { type: string } + - name: verbose + in: query + schema: { type: boolean, default: false } + - $ref: '#/components/parameters/LimitParam' + - $ref: '#/components/parameters/MarkerParam' + responses: + '200': + description: List of domains. + content: + application/json: + schema: { $ref: '#/components/schemas/DomainList' } + '400': { $ref: '#/components/responses/BadRequest' } + '503': { $ref: '#/components/responses/ServiceUnavailable' } + + /acls/{username}: + parameters: + - name: username + in: path + required: true + schema: { type: string } + - $ref: '#/components/parameters/DomainParam' + - $ref: '#/components/parameters/DomainHeader' + - $ref: '#/components/parameters/BucketParam' + get: + operationId: GetAcl + tags: [ACLs] + summary: Get the ACL entry for a user on a domain + description: | + Users may always read their own ACL entry (or the `default` entry) + with just "read" permission on the domain; reading another user's + entry requires "readACL" permission. + + Note: with no `domain` given, this raises an unhandled server-side + error rather than a clean 400 (unlike most domain-scoped GETs, + which explicitly check for a missing domain and 400). + responses: + '200': + description: ACL entry. + content: + application/json: + schema: + type: object + properties: + acl: { $ref: '#/components/schemas/ACL' } + hrefs: + type: array + items: { $ref: '#/components/schemas/Href' } + '400': { $ref: '#/components/responses/BadRequest' } + '403': { $ref: '#/components/responses/Forbidden' } + '404': { $ref: '#/components/responses/NotFound' } + '410': { $ref: '#/components/responses/Gone' } + put: + operationId: PutAcl + tags: [ACLs] + summary: Create or update the ACL entry for a user on a domain + description: | + Requires "updateACL" permission on the domain (checked the same + way as every other ACL/domain-mutating operation). + requestBody: + required: true + content: + application/json: + schema: + type: object + description: Any subset of the 6 ACL action keys, each a boolean. + properties: + create: { type: boolean } + read: { type: boolean } + update: { type: boolean } + delete: { type: boolean } + readACL: { type: boolean } + updateACL: { type: boolean } + responses: + '201': + description: ACL entry created/updated. + content: + application/json: + schema: { type: object } + '400': { $ref: '#/components/responses/BadRequest' } + '403': { $ref: '#/components/responses/Forbidden' } + '404': { $ref: '#/components/responses/NotFound' } + + /acls: + get: + operationId: GetAcls + tags: [ACLs] + summary: List all ACL entries on a domain + description: | + Unlike `GET /acls/{username}`, listing all ACL entries always + requires "readACL" permission (no self-record exception). + + Note: with no `domain` given, this raises an unhandled server-side + error rather than a clean 400 (see the same note on + `GET /acls/{username}`). + parameters: + - $ref: '#/components/parameters/DomainParam' + - $ref: '#/components/parameters/DomainHeader' + - $ref: '#/components/parameters/BucketParam' + responses: + '200': + description: List of ACL entries. + content: + application/json: + schema: + type: object + properties: + acls: + type: array + items: { $ref: '#/components/schemas/ACL' } + hrefs: + type: array + items: { $ref: '#/components/schemas/Href' } + '400': { $ref: '#/components/responses/BadRequest' } + '403': { $ref: '#/components/responses/Forbidden' } + '404': { $ref: '#/components/responses/NotFound' } + '410': { $ref: '#/components/responses/Gone' } + + /groups: + get: + operationId: GetGroups + tags: [Groups] + summary: List all group ids in a domain + parameters: + - $ref: '#/components/parameters/DomainParam' + - $ref: '#/components/parameters/DomainHeader' + - $ref: '#/components/parameters/BucketParam' + - $ref: '#/components/parameters/LimitParam' + - $ref: '#/components/parameters/MarkerParam' + responses: + '200': + description: List of group ids. + content: + application/json: + schema: + type: object + properties: + groups: + type: array + items: { type: string } + hrefs: + type: array + items: { $ref: '#/components/schemas/Href' } + '400': { $ref: '#/components/responses/BadRequest' } + '403': { $ref: '#/components/responses/Forbidden' } + post: + operationId: PostGroup + tags: [Groups] + summary: Create one or more groups + description: | + Accepts a single group-creation object, or a JSON list to batch-create + multiple groups in one request. `type` is not permitted in the body + (groups have no datatype). + + **Bug:** `implicit` is only forwarded to argument construction for + single-object creates. For a batch create (a JSON list with more + than one item), it is not passed through and always behaves as + `implicit=0` regardless of the query param - missing intermediate + groups will cause the create to fail rather than being created + implicitly. + parameters: + - $ref: '#/components/parameters/DomainParam' + - $ref: '#/components/parameters/DomainHeader' + - $ref: '#/components/parameters/BucketParam' + - name: implicit + in: query + schema: { type: boolean, default: false } + description: Allow implicit creation of missing intermediate parent groups when `h5path` is used. + requestBody: + required: false + content: + application/json: + schema: + oneOf: + - $ref: '#/components/schemas/CreateObjectRequest' + - type: array + items: { $ref: '#/components/schemas/CreateObjectRequest' } + responses: + '201': + description: Group(s) created. + content: + application/json: + schema: + oneOf: + - $ref: '#/components/schemas/GroupObject' + - type: object + properties: + objects: + type: array + items: { $ref: '#/components/schemas/GroupObject' } + '400': { $ref: '#/components/responses/BadRequest' } + '403': { $ref: '#/components/responses/Forbidden' } + '409': { $ref: '#/components/responses/Conflict' } + + /groups/: + get: + operationId: GetRootOrGroupByPath + tags: [Groups] + summary: Resolve a group by h5path + description: | + Registered separately from `GET /groups/{id}` (note the trailing + slash, no id segment) for the case where the caller doesn't yet + know any group id. + + **`h5path` is effectively required here, and must be absolute** + (start with `/`): the handler 400s with "Missing group id" if + there is neither a path `id` nor an `h5path`, so calling this route + with no `h5path` does NOT return the domain's root group as might + be expected - there is no way to fetch the root group without + already knowing its id (e.g. from the domain response's `root` + field). A relative `h5path` (not starting with `/`) also 400s on + this route, since there is no group id to resolve it against. + + There is no supported way to resolve a *relative* `h5path` without + an id: a `grpid` parameter for this purpose is mentioned in older + references, but it is never actually read anywhere in the + group-handling code, so it has no effect. + parameters: + - $ref: '#/components/parameters/DomainParam' + - $ref: '#/components/parameters/DomainHeader' + - $ref: '#/components/parameters/BucketParam' + - name: h5path + in: query + required: true + schema: { type: string } + description: Absolute path to resolve. Must start with "/". + - name: getalias + in: query + schema: { type: boolean, default: false } + - name: include_links + in: query + schema: { type: boolean, default: false } + - name: include_attrs + in: query + schema: { type: boolean, default: false } + responses: + '200': + description: Group JSON. + content: + application/json: + schema: { $ref: '#/components/schemas/GroupObject' } + '400': { $ref: '#/components/responses/BadRequest' } + '403': { $ref: '#/components/responses/Forbidden' } + '404': { $ref: '#/components/responses/NotFound' } + + /groups/{id}: + parameters: + - name: id + in: path + required: true + schema: { type: string } + - $ref: '#/components/parameters/DomainParam' + - $ref: '#/components/parameters/DomainHeader' + - $ref: '#/components/parameters/BucketParam' + get: + operationId: GetGroup + tags: [Groups] + summary: Get a group + description: | + A group can also be located by an absolute path via the `h5path` + query parameter, resolved relative to `id`. See `GET /groups/` for + resolving an absolute `h5path` with no known group id (there is no + way to fetch the root group itself without already knowing its id). + parameters: + - name: h5path + in: query + schema: { type: string } + description: Resolve this path (relative to `id`) to a group instead of using the path id directly. + - name: getalias + in: query + schema: { type: boolean, default: false } + description: | + Include an `alias` list of h5paths that resolve to this group. + Note: parsed with a raw truthiness check rather than real + boolean parsing, so `?getalias=0` is truthy and turns this on. + - name: include_links + in: query + schema: { type: boolean, default: false } + - name: include_attrs + in: query + schema: { type: boolean, default: false } + responses: + '200': + description: Group JSON. + content: + application/json: + schema: { $ref: '#/components/schemas/GroupObject' } + '400': { $ref: '#/components/responses/BadRequest' } + '403': { $ref: '#/components/responses/Forbidden' } + '404': { $ref: '#/components/responses/NotFound' } + delete: + operationId: DeleteGroup + tags: [Groups] + summary: Delete a group + description: Deleting a domain's root group is not allowed (403) - delete the domain instead. + responses: + '200': + description: Group deleted. + content: + application/json: + schema: { type: object } + '400': { $ref: '#/components/responses/BadRequest' } + '403': { $ref: '#/components/responses/Forbidden' } + '404': { $ref: '#/components/responses/NotFound' } + + /groups/{id}/links: + parameters: + - name: id + in: path + required: true + schema: { type: string } + - $ref: '#/components/parameters/DomainParam' + - $ref: '#/components/parameters/DomainHeader' + - $ref: '#/components/parameters/BucketParam' + get: + operationId: GetLinks + tags: [Links] + summary: List links in a group + parameters: + - name: follow_links + in: query + schema: { type: boolean, default: false } + description: | + Recursively follow links into sub-groups. Response shape + changes to a map of group id to link list. Note: when set, + `CreateOrder` and `Marker` are silently ignored (only `Limit` + is honored) - the crawler used for this mode doesn't forward + them. + - name: pattern + in: query + schema: { type: string } + description: Glob pattern filter on link title. + - name: CreateOrder + in: query + schema: { type: boolean, default: false } + description: Return links in creation order instead of name order. Ignored when `follow_links` is set. + - $ref: '#/components/parameters/LimitParam' + - $ref: '#/components/parameters/MarkerParam' + responses: + '200': + description: | + List of links (or, with `follow_links`, a map of group id to + link list). Note: the two response shapes also differ in + per-link content, not just array-vs-map - the non-crawler + (default) path decorates each hard link with extra + `collection`/`target`/`href` keys not present in the `Link` + schema; these decorations are absent when `follow_links` is used. + content: + application/json: + schema: + type: object + properties: + links: + oneOf: + - type: array + items: { $ref: '#/components/schemas/Link' } + - type: object + additionalProperties: + type: array + items: { $ref: '#/components/schemas/Link' } + hrefs: + type: array + items: { $ref: '#/components/schemas/Href' } + '400': { $ref: '#/components/responses/BadRequest' } + '403': { $ref: '#/components/responses/Forbidden' } + '500': { $ref: '#/components/responses/InternalError' } + post: + operationId: PostLinks + tags: [Links] + summary: Batch-fetch links across one or more groups + description: | + Despite the POST verb this is a read operation. Provide `titles` + to fetch specific link names for the path group, or `grp_ids` to + fetch links from multiple groups in one call. + + `titles` is mutually exclusive with `follow_links`, `Limit`, and + `CreateOrder` - combining `titles` with any of those three 400s. + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + titles: + type: array + items: { type: string } + description: Specific link titles to fetch. Cannot be combined with a `grp_ids` object (dict) form. + grp_ids: + description: Either a list of group ids (fetch all/`titles` links from each), or a map of group id to a list of titles for that group. + oneOf: + - type: array + items: { type: string } + - type: object + additionalProperties: + type: array + items: { type: string } + parameters: + - name: follow_links + in: query + schema: { type: boolean, default: false } + - name: CreateOrder + in: query + schema: { type: boolean, default: false } + - $ref: '#/components/parameters/LimitParam' + - name: pattern + in: query + schema: { type: string } + responses: + '200': + description: Links for the requested group(s). + content: + application/json: + schema: + type: object + properties: + links: + oneOf: + - type: array + items: { $ref: '#/components/schemas/Link' } + - type: object + additionalProperties: + type: array + items: { $ref: '#/components/schemas/Link' } + '400': { $ref: '#/components/responses/BadRequest' } + '403': { $ref: '#/components/responses/Forbidden' } + put: + operationId: PutLinks + tags: [Links] + summary: Batch-create links across one or more groups + description: | + When `grp_ids` targets multiple groups, the overall response status + comes from a crawler that returns the highest status code seen + across all the individual per-group writes - so in practice this + operation can also surface status codes beyond 200/201/400/403 + (e.g. 404, 500) if an individual target group turns out to be + invalid or a per-group write fails. + parameters: + - name: replace + in: query + schema: { type: boolean, default: false } + description: Allow overwriting existing links with the same title. + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + links: + type: object + additionalProperties: { $ref: '#/components/schemas/LinkCreate' } + description: Map of link title to link definition, written to the path group (or to every group in `grp_ids` if that's a list). + grp_ids: + description: "Either a list of target group ids (the single top-level `links` map is written to each), or a map of group id to `{\"links\": {...}}` for per-group distinct link sets." + oneOf: + - type: array + items: { type: string } + - type: object + additionalProperties: + type: object + properties: + links: + type: object + additionalProperties: { $ref: '#/components/schemas/LinkCreate' } + responses: + '200': + description: All submitted links were duplicates of existing links. + content: + application/json: + schema: { type: object } + '201': + description: At least one new link was created. + content: + application/json: + schema: { type: object } + '400': { $ref: '#/components/responses/BadRequest' } + '403': { $ref: '#/components/responses/Forbidden' } + delete: + operationId: DeleteLinks + tags: [Links] + summary: Delete one or more links from a group by title + parameters: + - name: titles + in: query + required: true + schema: { type: string } + description: Link titles to delete, joined by `separator`. + - name: separator + in: query + schema: { type: string, default: "/" } + description: Delimiter used to split `titles` into individual link names. + responses: + '200': + description: Links deleted (deleting a nonexistent link name is not an error). + content: + application/json: + schema: { type: object } + '400': { $ref: '#/components/responses/BadRequest' } + '403': { $ref: '#/components/responses/Forbidden' } + + /groups/{id}/links/{title}: + parameters: + - name: id + in: path + required: true + schema: { type: string } + - name: title + in: path + required: true + schema: { type: string } + - $ref: '#/components/parameters/DomainParam' + - $ref: '#/components/parameters/DomainHeader' + - $ref: '#/components/parameters/BucketParam' + get: + operationId: GetLink + tags: [Links] + summary: Get a single link + responses: + '200': + description: Link. + content: + application/json: + schema: + type: object + properties: + link: { $ref: '#/components/schemas/Link' } + created: { type: number } + lastModified: + type: number + description: Links are immutable once created, so this always equals `created`. + hrefs: + type: array + items: { $ref: '#/components/schemas/Href' } + '400': { $ref: '#/components/responses/BadRequest' } + '403': { $ref: '#/components/responses/Forbidden' } + '404': { $ref: '#/components/responses/NotFound' } + '500': { $ref: '#/components/responses/InternalError' } + put: + operationId: PutLink + tags: [Links] + summary: Create a single link + requestBody: + required: true + content: + application/json: + schema: { $ref: '#/components/schemas/LinkCreate' } + responses: + '200': + description: The link duplicated an existing identical link. + content: + application/json: + schema: { type: object } + '201': + description: Link created. + content: + application/json: + schema: { type: object } + '400': { $ref: '#/components/responses/BadRequest' } + '403': { $ref: '#/components/responses/Forbidden' } + delete: + operationId: DeleteLink + tags: [Links] + summary: Delete a single link + description: | + Note: unlike `GET`/`PUT` on this same resource, this handler + validates `title` with no exception handling around it - an + invalid link name raises an uncaught `ValueError` here rather than + a clean 400 (see the `500` response). + responses: + '200': + description: Link deleted. + content: + application/json: + schema: { type: object } + '400': { $ref: '#/components/responses/BadRequest' } + '500': { $ref: '#/components/responses/InternalError' } + '403': { $ref: '#/components/responses/Forbidden' } + + /datatypes: + get: + operationId: GetDatatypes + tags: [Datatypes] + summary: List all committed datatype ids in a domain + parameters: + - $ref: '#/components/parameters/DomainParam' + - $ref: '#/components/parameters/DomainHeader' + - $ref: '#/components/parameters/BucketParam' + - $ref: '#/components/parameters/LimitParam' + - $ref: '#/components/parameters/MarkerParam' + responses: + '200': + description: List of datatype ids. + content: + application/json: + schema: + type: object + properties: + datatypes: + type: array + items: { type: string } + hrefs: + type: array + items: { $ref: '#/components/schemas/Href' } + '400': { $ref: '#/components/responses/BadRequest' } + '403': { $ref: '#/components/responses/Forbidden' } + post: + operationId: PostDatatype + tags: [Datatypes] + summary: Create one or more committed datatypes + description: | + Note: `creationProperties` (from the shared `CreateObjectRequest` + schema) is accepted and forwarded by the SN but silently discarded + by the DN for datatypes - it is never persisted and never returned + by a subsequent GET. It has no actual effect for this operation + (unlike for dataset creation, where it is meaningful). + parameters: + - $ref: '#/components/parameters/DomainParam' + - $ref: '#/components/parameters/DomainHeader' + - $ref: '#/components/parameters/BucketParam' + - name: implicit + in: query + schema: { type: boolean, default: false } + requestBody: + required: true + content: + application/json: + schema: + oneOf: + - allOf: + - $ref: '#/components/schemas/CreateObjectRequest' + - type: object + required: [type] + - type: array + items: + allOf: + - $ref: '#/components/schemas/CreateObjectRequest' + - type: object + required: [type] + description: Multi-create items may not include a `shape` key. + responses: + '201': + description: Datatype(s) created. + content: + application/json: + schema: + oneOf: + - $ref: '#/components/schemas/DatatypeObject' + - type: object + properties: + objects: + type: array + items: { $ref: '#/components/schemas/DatatypeObject' } + '400': { $ref: '#/components/responses/BadRequest' } + '403': { $ref: '#/components/responses/Forbidden' } + '409': { $ref: '#/components/responses/Conflict' } + + /datatypes/: + get: + operationId: GetDatatypeByPath + tags: [Datatypes] + summary: Resolve a committed datatype by h5path + description: Registered separately from `GET /datatypes/{id}` (trailing slash, no id segment) for locating a datatype purely by `h5path`/`grpid` without already knowing its id. Otherwise identical to `GET /datatypes/{id}`. + parameters: + - $ref: '#/components/parameters/DomainParam' + - $ref: '#/components/parameters/DomainHeader' + - $ref: '#/components/parameters/BucketParam' + - name: h5path + in: query + required: true + schema: { type: string } + - name: grpid + in: query + schema: { type: string } + description: Parent group id used as the base for `h5path` resolution. Defaults to the domain root. + - name: getalias + in: query + schema: { type: boolean, default: false } + - name: include_attrs + in: query + schema: { type: boolean, default: false } + responses: + '200': + description: Datatype JSON. + content: + application/json: + schema: { $ref: '#/components/schemas/DatatypeObject' } + '400': { $ref: '#/components/responses/BadRequest' } + '403': { $ref: '#/components/responses/Forbidden' } + '404': { $ref: '#/components/responses/NotFound' } + + /datatypes/{id}: + parameters: + - name: id + in: path + required: true + schema: { type: string } + - $ref: '#/components/parameters/DomainParam' + - $ref: '#/components/parameters/DomainHeader' + - $ref: '#/components/parameters/BucketParam' + get: + operationId: GetDatatype + tags: [Datatypes] + summary: Get a committed datatype + description: | + A datatype can also be located by path via `h5path`/`grpid` + instead of a path id, using the separate `GET /datatypes/` route - + `h5path`/`grpid` are read only by that route's handler branch and + have no effect if sent to this `{id}`-based route. + parameters: + - name: getalias + in: query + schema: { type: boolean, default: false } + description: | + Include an `alias` list of h5paths that resolve to this + datatype. Note: parsed with a raw truthiness check, so + `?getalias=0` is truthy and turns this on. + - name: include_attrs + in: query + schema: { type: boolean, default: false } + description: Same raw-truthiness caveat as `getalias` applies here too. + responses: + '200': + description: Datatype JSON. + content: + application/json: + schema: { $ref: '#/components/schemas/DatatypeObject' } + '400': { $ref: '#/components/responses/BadRequest' } + '403': { $ref: '#/components/responses/Forbidden' } + '404': { $ref: '#/components/responses/NotFound' } + '410': { $ref: '#/components/responses/Gone' } + delete: + operationId: DeleteDatatype + tags: [Datatypes] + summary: Delete a committed datatype + responses: + '200': + description: Datatype deleted. + content: + application/json: + schema: { type: object } + '400': { $ref: '#/components/responses/BadRequest' } + '403': { $ref: '#/components/responses/Forbidden' } + '404': { $ref: '#/components/responses/NotFound' } + '410': { $ref: '#/components/responses/Gone' } + + /datasets: + get: + operationId: GetDatasets + tags: [Datasets] + summary: List all dataset ids in a domain + parameters: + - $ref: '#/components/parameters/DomainParam' + - $ref: '#/components/parameters/DomainHeader' + - $ref: '#/components/parameters/BucketParam' + - $ref: '#/components/parameters/LimitParam' + - $ref: '#/components/parameters/MarkerParam' + responses: + '200': + description: List of dataset ids. + content: + application/json: + schema: + type: object + properties: + datasets: + type: array + items: { type: string } + hrefs: + type: array + items: { $ref: '#/components/schemas/Href' } + '400': { $ref: '#/components/responses/BadRequest' } + '403': { $ref: '#/components/responses/Forbidden' } + post: + operationId: PostDataset + tags: [Datasets] + summary: Create one or more datasets + description: | + `type` is required. `shape` defaults to scalar if omitted. If + `creationProperties.layout` is not given, HSDS auto-generates a + chunk layout from the shape/type/config chunk-size settings. + `creationProperties.filters` is restricted to HSDS's supported + filter set. An initial `value` may be supplied for creates whose + data maps to exactly one chunk - this constraint applies per + object and works the same for a single create or a batch + (multi-item list) create; it is not restricted to non-batch + creates. + + **Bug:** for a batch create (list with more than one item), + `implicit` is not forwarded to argument construction and always + behaves as `implicit=0` regardless of the query param, same as for + `POST /groups`. + parameters: + - $ref: '#/components/parameters/DomainParam' + - $ref: '#/components/parameters/DomainHeader' + - $ref: '#/components/parameters/BucketParam' + - name: implicit + in: query + schema: { type: boolean, default: false } + requestBody: + required: true + content: + application/json: + schema: + oneOf: + - $ref: '#/components/schemas/CreateDatasetRequest' + - type: array + items: { $ref: '#/components/schemas/CreateDatasetRequest' } + responses: + '201': + description: Dataset(s) created. + content: + application/json: + schema: + oneOf: + - $ref: '#/components/schemas/DatasetObject' + - type: object + properties: + objects: + type: array + items: { $ref: '#/components/schemas/DatasetObject' } + '400': { $ref: '#/components/responses/BadRequest' } + '403': { $ref: '#/components/responses/Forbidden' } + '409': { $ref: '#/components/responses/Conflict' } + '500': { $ref: '#/components/responses/InternalError' } + + /datasets/: + get: + operationId: GetDatasetByPath + tags: [Datasets] + summary: Resolve a dataset by h5path + description: Registered separately from `GET /datasets/{id}` (trailing slash, no id segment) for locating a dataset purely by `h5path`/`grpid` without already knowing its id. Otherwise identical to `GET /datasets/{id}`. + parameters: + - $ref: '#/components/parameters/DomainParam' + - $ref: '#/components/parameters/DomainHeader' + - $ref: '#/components/parameters/BucketParam' + - name: h5path + in: query + required: true + schema: { type: string } + - name: grpid + in: query + schema: { type: string } + description: Parent group id used as the base for `h5path` resolution. Defaults to the domain root. + - name: getalias + in: query + schema: { type: boolean, default: false } + description: | + Include an `alias` list of h5paths that resolve to this + dataset. Bug: on this specific route, the flag that gates + adding `alias` to the response is only ever set on the + path-`id`-based branch of the shared handler, which never + executes for this h5path-resolution route - so `getalias` has + no effect here (unlike on `GET /datasets/{id}`, where it works). + - name: include_attrs + in: query + schema: { type: boolean, default: false } + - name: verbose + in: query + schema: { type: boolean, default: false } + - name: ignore_nan + in: query + schema: { type: boolean, default: false } + responses: + '200': + description: Dataset JSON. + content: + application/json: + schema: { $ref: '#/components/schemas/DatasetObject' } + '400': { $ref: '#/components/responses/BadRequest' } + '403': { $ref: '#/components/responses/Forbidden' } + '404': { $ref: '#/components/responses/NotFound' } + + /datasets/{id}: + parameters: + - name: id + in: path + required: true + schema: { type: string } + - $ref: '#/components/parameters/DomainParam' + - $ref: '#/components/parameters/DomainHeader' + - $ref: '#/components/parameters/BucketParam' + get: + operationId: GetDataset + tags: [Datasets] + summary: Get dataset metadata + description: | + A dataset can also be located by path via `h5path`/`grpid` using + the separate `GET /datasets/` route - `h5path`/`grpid` have no + effect if sent to this `{id}`-based route (the path `id` is + always present here, so the h5path-resolution branch never runs). + parameters: + - name: getalias + in: query + schema: { type: boolean, default: false } + description: | + Include an `alias` list of h5paths that resolve to this + dataset. Note: parsed with a raw truthiness check, so + `?getalias=0` is truthy and turns this on. + - name: include_attrs + in: query + schema: { type: boolean, default: false } + - name: verbose + in: query + schema: { type: boolean, default: false } + description: Include num_chunks/allocated_size (schema2 domains only). + - name: ignore_nan + in: query + schema: { type: boolean, default: false } + responses: + '200': + description: Dataset JSON. + content: + application/json: + schema: { $ref: '#/components/schemas/DatasetObject' } + '400': { $ref: '#/components/responses/BadRequest' } + '403': { $ref: '#/components/responses/Forbidden' } + '404': { $ref: '#/components/responses/NotFound' } + delete: + operationId: DeleteDataset + tags: [Datasets] + summary: Delete a dataset and all its allocated chunks + responses: + '200': + description: Dataset deleted. + content: + application/json: + schema: { type: object } + '400': { $ref: '#/components/responses/BadRequest' } + '403': { $ref: '#/components/responses/Forbidden' } + '404': { $ref: '#/components/responses/NotFound' } + + /datasets/{id}/shape: + parameters: + - name: id + in: path + required: true + schema: { type: string } + - $ref: '#/components/parameters/DomainParam' + - $ref: '#/components/parameters/DomainHeader' + - $ref: '#/components/parameters/BucketParam' + get: + operationId: GetDatasetShape + tags: [Datasets] + summary: Get a dataset's shape + responses: + '200': + description: Dataset shape. + content: + application/json: + schema: + type: object + properties: + shape: { $ref: '#/components/schemas/ShapeSpec' } + created: { type: number } + lastModified: { type: number } + hrefs: + type: array + items: { $ref: '#/components/schemas/Href' } + '400': { $ref: '#/components/responses/BadRequest' } + '403': { $ref: '#/components/responses/Forbidden' } + '404': { $ref: '#/components/responses/NotFound' } + put: + operationId: PutDatasetShape + tags: [Datasets] + summary: Resize (extend or shrink) an extensible dataset + description: | + Provide exactly one of `shape` (new absolute dims) or `extend` + (grow by this amount along `extend_dim`, default axis 0). The + dataset must have simple, extensible shape (a `maxdims` key). + Growing past a finite `maxdims` bound returns 409. + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + shape: + description: New absolute dimensions (same rank as the current shape). + oneOf: + - type: integer + - type: array + items: { type: integer } + extend: + type: integer + description: Amount to grow the dataset by along extend_dim. Mutually exclusive with `shape`. Must be strictly positive - `extend<=0` 400s. + extend_dim: + type: integer + default: 0 + description: Axis to extend when using `extend`. + responses: + '201': + description: Shape updated. Body may include a `selection` describing the newly added region. + content: + application/json: + schema: + type: object + properties: + selection: + type: string + '400': { $ref: '#/components/responses/BadRequest' } + '403': { $ref: '#/components/responses/Forbidden' } + '404': { $ref: '#/components/responses/NotFound' } + '409': { $ref: '#/components/responses/Conflict' } + + /datasets/{id}/type: + parameters: + - name: id + in: path + required: true + schema: { type: string } + - $ref: '#/components/parameters/DomainParam' + - $ref: '#/components/parameters/DomainHeader' + - $ref: '#/components/parameters/BucketParam' + get: + operationId: GetDatasetType + tags: [Datasets] + summary: Get a dataset's datatype + responses: + '200': + description: Dataset type. + content: + application/json: + schema: + type: object + properties: + type: { $ref: '#/components/schemas/TypeSpec' } + hrefs: + type: array + items: { $ref: '#/components/schemas/Href' } + '400': { $ref: '#/components/responses/BadRequest' } + '403': { $ref: '#/components/responses/Forbidden' } + '404': { $ref: '#/components/responses/NotFound' } + + /datasets/{id}/value: + parameters: + - name: id + in: path + required: true + schema: { type: string } + - $ref: '#/components/parameters/DomainParam' + - $ref: '#/components/parameters/DomainHeader' + - $ref: '#/components/parameters/BucketParam' + get: + operationId: GetValue + tags: [Value] + summary: Read dataset data + description: | + Returns either a JSON `value` array, or (if `Accept: application/octet-stream` + is sent) raw binary data, laid out in row-major order for the + selected region. Large binary responses may be transparently + streamed/paginated by the server; this does not change the response + semantics from the client's point of view. + + ### Selection syntax (`select`) + `select=[,,...]`. Each dimension may be `:` (full + extent), `n` (single index), `n:m` or `n:m:s` (start[:stop[:step]]), + or a bracketed coordinate list `[c1,c2,...]`. + + **The enclosing brackets are NOT actually optional for a 1-D + select, despite older docs suggesting otherwise.** The parser + unconditionally strips the first and last character of the + `select` string as if they were `[`/`]`. A bracket-less multi-char + 1-D select like `select=0:5` is silently corrupted into `:` (i.e. + treated as the full extent) rather than rejected or honored as + `0:5`; a bracket-less single index like `select=3` does at least + fail with an error (the stripped string is empty). Always wrap + `select` in `[...]`, even for 1-D. + + ### Query syntax (`query`) + A SQL-like filter string, evaluated per-element (or per-row for + compound types): + `"_ > 1.0"`, `"symbol == b'AAPL'"`, + `"symbol IN ('AAPL','EBAY') AND date > 20170102"`. Operators: + `==`, `!=`, `<`, `>`, `<=`, `>=`, `IN`, combined with + `AND`/`OR`/`NOT`/`XOR`. For a simple (non-compound) dtype the only + allowed variable name is `_`; for compound types, any field name. + Supported on datasets of any rank (not just rank-1). + + ### Region reference selection (`regionref`) + Instead of spelling out `select`, a selection can be taken from a + previously stored HDF5 region reference (type `H5T_REFERENCE` / + base `H5T_STD_REF_DSETREG`). `regionref` names where that reference + is stored, as one of exactly these three forms (note: these use a + leading `/` and a plural `attributes` segment - unlike the bare + `"groups/"`/`"datasets/"` form used internally for + object-reference *values* elsewhere in the API): + - `/groups//attributes/` + - `/datasets//attributes/` + - `/datasets/` + + When `regionref` is given, `select` (if present) applies to *that + source*, not to this dataset - it picks out one element from an + array of region references (e.g. `select=[4]` for the 5th element). + If `select` is omitted, the source itself must resolve to exactly + one element (a scalar attribute, or a single-element array/dataset), + otherwise the request is rejected with 400. + + The resolved region reference's own selection is then applied + against *this* dataset (the `{id}` in the URL) - the dataset id + embedded in the reference itself is ignored. The reference's + selection rank must match this dataset's rank (400 otherwise), and + it must not select outside this dataset's actual extent (400 + otherwise) - but the shapes themselves don't otherwise need to + match, since a region reference's reconstructed shape is only a + synthetic bounding box, not the original dataset's true shape. + + If the resolved region reference is null/unbound, the response is + `204 No Content`. `regionref` cannot be combined with `query`. + parameters: + - name: select + in: query + schema: { type: string } + description: | + Hyperslab/coordinate selection string (see syntax above). + Defaults to the entire dataset. When `regionref` is given, + applies to the regionref source instead (see `regionref` + description). + - name: fields + in: query + schema: { type: string } + description: Colon-separated list of compound-type field names to include. + - name: query + in: query + schema: { type: string } + description: Query-filter string (see syntax above). + - name: Limit + in: query + schema: { type: integer } + description: Maximum number of rows to return when `query` is used. + - name: reduce_dim + in: query + schema: { type: boolean, default: false } + description: Drop length-1 dimensions from the returned JSON value. + - name: ignore_nan + in: query + schema: { type: boolean, default: false } + - name: regionref + in: query + schema: { type: string } + description: | + Take the selection from a stored region reference instead of + `select` (see syntax above). One of + `/groups//attributes/`, + `/datasets//attributes/`, or `/datasets/`. + responses: + '200': + description: Dataset data. + content: + application/json: + schema: + type: object + properties: + value: {} + hrefs: + type: array + items: { $ref: '#/components/schemas/Href' } + application/octet-stream: + schema: + type: string + format: binary + '204': + description: | + No Content. Returned only when `regionref` resolves to a + null/unbound region reference. + '400': { $ref: '#/components/responses/BadRequest' } + '403': { $ref: '#/components/responses/Forbidden' } + '404': + description: | + Not Found. Returned when `regionref` names an attribute that + doesn't exist. + content: + text/plain: + schema: { type: string, example: "404: Not Found" } + '413': + description: | + Payload Too Large. Note: for a JSON response, this check is + based on the size of the *unfiltered* `select` region, not the + size of the actual (potentially much smaller) query-matched + result - so a `query` over a large `select` region can 413 + even when the matched data itself would have been small. The + check is skipped only for binary (`application/octet-stream`) + responses with streaming enabled. + content: + text/plain: + schema: { type: string, example: "413: Request too large" } + '500': { $ref: '#/components/responses/InternalError' } + + put: + operationId: PutValue + tags: [Value] + summary: Write dataset data + description: | + Accepts either a JSON body (`Content-Type: application/json`) or raw + binary data (`Content-Type: application/octet-stream`). The JSON body + supports several mutually-exclusive write modes: whole/selected-region + write via `value`/`value_base64` (with `select`/`start`/`stop`/`step` + for the target region), point-selection write via `points`, or + append-mode write via `append`/`append_dim` (only for extensible + datasets, and not combinable with `select` or `points`). + + Binary writes larger than the server's `max_request_size` are + transparently paginated by the server (variable-length-typed data + cannot be paginated and returns 413 if oversized). + + ### Query-based update (`query`) + Instead of `value`/`points`/`append`, pass `query` (same filter + syntax as `GET`'s `query` param, see above) plus a JSON body + representing a single element's worth of data - that one value is + broadcast across every element that matches the query. Works on + datasets of any rank. + + - For a **compound** dtype, the body is a JSON object mapping field + name(s) to value(s), e.g. `{"open": 999}` - only the given + field(s) are updated on each matching row; fields not mentioned + are left unchanged. + - For a **simple** (non-compound) dtype, the body is just the bare + value to write (or `{"value": }`). + + The response is the list of **global dataset indices** that + matched (and were updated) - see the `200` response below - not + the row/element data itself. + + **Known bug:** when `Limit` is set and the matching rows span more + than one chunk, the limit is applied independently within each + chunk rather than coordinated across chunks - so more than `Limit` + rows total can end up updated. This is a known, deliberately + deferred issue. + + ### Region reference selection (`regionref`) + As with `GET`'s `regionref` param (see that operation's + description for the exact accepted path forms and the + `select`-repurposing semantics), the write selection can be taken + from a stored region reference instead of `select`/`start`/`stop`/ + `step`. Not combinable with `points`, `append`, or `query`. If the + resolved region reference's selection shape doesn't match the shape + of the data being written, or if it resolves to a null/unbound + region reference, the request is rejected with 400. + parameters: + - name: select + in: query + schema: { type: string } + - name: regionref + in: query + schema: { type: string } + description: | + Take the write selection from a stored region reference + instead of `select` (see this operation's description, and + `GET`'s `regionref` param, for the exact syntax). + - name: fields + in: query + schema: { type: string } + - name: append + in: query + schema: { type: integer } + description: Number of rows to append (append mode). + - name: append_dim + in: query + schema: { type: integer, default: 0 } + - name: query + in: query + schema: { type: string } + description: | + Query-based update filter (see this operation's description). + Cannot be combined with `append`. + - name: Limit + in: query + schema: { type: integer } + description: | + Maximum number of matching rows to update, when `query` is + used. Known bug: not correctly coordinated across chunks when + matches span more than one chunk - see this operation's + description. + - name: element_count + in: query + schema: { type: integer } + description: Number of elements represented by the request body, for broadcast writes. + requestBody: + required: true + content: + application/json: + schema: + type: object + description: | + This shape applies to the regular write modes + (value/value_base64/points/append). When `query` is used + (either as a query param, or as the `query` property here), + the body is NOT this envelope - it is instead a single + element's worth of update data: a JSON object of field + name to value for a compound dtype, or the bare value (or + `{"value": }`) for a simple dtype. See this + operation's description. + properties: + value: {} + value_base64: + type: string + description: Base64-encoded data, alternative to `value` for binary-safe JSON transport. + start: + oneOf: [{ type: integer }, { type: array, items: { type: integer } }] + stop: + oneOf: [{ type: integer }, { type: array, items: { type: integer } }] + step: + oneOf: [{ type: integer }, { type: array, items: { type: integer } }] + select: + type: string + points: + type: array + description: List of point indices (rank 1) or coordinate tuples (rank > 1) to write. + items: {} + query: + type: string + description: Alternative to the `query` query-parameter - specifies the filter string via the body instead. + append: + type: integer + append_dim: + type: integer + element_count: + type: integer + fields: + type: string + application/octet-stream: + schema: + type: string + format: binary + responses: + '200': + description: | + Write successful (empty body), or - in query-update mode - the + list of global dataset indices that matched and were updated + (see the `indices` property below). + content: + application/json: + schema: + type: object + properties: + indices: + type: array + description: | + Only present in query-update mode. List of global + dataset index coordinate tuples that matched (and + were updated) - e.g. `[[1], [4], [7], [10]]` for a + rank-1 dataset, or `[[2, 3], [5, 1]]` for rank-2. + items: + type: array + items: { type: integer } + hrefs: + type: array + items: { $ref: '#/components/schemas/Href' } + application/octet-stream: + schema: + type: string + format: binary + description: "Query-update mode with `Accept: application/octet-stream` - the same indices, as raw little-endian int64 coordinate tuples." + '400': { $ref: '#/components/responses/BadRequest' } + '403': { $ref: '#/components/responses/Forbidden' } + '409': { $ref: '#/components/responses/Conflict' } + '413': { $ref: '#/components/responses/TooLarge' } + '500': { $ref: '#/components/responses/InternalError' } + + post: + operationId: PostValue + tags: [Value] + summary: Read dataset data via point selection (or a selection given in the body) + description: | + Accepts a JSON body with either a `points` array (point selection - + a flat list of indices for rank-1 datasets, or a list of coordinate + tuples for higher rank), or `select`/`start`/`stop`/`step` keys + (equivalent to GET's `select` query param, but supplied in the body). + Exactly one of these forms must be present. A binary body + (`Content-Type: application/octet-stream`) is also accepted for + point selection only, as a flat array of unsigned 64-bit point + indices/coordinates. + + Not supported for null-space or scalar-space datasets. Unlike GET, + POST has no pagination fallback: a response whose size would exceed + `max_request_size` fails with 413 rather than streaming. + parameters: + - name: select + in: query + schema: { type: string } + - name: fields + in: query + schema: { type: string } + - name: ignore_nan + in: query + schema: { type: boolean, default: false } + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + points: + type: array + items: {} + description: Flat list of indices (rank 1) or list of coordinate tuples (rank > 1). + select: + type: string + start: { oneOf: [{ type: integer }, { type: array, items: { type: integer } }] } + stop: { oneOf: [{ type: integer }, { type: array, items: { type: integer } }] } + step: { oneOf: [{ type: integer }, { type: array, items: { type: integer } }] } + application/octet-stream: + schema: + type: string + format: binary + description: Flat array of unsigned 64-bit point indices/coordinates. + responses: + '200': + description: Selected dataset data. + content: + application/json: + schema: + type: object + properties: + value: {} + hrefs: + type: array + items: { $ref: '#/components/schemas/Href' } + application/octet-stream: + schema: + type: string + format: binary + '400': { $ref: '#/components/responses/BadRequest' } + '403': { $ref: '#/components/responses/Forbidden' } + '413': { $ref: '#/components/responses/TooLarge' } + + /datasets/{id}/query: + parameters: + - name: id + in: path + required: true + schema: { type: string } + - $ref: '#/components/parameters/DomainParam' + - $ref: '#/components/parameters/DomainHeader' + - $ref: '#/components/parameters/BucketParam' + get: + operationId: GetQuery + tags: [Value] + summary: Get the indices of dataset elements matching a query + description: | + Works like `GET /datasets/{id}/value` with a `query` parameter, + except `query` is required here (400 if omitted), and the response + is the **global dataset indices** of the matching elements, not + their values - the same `indices` response format used by + `PUT /datasets/{id}/value`'s query-based update mode. Supported on + datasets of any rank. Nothing is modified by this operation. + + **Known bug:** when `Limit` is set and the matching rows span more + than one chunk, the limit is applied independently within each + chunk rather than coordinated across chunks - so more than `Limit` + indices total can be returned. This is a known, deliberately + deferred issue (same as `PUT /datasets/{id}/value`'s query mode). + parameters: + - name: select + in: query + schema: { type: string } + description: Hyperslab/coordinate selection string, restricting which elements are considered. Defaults to the entire dataset. + - name: fields + in: query + schema: { type: string } + description: Colon-separated list of compound-type field names to restrict the query to. + - name: query + in: query + required: true + schema: { type: string } + description: Query-filter string (same syntax as `GET /datasets/{id}/value`'s `query` param). + - name: Limit + in: query + schema: { type: integer } + description: Maximum number of matching indices to return. See the known bug noted above for multi-chunk datasets. + responses: + '200': + description: Matching indices. + content: + application/json: + schema: + type: object + properties: + indices: + type: array + description: | + List of global dataset index coordinate tuples that + matched - e.g. `[[1], [4], [7], [10]]` for a rank-1 + dataset, or `[[2, 3], [5, 1]]` for rank-2. + items: + type: array + items: { type: integer } + hrefs: + type: array + items: { $ref: '#/components/schemas/Href' } + application/octet-stream: + schema: + type: string + format: binary + description: The same indices, as raw little-endian int64 coordinate tuples. + '400': { $ref: '#/components/responses/BadRequest' } + '403': { $ref: '#/components/responses/Forbidden' } + '404': { $ref: '#/components/responses/NotFound' } + '413': + description: | + Payload Too Large. Based on the size of the *unfiltered* + `select` region, not the size of the actual (potentially much + smaller) query-matched index list - same caveat as + `GET /datasets/{id}/value`'s query mode. + content: + text/plain: + schema: { type: string, example: "413: Request too large" } + '500': { $ref: '#/components/responses/InternalError' } + + # Attribute endpoints are identical (modulo the owning collection name) + # across groups, datasets, and datatypes, so they are factored into + # reusable Path Item Objects under components.pathItems and referenced + # here via $ref (OpenAPI 3.1 Paths Object entries may be References). + /groups/{id}/attributes: + $ref: '#/components/pathItems/AttributesCollection' + /datasets/{id}/attributes: + $ref: '#/components/pathItems/AttributesCollection' + /datatypes/{id}/attributes: + $ref: '#/components/pathItems/AttributesCollection' + + /groups/{id}/attributes/{name}: + $ref: '#/components/pathItems/AttributeItem' + /datasets/{id}/attributes/{name}: + $ref: '#/components/pathItems/AttributeItem' + /datatypes/{id}/attributes/{name}: + $ref: '#/components/pathItems/AttributeItem' + + /groups/{id}/attributes/{name}/value: + $ref: '#/components/pathItems/AttributeValue' + /datasets/{id}/attributes/{name}/value: + $ref: '#/components/pathItems/AttributeValue' + /datatypes/{id}/attributes/{name}/value: + $ref: '#/components/pathItems/AttributeValue' + +components: + securitySchemes: + basicAuth: + type: http + scheme: basic + description: | + Username/password Basic auth. May be omitted entirely on read (GET) + requests if the server is configured with `allow_noauth: true`, in + which case the request is treated as user `default`. + + parameters: + DomainParam: + name: domain + in: query + description: Domain to operate on. Alternative to the `X-Hdf-domain` header; one of the two is required for every operation except `GET /domains`. + schema: { type: string } + DomainHeader: + name: X-Hdf-domain + in: header + description: Domain to operate on. Alternative to the `domain` query parameter. + schema: { type: string } + BucketParam: + name: bucket + in: query + description: Storage bucket to use, if not the server's configured default bucket. + schema: { type: string } + LimitParam: + name: Limit + in: query + description: Maximum number of items to return in one page. + schema: { type: integer } + MarkerParam: + name: Marker + in: query + description: Item id (or, for domain listings, name) to start the next page after, for pagination. + schema: { type: string } + + responses: + BadRequest: + description: | + Bad Request - malformed request syntax, invalid parameter value, + invalid JSON body, or a request that is otherwise semantically + invalid (e.g. an unrecognized query filter). + content: + text/plain: + schema: { type: string, example: "400: Invalid domain name" } + Unauthorized: + description: Unauthorized - authentication is required and was not supplied or was invalid. + content: + text/plain: + schema: { type: string, example: "401: Unauthorized" } + Forbidden: + description: Forbidden - the authenticated (or anonymous) user does not have the required ACL permission for this operation. + content: + text/plain: + schema: { type: string, example: "403: Forbidden" } + NotFound: + description: Not Found - the domain, object, link, or attribute does not exist. + content: + text/plain: + schema: { type: string, example: "404: Not Found" } + Conflict: + description: Conflict - the object/link/domain already exists, or the request conflicts with the current state (e.g. resizing past maxdims, deleting a non-empty folder). + content: + text/plain: + schema: { type: string, example: "409: Conflict" } + Gone: + description: Gone - the domain/object existed previously but has been deleted. + content: + text/plain: + schema: { type: string, example: "410: Gone" } + TooLarge: + description: Payload Too Large - the request or response body would exceed the server's configured `max_request_size`. + content: + text/plain: + schema: { type: string, example: "413: Request too large" } + InternalError: + description: Internal Server Error. + content: + text/plain: + schema: { type: string, example: "500: Internal Server Error" } + ServiceUnavailable: + description: Service Unavailable - the server is too busy (task queue full) or a required back-end node is not responding. + content: + text/plain: + schema: { type: string, example: "503: Service Unavailable" } + + schemas: + Href: + type: object + properties: + rel: { type: string, example: self } + href: { type: string, format: uri } + + HrefList: + type: array + items: { $ref: '#/components/schemas/Href' } + + ACL: + type: object + description: Permission bits for one user (or the special `default` user) on a domain. + properties: + userName: { type: string } + create: { type: boolean } + read: { type: boolean } + update: { type: boolean } + delete: { type: boolean } + readACL: { type: boolean } + updateACL: { type: boolean } + + DomainResponse: + type: object + description: | + Note: despite `GET /acls`/`GET /acls/{username}` existing as + separate endpoints, this object never actually includes an `acls` + map of its own - the domain's ACLs are only obtainable through + those dedicated endpoints. + properties: + root: { type: string, description: Id of the domain's root group. } + owner: { type: string } + class: + type: string + enum: [domain, folder] + description: Not populated in the `201` response body returned by `PUT /` (a known gap) - only present when subsequently fetched via `GET /`. + created: { type: number } + lastModified: { type: number } + num_groups: { type: integer } + num_datasets: { type: integer } + num_datatypes: { type: integer } + num_objects: { type: integer } + num_chunks: { type: integer } + num_linked_chunks: { type: integer } + total_size: { type: integer } + allocated_bytes: { type: integer } + metadata_bytes: { type: integer } + linked_bytes: { type: integer } + md5_sum: { type: string } + scan_info: + type: object + description: Details of the most recent background rescan of the domain's root group. + limits: + type: object + description: Server-configured limits relevant to this domain (e.g. max_request_size, max_chunks_per_request). + compressors: + type: array + items: { type: string } + description: Compression filters available on this server. + version: { type: string } + domain_objs: + type: object + description: Present only when `getobjs=1` was requested. Map of object id to that object's JSON. + dn_ids: + type: array + description: Present only when `getdnids=1` was requested. + items: { type: string } + hrefs: { $ref: '#/components/schemas/HrefList' } + + DomainList: + type: object + properties: + domains: + type: array + items: + type: object + properties: + name: { type: string } + owner: { type: string } + class: { type: string, enum: [domain, folder] } + created: { type: number } + lastModified: { type: number } + hrefs: { $ref: '#/components/schemas/HrefList' } + + ObjectResponse: + description: Group, Dataset, or Datatype JSON, as resolved from an h5path. + oneOf: + - $ref: '#/components/schemas/GroupObject' + - $ref: '#/components/schemas/DatasetObject' + - $ref: '#/components/schemas/DatatypeObject' + + Link: + type: object + properties: + title: { type: string } + class: + type: string + enum: [H5L_TYPE_HARD, H5L_TYPE_SOFT, H5L_TYPE_EXTERNAL, H5L_TYPE_ERROR] + collection: + type: string + enum: [groups, datasets, datatypes] + description: Present for hard links - the collection the target object belongs to. + id: + type: string + description: Present for hard links - the target object's id. + h5path: + type: string + description: Present for soft/external links - the path the link resolves to. + file: + type: string + description: | + Present for external links - the domain the link points into. + Note: the wire/response key is `file`, not `h5domain` - `h5domain` + is only accepted as a deprecated fallback name by the underlying + h5json link-path helper and is never what a real response contains. + created: { type: number } + + LinkCreate: + type: object + description: Exactly one of `id`, `h5path` (soft link), or `h5path`+`file` (external link) must be given. + properties: + id: + type: string + description: Target object id, for a hard link. + h5path: + type: string + description: Target path, for a soft link (or, combined with `file`, an external link). + file: + type: string + description: Target domain, for an external link. + + TypeSpec: + description: | + HDF5 datatype, expressed as either a predefined type string, a + committed-type reference (an object id beginning with `t-`), or a + full type-description object. + oneOf: + - type: string + description: Predefined type string, e.g. H5T_STD_I32LE, H5T_IEEE_F64LE, H5T_STD_U8LE. + example: H5T_STD_I32LE + - type: string + description: Reference to an existing committed datatype, by object id. + example: t-c9648b48-13e2-11e7-9beb-0242ac110008 + - type: object + required: [class] + properties: + class: + type: string + enum: + - H5T_INTEGER + - H5T_FLOAT + - H5T_STRING + - H5T_VLEN + - H5T_OPAQUE + - H5T_ARRAY + - H5T_REFERENCE + - H5T_ENUM + - H5T_COMPOUND + base: + description: Base type for H5T_VLEN, H5T_ARRAY, and H5T_ENUM classes. + $ref: '#/components/schemas/TypeSpec' + dims: + description: Array dimensions, for H5T_ARRAY. + oneOf: + - type: integer + - type: array + items: { type: integer } + length: + description: Fixed length in bytes (H5T_STRING, H5T_OPAQUE), or the string "H5T_VARIABLE" for variable-length strings/vlen. + oneOf: + - type: integer + - type: string + charSet: + type: string + enum: [H5T_CSET_ASCII, H5T_CSET_UTF8] + strPad: + type: string + enum: [H5T_STR_NULLTERM, H5T_STR_NULLPAD, H5T_STR_SPACEPAD] + size: + description: Element size in bytes, for H5T_REFERENCE. + type: integer + fields: + description: Member fields, for H5T_COMPOUND (order is significant). + type: array + items: + type: object + required: [name, type] + properties: + name: { type: string } + type: { $ref: '#/components/schemas/TypeSpec' } + mapping: + description: Name-to-value map, for H5T_ENUM. + type: object + additionalProperties: { type: integer } + + ShapeSpec: + type: object + properties: + class: + type: string + enum: [H5S_NULL, H5S_SCALAR, H5S_SIMPLE] + dims: + description: Current extent per dimension. Omitted for H5S_NULL/H5S_SCALAR. + oneOf: + - type: integer + - type: array + items: { type: integer } + maxdims: + description: Maximum extent per dimension (a dim value of 0 means unlimited). Only present if the dataset is extensible. + oneOf: + - type: integer + - type: array + items: { type: integer } + + CreationProperties: + type: object + description: HDF5 dataset creation properties. + properties: + layout: + type: object + properties: + class: + type: string + enum: [H5D_CHUNKED, H5D_CONTIGUOUS, H5D_COMPACT] + dims: + type: array + items: { type: integer } + description: Chunk dimensions, for H5D_CHUNKED. + fillValue: {} + allocTime: + type: string + enum: [H5D_ALLOC_TIME_DEFAULT, H5D_ALLOC_TIME_EARLY, H5D_ALLOC_TIME_INCR, H5D_ALLOC_TIME_LATE] + filters: + type: array + description: Chunk compression/transform filters, restricted to HSDS's supported filter set (e.g. gzip/deflate, shuffle, szip, bitshuffle, blosc-family filters). + items: + type: object + properties: + class: { type: string } + id: { type: integer } + name: { type: string } + level: { type: integer } + + CreateObjectRequest: + type: object + description: Body for creating a group or committed datatype (and the common fields shared with dataset creation). + properties: + h5path: + type: string + description: Path (relative to parent_id, or the domain root) at which to link the new object. Missing intermediate groups require `implicit=1`. + parent_id: + type: string + description: Base group id that h5path is resolved relative to. Defaults to the domain root. + id: + type: string + description: Client-supplied object id (schema2 uuid) to use for the new object, instead of a server-generated one. + link: + type: object + description: Alternative to `h5path`/`parent_id` for placing a hard link to the new object at creation time. + properties: + id: + type: string + description: Group id to link the new object into. + name: + type: string + description: Link title to use. + attributes: + type: object + description: Attributes to create on the new object in the same request, keyed by name. + additionalProperties: { $ref: '#/components/schemas/Attribute' } + links: + type: object + description: Links to create on the new object (if it's a group) in the same request, keyed by title. + additionalProperties: { $ref: '#/components/schemas/LinkCreate' } + type: + description: Required for datatype creation; omitted for groups. + $ref: '#/components/schemas/TypeSpec' + creationProperties: { $ref: '#/components/schemas/CreationProperties' } + + CreateDatasetRequest: + allOf: + - $ref: '#/components/schemas/CreateObjectRequest' + - type: object + required: [type] + properties: + shape: + description: Defaults to scalar (H5S_SCALAR) if omitted. + $ref: '#/components/schemas/ShapeSpec' + value: + description: Initial data. The object's extent must map to exactly one chunk; this applies per-object and works the same whether this is a single create or one item of a batch (list) create. + + Attribute: + type: object + required: [type] + description: | + Used both as the create request body (only `type`/`shape`/`value` + are meaningful there) and as a response shape, which varies by + context: attribute-listing responses (`GET`/`POST` on the + collection) include a single `href` string per item; the + single-attribute `GET .../attributes/{name}` response instead + includes `lastModified` (always equal to `created` - attributes + aren't otherwise versioned) and an `hrefs` array, and may include + `encoding` if the value was base64-encoded. + properties: + name: { type: string } + type: { $ref: '#/components/schemas/TypeSpec' } + shape: { $ref: '#/components/schemas/ShapeSpec' } + value: {} + created: { type: number } + lastModified: + type: number + description: Only present on the single-attribute GET response. Always equal to `created`. + encoding: + type: string + enum: [base64] + description: Only present if the value was returned base64-encoded. + href: + type: string + description: Only present as a list item within collection-level attribute responses. + hrefs: + description: Only present on the single-attribute GET response (in place of `href`). + $ref: '#/components/schemas/HrefList' + + GroupObject: + type: object + properties: + id: { type: string } + root: { type: string } + domain: { type: string } + created: { type: number } + lastModified: { type: number } + linkCount: { type: integer } + attributeCount: { type: integer } + alias: + type: array + items: { type: string } + description: Present only when `getalias=1` was requested. + links: + type: array + description: Present only when `include_links=1` was requested. + items: { $ref: '#/components/schemas/Link' } + attributes: + type: array + description: Present only when `include_attrs=1` was requested. + items: { $ref: '#/components/schemas/Attribute' } + hrefs: { $ref: '#/components/schemas/HrefList' } + + DatatypeObject: + type: object + properties: + id: { type: string } + root: { type: string } + domain: { type: string } + type: { $ref: '#/components/schemas/TypeSpec' } + created: { type: number } + lastModified: { type: number } + attributeCount: { type: integer } + alias: + type: array + items: { type: string } + attributes: + type: array + items: { $ref: '#/components/schemas/Attribute' } + hrefs: { $ref: '#/components/schemas/HrefList' } + + DatasetObject: + type: object + properties: + id: { type: string } + root: { type: string } + domain: { type: string } + type: { $ref: '#/components/schemas/TypeSpec' } + shape: { $ref: '#/components/schemas/ShapeSpec' } + creationProperties: { $ref: '#/components/schemas/CreationProperties' } + layout: + type: object + description: Normally folded into `creationProperties.layout` instead. Can appear as a separate top-level key in deployments without a DN tier, where the raw stored object is returned as-is. + created: { type: number } + lastModified: { type: number } + attributeCount: { type: integer } + num_chunks: + type: integer + description: Present only when `verbose=1` is requested on a schema2 domain. + allocated_size: + type: integer + description: Present only when `verbose=1` is requested on a schema2 domain. + alias: + type: array + items: { type: string } + attributes: + type: array + items: { $ref: '#/components/schemas/Attribute' } + hrefs: { $ref: '#/components/schemas/HrefList' } + + pathItems: + AttributesCollection: + parameters: + - name: id + in: path + required: true + schema: { type: string } + - $ref: '#/components/parameters/DomainParam' + - $ref: '#/components/parameters/DomainHeader' + - $ref: '#/components/parameters/BucketParam' + get: + tags: [Attributes] + summary: List attributes on a group/dataset/datatype + parameters: + - name: IncludeData + in: query + schema: { type: boolean, default: false } + description: Include each attribute's `value` in the listing (by default only metadata is returned). + - name: pattern + in: query + schema: { type: string } + description: Glob pattern filter on attribute name. + - name: encoding + in: query + schema: { type: string, enum: [base64] } + description: Encode binary attribute values as base64 in the JSON response. + - name: ignore_nan + in: query + schema: { type: boolean, default: false } + - name: follow_links + in: query + schema: { type: boolean, default: false } + description: | + Recursively list attributes of sub-groups too. Changes the + response shape: `attributes` becomes a map of object id to + either that object's attribute list, or `{"status": }` + if that particular object couldn't be crawled. + - name: max_data_size + in: query + schema: { type: integer } + description: Cap on the size of attribute data included with `IncludeData`. + - name: CreateOrder + in: query + schema: { type: boolean, default: false } + description: Return attributes in creation order instead of name order. + - $ref: '#/components/parameters/LimitParam' + - $ref: '#/components/parameters/MarkerParam' + responses: + '200': + description: | + List of attributes (or, with `follow_links`, a map of object + id to attribute list/status - see the `follow_links` + parameter). + content: + application/json: + schema: + type: object + properties: + attributes: + oneOf: + - type: array + items: { $ref: '#/components/schemas/Attribute' } + - type: object + additionalProperties: {} + hrefs: { $ref: '#/components/schemas/HrefList' } + '400': { $ref: '#/components/responses/BadRequest' } + '403': { $ref: '#/components/responses/Forbidden' } + '404': { $ref: '#/components/responses/NotFound' } + post: + tags: [Attributes] + summary: Batch-fetch specific named attributes + description: | + Despite the POST verb this is a read operation, analogous to + `POST .../links` - it lets a client fetch a specific set of + attributes by name in one round trip instead of listing all of + them, optionally across multiple objects via `obj_ids`. + parameters: + - name: IncludeData + in: query + schema: { type: boolean, default: true } + - name: encoding + in: query + schema: { type: string, enum: [base64] } + - name: max_data_size + in: query + schema: { type: integer } + - name: ignore_nan + in: query + schema: { type: boolean, default: false } + description: Passed as a query parameter, not a body property (despite the `ignore_nan` body key some older references show). + requestBody: + required: true + content: + application/json: + schema: + type: object + required: [attr_names] + properties: + attr_names: + type: array + items: { type: string } + description: Attribute names to fetch. Note the body key is `attr_names`, not `attributes`. + obj_ids: + description: Fetch from multiple objects in one call - either a list of object ids (same `attr_names` for each), or a map of object id to a list of names for that object. + oneOf: + - type: array + items: { type: string } + - type: object + additionalProperties: + type: array + items: { type: string } + responses: + '200': + description: | + The requested attributes (names not present on the object are + simply omitted). When `obj_ids` targets more than one object, + `attributes` is instead a map of object id to that object's + attribute list. + content: + application/json: + schema: + type: object + properties: + attributes: + oneOf: + - type: array + items: { $ref: '#/components/schemas/Attribute' } + - type: object + additionalProperties: + type: array + items: { $ref: '#/components/schemas/Attribute' } + hrefs: { $ref: '#/components/schemas/HrefList' } + '400': { $ref: '#/components/responses/BadRequest' } + '403': { $ref: '#/components/responses/Forbidden' } + put: + tags: [Attributes] + summary: Batch-create attributes + parameters: + - name: replace + in: query + schema: { type: boolean, default: false } + description: Allow overwriting attributes that already exist under the same name (see the `200`-vs-`201`/`409` note below). + requestBody: + required: true + content: + application/json: + schema: + type: object + required: [attributes] + properties: + attributes: + type: object + additionalProperties: { $ref: '#/components/schemas/Attribute' } + description: Map of attribute name to its type/shape/value definition. + obj_ids: + description: Create the same `attributes` map on multiple objects in one call - either a list of object ids, or a map of object id to a per-object attributes map. + oneOf: + - type: array + items: { type: string } + - type: object + additionalProperties: + type: object + properties: + attributes: + type: object + additionalProperties: { $ref: '#/components/schemas/Attribute' } + responses: + '200': + description: Every submitted attribute was byte-for-byte identical to an attribute that already existed under that name, so nothing was written. + content: + application/json: + schema: { type: object } + '201': + description: At least one attribute was newly created (or, with `replace=1`, overwritten). + content: + application/json: + schema: { type: object } + '400': { $ref: '#/components/responses/BadRequest' } + '403': { $ref: '#/components/responses/Forbidden' } + '409': + description: An attribute name already exists with different content and `replace` was not set. + content: + text/plain: + schema: { type: string, example: "409: Conflict" } + delete: + tags: [Attributes] + summary: Batch-delete attributes by name + parameters: + - name: attr_names + in: query + required: true + schema: { type: string } + description: Attribute names to delete, joined by `separator`. + - name: separator + in: query + schema: { type: string, default: "/" } + description: Delimiter used to split `attr_names` into individual attribute names. + - name: encoding + in: query + schema: { type: string, enum: [base64] } + description: Decode `attr_names` as base64 first, for names containing characters that can't otherwise be encoded in the delimited string. + responses: + '200': + description: Attribute(s) deleted. + content: + application/json: + schema: { type: object } + '400': { $ref: '#/components/responses/BadRequest' } + '403': { $ref: '#/components/responses/Forbidden' } + '404': + description: | + At least one requested name does not currently exist on the + object. Unlike some other batch-delete operations in this + API, deleting a name that was never present is NOT a no-op + here - it 404s. (Re-deleting a name already deleted earlier + in the same server session is a no-op, since the DN tracks + recently-deleted names in memory - but a name that was + simply never created 404s.) + content: + text/plain: + schema: { type: string, example: "404: Not Found" } + + AttributeItem: + parameters: + - name: id + in: path + required: true + schema: { type: string } + - name: name + in: path + required: true + schema: { type: string } + - $ref: '#/components/parameters/DomainParam' + - $ref: '#/components/parameters/DomainHeader' + - $ref: '#/components/parameters/BucketParam' + get: + tags: [Attributes] + summary: Get a single attribute (metadata and value) + parameters: + - name: IncludeData + in: query + schema: { type: boolean, default: true } + description: Include the attribute's `value`. Defaults to true here (unlike the collection GET, which defaults it to false) - set to false to fetch only metadata. + - name: encoding + in: query + schema: { type: string, enum: [base64] } + - name: ignore_nan + in: query + schema: { type: boolean, default: false } + responses: + '200': + description: Attribute. + content: + application/json: + schema: { $ref: '#/components/schemas/Attribute' } + '400': { $ref: '#/components/responses/BadRequest' } + '403': { $ref: '#/components/responses/Forbidden' } + '404': { $ref: '#/components/responses/NotFound' } + put: + tags: [Attributes] + summary: Create (or, with `replace=1`, overwrite) a single attribute + description: | + By default this only creates a new attribute (409 if the name + already exists with different content). Passing `?replace=1` + allows overwriting an existing attribute of the same name - + attributes are not strictly immutable, contrary to older + documentation of this endpoint. + parameters: + - name: replace + in: query + schema: { type: boolean, default: false } + description: Allow overwriting an existing attribute of this name. + requestBody: + required: true + content: + application/json: + schema: { $ref: '#/components/schemas/Attribute' } + responses: + '200': + description: The submitted attribute was byte-for-byte identical to the attribute that already existed under this name, so nothing was written. + content: + application/json: + schema: { type: object } + '201': + description: Attribute created (or, with `replace=1`, overwritten with different content). + content: + application/json: + schema: { type: object } + '400': { $ref: '#/components/responses/BadRequest' } + '403': { $ref: '#/components/responses/Forbidden' } + '409': { $ref: '#/components/responses/Conflict' } + delete: + tags: [Attributes] + summary: Delete a single attribute + responses: + '200': + description: Attribute deleted. + content: + application/json: + schema: { type: object } + '400': { $ref: '#/components/responses/BadRequest' } + '403': { $ref: '#/components/responses/Forbidden' } + '404': { $ref: '#/components/responses/NotFound' } + + AttributeValue: + parameters: + - name: id + in: path + required: true + schema: { type: string } + - name: name + in: path + required: true + schema: { type: string } + - $ref: '#/components/parameters/DomainParam' + - $ref: '#/components/parameters/DomainHeader' + - $ref: '#/components/parameters/BucketParam' + get: + tags: [Attributes] + summary: Get just the value of an attribute + description: | + Content negotiation is supported: send `Accept: application/octet-stream` + to get the raw value as binary instead of the default JSON envelope. + parameters: + - name: encoding + in: query + schema: { type: string, enum: [base64] } + - name: ignore_nan + in: query + schema: { type: boolean, default: false } + responses: + '200': + description: Attribute value. + content: + application/json: + schema: + type: object + properties: + value: {} + hrefs: { $ref: '#/components/schemas/HrefList' } + application/octet-stream: + schema: { type: string, format: binary } + '400': { $ref: '#/components/responses/BadRequest' } + '403': { $ref: '#/components/responses/Forbidden' } + '404': { $ref: '#/components/responses/NotFound' } + put: + tags: [Attributes] + summary: Update the value of an existing attribute + description: | + **This is the opposite of create** despite the resource's PUT + verb elsewhere in this API usually meaning "create": the + attribute must already exist - this 404s if it does not - and + the value is always overwritten in place (unconditionally, no + `replace` param needed or accepted). + + The `type`/`shape` fields are NOT accepted from the request body + even though they may appear in older references - the existing + attribute's stored type/shape are reused, and anything sent for + `type`/`shape` here is silently ignored. Only `value` (or + `value`+`encoding: base64`) has any effect. + + Accepts either a JSON body or, with + `Content-Type: application/octet-stream`, a raw binary value. + + The response status is always `200`, even when the DN's + underlying write actually changed the stored value (which would + otherwise be `201`) - the real status is computed but discarded + before the SN's response is sent, so callers cannot distinguish + "value changed" from "value was already identical" via status + code here. + requestBody: + required: true + content: + application/json: + schema: + type: object + required: [value] + properties: + value: {} + encoding: + type: string + enum: [base64] + application/octet-stream: + schema: { type: string, format: binary } + responses: + '200': + description: Value updated (this status is returned unconditionally on success - see description). + content: + application/json: + schema: { type: object } + '400': { $ref: '#/components/responses/BadRequest' } + '403': { $ref: '#/components/responses/Forbidden' } + '404': + description: The attribute does not already exist. Unlike `PUT .../attributes/{name}`, this endpoint never creates a new attribute. + content: + text/plain: + schema: { type: string, example: "404: Not Found" } diff --git a/pyproject.toml b/pyproject.toml index af575c13..1a331884 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [build-system] requires = [ - "setuptools >=61", + "setuptools >=77", ] build-backend = "setuptools.build_meta" @@ -13,13 +13,12 @@ authors = [ maintainers = [ {name = "John Readey", email = "jreadey@hdfgroup.org"}, ] -license = {text = "Apache Software License"} +license = "Apache-2.0" classifiers = [ "Development Status :: 5 - Production/Stable", "Intended Audience :: Developers", "Intended Audience :: Information Technology", "Intended Audience :: Science/Research", - "License :: OSI Approved :: Apache Software License", "Operating System :: Unix", "Operating System :: POSIX :: Linux", "Operating System :: MacOS :: MacOS X", @@ -30,8 +29,8 @@ classifiers = [ "Topic :: Database", "Topic :: Software Development :: Libraries :: Python Modules", ] -requires-python = ">=3.8" -version = "0.9.2" +requires-python = ">=3.11" +version = "1.0.0" dependencies = [ "aiohttp == 3.9.4", @@ -42,9 +41,10 @@ dependencies = [ "bitshuffle >=0.5.2", "cryptography", "h5py >= 3.6.0", + "h5json@git+https://github.com/HDFGroup/hdf5-json@master", "importlib_resources", - "numcodecs", - "numpy >=2.0.0rc1; python_version>='3.9'", + "numcodecs <= 0.15.1", + "numpy >=2.0.0", "psutil", "pyjwt", "pytz", @@ -66,7 +66,7 @@ Data can be stored in either a POSIX files system, or using object-based storage AWS S3, Azure Blob Storage, or [MinIO](https://min.io). HSDS can be run a single machine or on a cluster using Kubernetes (or AKS on Microsoft Azure). -In addition, HSDS can b e run in serverless mode with AWS Lambda or h5pyd local mode. +In addition, HSDS can be run in serverless mode using h5pyd local mode. """ content-type = "text/x-rst" diff --git a/requirements.txt b/requirements.txt index 5aa9d39b..768d1a8d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,18 +1,54 @@ -aiohttp==3.9.4 aiobotocore==2.13.0 -aiohttp_cors -aiofiles -azure-storage-blob -cryptography -h5py>=3.6.0 -numcodecs -numpy>=2.0.0rc1 -psutil -pyjwt -pytz -pyyaml -requests<=2.31.0 -requests-unixsocket -simplejson -s3fs -urllib3<2.0.0 +aiofiles==25.1.0 +aiohttp==3.9.4 +aiohttp-cors==0.8.1 +aioitertools==0.13.0 +aiosignal==1.4.0 +attrs==26.1.0 +azure-core==1.41.0 +azure-storage-blob==12.30.0 +bitshuffle==0.5.2 +botocore==1.34.106 +build==1.5.0 +certifi==2026.7.22 +cffi==2.1.1 +charset-normalizer==3.5.1 +cryptography==50.0.1 +Cython==3.3.0 +Deprecated==1.3.1 +frozenlist==1.8.0 +fsspec==2026.1.0 +h5json @ git+https://github.com/HDFGroup/hdf5-json@master +h5py==3.16.0 +idna==3.19 +importlib_resources==7.1.0 +isodate==0.7.2 +jmespath==1.1.0 +jsonschema==4.26.0 +jsonschema-specifications==2025.9.1 +multidict==6.7.1 +numcodecs==0.15.1 +numpy==2.4.6 +openapi-spec-validator==0.9.0 +propcache==0.5.2 +psutil==7.2.2 +pycparser==3.0 +pyflakes==3.4.0 +PyJWT==2.13.0 +pyproject_hooks==1.2.0 +python-dateutil==2.9.0.post0 +pytz==2026.3.post1 +PyYAML==6.0.3 +referencing==0.37.0 +requests==2.31.0 +requests-unixsocket==0.4.1 +rpds-py==2026.6.3 +s3fs==2026.1.0 +setuptools==84.0.0 +simplejson==4.1.2 +six==1.17.0 +typing_extensions==4.16.0 +urllib3==1.26.20 +wheel==0.48.0 +wrapt==1.17.3 +yarl==1.24.5 diff --git a/runall.sh b/runall.sh index 5823e7ad..7ddbb575 100755 --- a/runall.sh +++ b/runall.sh @@ -29,11 +29,12 @@ config_value() { # script to startup hsds service if [[ $# -eq 1 ]] && ([[ $1 == "-h" ]] || [[ $1 == "--help" ]]); then - echo "Usage: runall.sh [--no-docker] [--no-docker-tcp] [--stop] [--config] [dn_count] [sn_count]" + echo "Usage: runall.sh [--no-docker] [--no-docker-tcp] [--stop] [--config] [--swagger] [dn_count] [sn_count]" echo " --no-docker: run server as set of processes rather than Docker containers (using unix sockets)" echo " --no-docker-tcp: run server as set of processes rather than Docker containers (using tcp)" echo " --stop: shutdown the server (Docker only)" echo " --config: view config options" + echo " --swagger: also launch a swagger-ui container configured to talk to the HSDS service (Docker only)" echo " count: set number of DN processes/containers (default is 4)" exit 1 fi @@ -52,6 +53,8 @@ while [[ $# -gt 0 ]]; do export DOCKER_CMD="down" elif [[ $1 == "--config" ]]; then PRINT_CONFIG=1 + elif [[ $1 == "--swagger" ]]; then + export SWAGGER=1 elif [[ -z ${DN_CORES} ]]; then export DN_CORES=$1 else @@ -60,6 +63,11 @@ while [[ $# -gt 0 ]]; do shift done +if [[ ${NO_DOCKER} ]] && [[ ${SWAGGER} ]]; then + echo "--swagger is only supported with docker (not compatible with --no-docker or --no-docker-tcp)" + exit 1 +fi + if [[ -z $CONFIG_DIR ]]; then export CONFIG_DIR="admin/config" @@ -107,6 +115,13 @@ else export SN_PORT_RANGE=$SN_PORT-$((SN_PORT + SN_CORES - 1)) fi +if [[ -z ${SWAGGER_PORT} ]]; then + # set even when --swagger isn't passed: the "down" path always + # references the swagger compose file too (see below), so this avoids + # a "variable not set" warning from docker compose in that case + export SWAGGER_PORT=8080 +fi + if [[ ${NO_DOCKER} ]]; then # setup extra envs needed when not using docker @@ -199,33 +214,66 @@ if [[ $NO_DOCKER ]] ; then fi # this will run until server is killed by ^C else + COMPOSE_FILES="-f ${COMPOSE_FILE}" + if [[ ${SWAGGER} ]]; then + COMPOSE_FILES="${COMPOSE_FILES} -f admin/docker/docker-compose.swagger.yml" + fi + if [[ $DOCKER_CMD == "down" ]]; then - # use the compose file to shutdown the sevice - echo "Running docker compose -f ${COMPOSE_FILE} down" - docker compose -f ${COMPOSE_FILE} down + # Always include the swagger compose file here, regardless of whether + # --swagger was passed to this invocation: there's no persisted record + # of whether the running cluster was started with --swagger, and + # `docker compose down` for a service that isn't running is a no-op, + # so this is the only reliable way to make sure a leftover + # swagger-ui container (which would otherwise keep the shared network + # in use and block its removal) actually gets torn down. + DOWN_COMPOSE_FILES="${COMPOSE_FILES}" + if [[ -z ${SWAGGER} ]]; then + DOWN_COMPOSE_FILES="${DOWN_COMPOSE_FILES} -f admin/docker/docker-compose.swagger.yml" + fi + echo "Running docker compose ${DOWN_COMPOSE_FILES} down" + docker compose ${DOWN_COMPOSE_FILES} down exit 0 # can quit now else - echo "Running docker compose -f ${COMPOSE_FILE} up -d --scale sn=${SN_CORES} --scale dn=${DN_CORES}" - docker compose -f ${COMPOSE_FILE} up -d --scale sn=${SN_CORES} --scale dn=${DN_CORES} + if [[ -z ${SWAGGER} ]]; then + # if a swagger-ui container from a previous --swagger run is still + # around, remove it explicitly rather than passing --remove-orphans + # to `docker compose up`: that flag's orphan cleanup can race with + # network setup for the scaled sn/dn services, leaving them stuck + # with "failed to set up container networking: network ... not + # found" errors. + swagger_container="${COMPOSE_PROJECT_NAME}-swagger-ui-1" + if docker ps -a --format '{{.Names}}' | grep -qx "${swagger_container}"; then + echo "removing orphaned ${swagger_container} container" + docker rm -f "${swagger_container}" >/dev/null + fi + fi + echo "Running docker compose ${COMPOSE_FILES} up -d --scale sn=${SN_CORES} --scale dn=${DN_CORES}" + docker compose ${COMPOSE_FILES} up -d --scale sn=${SN_CORES} --scale dn=${DN_CORES} fi - # wait for the server to be ready + # wait for the server to be up and reporting state READY + READY= for i in {1..120} do - STATUS_CODE=`curl -s -o /dev/null -w "%{http_code}" http://localhost:${SN_PORT}/about` - if [[ $STATUS_CODE == "200" ]]; then - echo "service ready!" + if HSDS_ENDPOINT="http://localhost:${SN_PORT}" python3 tools/status_check.py --no-stream --quiet; then + echo "HSDS ready at: http://localhost:${SN_PORT}" + READY=1 break else - echo "${i}: waiting for server startup (status: ${STATUS_CODE}) " + echo "${i}: waiting for server startup" sleep 1 fi done - if [[ $STATUS_CODE != "200" ]]; then + if [[ -z ${READY} ]]; then echo "service failed to start" echo "SN_1 logs:" docker logs --tail 100 hsds_sn_1 exit 1 fi + + if [[ ${SWAGGER} ]]; then + echo "Swagger UI available at http://localhost:${SWAGGER_PORT}" + fi fi diff --git a/testall.py b/testall.py index 1e8ea348..971e6154 100755 --- a/testall.py +++ b/testall.py @@ -15,13 +15,13 @@ PYTHON_CMD = "python" # change to "python3" if "python" invokes python version 2.x -unit_tests = ('array_util_test', 'chunk_util_test', 'compression_test', 'domain_util_test', - 'dset_util_test', 'hdf5_dtype_test', 'id_util_test', 'lru_cache_test', - 'shuffle_test', 'rangeget_util_test') +unit_tests = ('chunk_util_test', 'compression_test', 'domain_util_test', + 'dset_util_test', 'glob_parser_test', 'lru_cache_test', 'openapi_test', + 'rangeget_util_test', 'shuffle_test', 'stor_util_test') integ_tests = ('uptest', 'setup_test', 'domain_test', 'group_test', 'link_test', 'attr_test', 'datatype_test', 'dataset_test', - 'acl_test', 'value_test', # 'filter_test', + 'acl_test', 'value_test', 'broadcast_test', 'filter_test', 'pointsel_test', 'query_test', 'vlen_test') skip_unit = False diff --git a/tests/integ/acl_test.py b/tests/integ/acl_test.py index 2af5e24c..232f0859 100644 --- a/tests/integ/acl_test.py +++ b/tests/integ/acl_test.py @@ -239,6 +239,45 @@ def testPutAcl(self): rsp = self.session.get(req, headers=headers) self.assertEqual(rsp.status_code, 403) # Forbidden + def testPutAclPermissionCheck(self): + """ + Regression test for a fixed security gap: PUT /acls/{username} + used to perform no ACL/permission check at all, so any + authenticated user - even one with no permissions whatsoever on + the domain - could grant or revoke any permission (including + updateACL, i.e. ownership-equivalent access) for any user on the + domain. PUT_ACL (hsds/domain_sn.py) now calls aclCheck with the + "updateACL" action before forwarding the write to the DN, same as + every other ACL/domain-mutating operation. + """ + print("testPutAclPermissionCheck", self.base_domain) + + user2name = config.get("user2_name") + if not user2name: + print("user2_name not set") + return + + # test_user2 has no ACL entry at all on this domain (only the + # owner, test_user1, does by default) - so test_user2 should not + # be able to grant/revoke ANY permission for ANY user here, + # including a third, unrelated user ("joebob") who isn't even + # test_user2 themselves. + headers = helper.getRequestHeaders(domain=self.base_domain, username=user2name) + + req = helper.getEndpoint() + "/acls/joebob" + perm = { + "create": True, + "read": True, + "update": True, + "delete": True, + "readACL": True, + "updateACL": True, + } + rsp = self.session.put(req, headers=headers, data=json.dumps(perm)) + # test_user2 lacks updateACL permission on this domain, so this + # must be rejected + self.assertEqual(rsp.status_code, 403) + def testGroupAcl(self): print("testPutAcl", self.base_domain) headers = helper.getRequestHeaders(domain=self.base_domain) @@ -282,7 +321,7 @@ def testGroupAcl(self): req = helper.getEndpoint() + "/acls/" + user2name headers = helper.getRequestHeaders(domain=self.base_domain, username=user2name) rsp = self.session.get(req, headers=headers) - self.assertTrue(rsp.status_code in (403, 404)) # forbiden or not found + self.assertTrue(rsp.status_code in (403, 404)) # forbidden or not found # The default ACL should be fetchable by test_user2 as well... if config.get("default_public"): @@ -293,7 +332,7 @@ def testGroupAcl(self): rsp = self.session.get(req, headers=headers) self.assertEqual(rsp.status_code, 200) # ok else: - self.assertTrue(rsp.status_code in (403, 404)) # forbiden or not found + self.assertTrue(rsp.status_code in (403, 404)) # forbidden or not found # test_user2 shouldn't be able to read test_user1's ACL username = config.get("user_name") diff --git a/tests/integ/attr_test.py b/tests/integ/attr_test.py index de54c5ea..8816976d 100644 --- a/tests/integ/attr_test.py +++ b/tests/integ/attr_test.py @@ -12,6 +12,7 @@ from copy import copy import unittest import json +import time import numpy as np import base64 import helper @@ -500,6 +501,66 @@ def testPutFixedString(self): self.assertTrue("length" in type_json) self.assertEqual(type_json["length"], 7) + def testUseTimestamp(self): + # Test PUT value for 1d attribute with timestamp included + print("testUseTimestamp", self.base_domain) + + headers = helper.getRequestHeaders(domain=self.base_domain) + req = self.endpoint + "/" + + # Get root uuid + rsp = self.session.get(req, headers=headers) + self.assertEqual(rsp.status_code, 200) + rspJson = json.loads(rsp.text) + root_uuid = rspJson["root"] + helper.validateId(root_uuid) + + def _create_attr(attr_name, ts=None): + + # create attr + fixed_str_type = { + "charSet": "H5T_CSET_ASCII", + "class": "H5T_STRING", + "length": 12, + "strPad": "H5T_STR_NULLPAD", + } + data = {"type": fixed_str_type, "value": "XYZ"} + if ts: + data["created"] = ts + req = self.endpoint + "/groups/" + root_uuid + "/attributes/" + attr_name + rsp = self.session.put(req, data=json.dumps(data), headers=headers) + self.assertEqual(rsp.status_code, 201) + + def _check_attr_ts(attr_name, min_ts=None, max_ts=None): + # read attr + req = self.endpoint + "/groups/" + root_uuid + "/attributes/" + attr_name + rsp = self.session.get(req, headers=headers) + self.assertEqual(rsp.status_code, 200) + rspJson = json.loads(rsp.text) + self.assertTrue("hrefs" in rspJson) + self.assertTrue("value" in rspJson) + self.assertEqual(rspJson["value"], "XYZ") + self.assertTrue("type" in rspJson) + self.assertTrue("created" in rspJson) + if min_ts: + self.assertGreaterEqual(rspJson["created"], min_ts) + if max_ts: + self.assertLessEqual(rspJson["created"], max_ts) + + now = time.time() + # server-based timestamp + _create_attr("a1") + _check_attr_ts("a1", min_ts=(now - 1), max_ts=(now + 1)) + # client assigned timestamp + _create_attr("a2", ts=now) + _check_attr_ts("a2", min_ts=now, max_ts=now) + # client assigned with small time-skew, ok + _create_attr("a3", ts=int(now)) + _check_attr_ts("a3", min_ts=int(now), max_ts=int(now)) + # client assigned with large time-skew, ignored + _create_attr("a4", ts=999) + _check_attr_ts("a4", min_ts=now, max_ts=(now + 1)) + def testPutFixedStringNullTerm(self): # Test PUT value for 1d attribute with fixed length string/null terminated types print("testPutFixedStringNullTerm", self.base_domain) @@ -727,7 +788,7 @@ def testPutNonUTF8String(self): helper.validateId(root_uuid) # create attr - data = b'\xfe\xff' # invlaid UTF sequence + data = b'\xfe\xff' # invalid UTF sequence num_bytes = len(data) fixed_str_type = { @@ -992,6 +1053,63 @@ def testPutCompound(self): self.assertTrue("value" in rspJson) self.assertEqual(rspJson["value"], [42, 0.42]) + def testPutCompoundArrayField(self): + # compound attribute with an array-typed field - no other + # attribute test exercises H5T_ARRAY at all, bare or + # compound-wrapped (dataset coverage for this dtype shape is in + # tests/integ/value_test.py's testCreateArrayDataset and friends) + print("testPutCompoundArrayField", self.base_domain) + headers = helper.getRequestHeaders(domain=self.base_domain) + req = self.endpoint + "/" + + # Get root uuid + rsp = self.session.get(req, headers=headers) + self.assertEqual(rsp.status_code, 200) + rspJson = json.loads(rsp.text) + root_id = rspJson["root"] + helper.validateId(root_id) + + array_datatype = { + "class": "H5T_ARRAY", + "base": { + "class": "H5T_INTEGER", + "base": "H5T_STD_I32LE" + }, + "dims": [3] + } + fields = ( + {"name": "temp", "type": array_datatype}, + {"name": "pressure", "type": "H5T_IEEE_F32LE"}, + ) + datatype = {"class": "H5T_COMPOUND", "fields": fields} + value = ([1, 2, 3], 0.5) + + attr_name = "attr_compound_array" + payload = {"type": datatype, "value": value} + req = self.endpoint + "/groups/" + root_id + "/attributes/" + attr_name + rsp = self.session.put(req, data=json.dumps(payload), headers=headers) + self.assertEqual(rsp.status_code, 201) # create attribute + + # read back the attribute + rsp = self.session.get(req, headers=headers) + self.assertEqual(rsp.status_code, 200) + rspJson = json.loads(rsp.text) + self.assertTrue("value" in rspJson) + self.assertEqual(rspJson["value"], [[1, 2, 3], 0.5]) + + self.assertTrue("type" in rspJson) + rsp_type = rspJson["type"] + self.assertEqual(rsp_type["class"], "H5T_COMPOUND") + rsp_fields = rsp_type["fields"] + self.assertEqual(len(rsp_fields), 2) + self.assertEqual(rsp_fields[0]["name"], "temp") + self.assertEqual(rsp_fields[0]["type"]["class"], "H5T_ARRAY") + self.assertEqual(rsp_fields[0]["type"]["dims"], [3]) + self.assertEqual(rsp_fields[1]["name"], "pressure") + + self.assertTrue("shape" in rspJson) + self.assertEqual(rspJson["shape"]["class"], "H5S_SCALAR") + def testPutObjReference(self): print("testPutObjReference", self.base_domain) headers = helper.getRequestHeaders(domain=self.base_domain) @@ -1250,6 +1368,187 @@ def testPutCompoundObjReference(self): self.assertTrue("value" in rspJson) self.assertEqual(rspJson["value"], [[dset_id, 0], ]) + def testPutRegionReference(self): + print("testPutRegionReference", self.base_domain) + headers = helper.getRequestHeaders(domain=self.base_domain) + req = self.endpoint + "/" + + rsp = self.session.get(req, headers=headers) + self.assertEqual(rsp.status_code, 200) + rspJson = json.loads(rsp.text) + root_id = rspJson["root"] + + # create group "g1" + payload = {"link": {"id": root_id, "name": "g1_regionref"}} + req = helper.getEndpoint() + "/groups" + rsp = self.session.post(req, data=json.dumps(payload), headers=headers) + self.assertEqual(rsp.status_code, 201) + rspJson = json.loads(rsp.text) + g1_id = rspJson["id"] + self.assertTrue(helper.validateId(g1_id)) + + # create dataset "dset" that will be the target of the region reference + payload = { + "type": "H5T_STD_I32LE", + "shape": [10, 10], + "link": {"id": root_id, "name": "dset_regionref"}, + } + req = self.endpoint + "/datasets" + rsp = self.session.post(req, data=json.dumps(payload), headers=headers) + self.assertEqual(rsp.status_code, 201) + rspJson = json.loads(rsp.text) + dset_id = rspJson["id"] + self.assertTrue(helper.validateId(dset_id)) + + # create attr of g1 that is a region reference to a hyperslab + # selection of dset (rows 1:4, cols 2:6) + ref_type = {"class": "H5T_REFERENCE", "base": "H5T_STD_REF_DSETREG"} + attr_name = "region_ref" + value = { + "id": dset_id[2:], # bare uuid, no 'd-' prefix + "select_type": "H5S_SEL_HYPERSLABS", + "selection": [[[1, 2], [3, 5]]], + } + data = {"type": ref_type, "value": value} + req = self.endpoint + "/groups/" + g1_id + "/attributes/" + attr_name + rsp = self.session.put(req, data=json.dumps(data), headers=headers) + self.assertEqual(rsp.status_code, 201) + + # read back the attribute and verify the type, space, and value + req = self.endpoint + "/groups/" + g1_id + "/attributes/" + attr_name + rsp = self.session.get(req, headers=headers) + self.assertEqual(rsp.status_code, 200) + rspJson = json.loads(rsp.text) + self.assertTrue("type" in rspJson) + rsp_type = rspJson["type"] + self.assertTrue("base" in rsp_type) + self.assertEqual(rsp_type["base"], "H5T_STD_REF_DSETREG") + self.assertTrue("class" in rsp_type) + self.assertEqual(rsp_type["class"], "H5T_REFERENCE") + self.assertTrue("shape" in rspJson) + rsp_shape = rspJson["shape"] + self.assertTrue("class" in rsp_shape) + self.assertEqual(rsp_shape["class"], "H5S_SCALAR") + self.assertTrue("value" in rspJson) + rsp_value = rspJson["value"] + self.assertEqual(rsp_value["id"], dset_id[2:]) + self.assertEqual(rsp_value["select_type"], "H5S_SEL_HYPERSLABS") + self.assertEqual(rsp_value["selection"], [[[1, 2], [3, 5]]]) + + def testPutPointsRegionReference(self): + print("testPutPointsRegionReference", self.base_domain) + headers = helper.getRequestHeaders(domain=self.base_domain) + req = self.endpoint + "/" + + rsp = self.session.get(req, headers=headers) + self.assertEqual(rsp.status_code, 200) + rspJson = json.loads(rsp.text) + root_id = rspJson["root"] + + # create group "g1" + payload = {"link": {"id": root_id, "name": "g1_pointsregionref"}} + req = helper.getEndpoint() + "/groups" + rsp = self.session.post(req, data=json.dumps(payload), headers=headers) + self.assertEqual(rsp.status_code, 201) + rspJson = json.loads(rsp.text) + g1_id = rspJson["id"] + self.assertTrue(helper.validateId(g1_id)) + + # create dataset "dset" that will be the target of the region reference + payload = { + "type": "H5T_STD_I32LE", + "shape": [10, 10], + "link": {"id": root_id, "name": "dset_pointsregionref"}, + } + req = self.endpoint + "/datasets" + rsp = self.session.post(req, data=json.dumps(payload), headers=headers) + self.assertEqual(rsp.status_code, 201) + rspJson = json.loads(rsp.text) + dset_id = rspJson["id"] + self.assertTrue(helper.validateId(dset_id)) + + # create attr of g1 that is a region reference to a point + # selection of dset + ref_type = {"class": "H5T_REFERENCE", "base": "H5T_STD_REF_DSETREG"} + attr_name = "region_ref_points" + value = { + "id": dset_id[2:], + "select_type": "H5S_SEL_POINTS", + "selection": [[0, 0], [2, 2], [4, 4]], + } + data = {"type": ref_type, "value": value} + req = self.endpoint + "/groups/" + g1_id + "/attributes/" + attr_name + rsp = self.session.put(req, data=json.dumps(data), headers=headers) + self.assertEqual(rsp.status_code, 201) + + # read back the attribute and verify the type, space, and value + req = self.endpoint + "/groups/" + g1_id + "/attributes/" + attr_name + rsp = self.session.get(req, headers=headers) + self.assertEqual(rsp.status_code, 200) + rspJson = json.loads(rsp.text) + self.assertTrue("type" in rspJson) + rsp_type = rspJson["type"] + self.assertEqual(rsp_type["base"], "H5T_STD_REF_DSETREG") + self.assertEqual(rsp_type["class"], "H5T_REFERENCE") + self.assertTrue("value" in rspJson) + rsp_value = rspJson["value"] + self.assertEqual(rsp_value["id"], dset_id[2:]) + self.assertEqual(rsp_value["select_type"], "H5S_SEL_POINTS") + self.assertEqual(rsp_value["selection"], [[0, 0], [2, 2], [4, 4]]) + + def testPutOpaqueAttribute(self): + print("testPutOpaqueAttribute", self.base_domain) + headers = helper.getRequestHeaders(domain=self.base_domain) + req = self.endpoint + "/" + + rsp = self.session.get(req, headers=headers) + self.assertEqual(rsp.status_code, 200) + rspJson = json.loads(rsp.text) + root_id = rspJson["root"] + + opaque_type = {"class": "H5T_OPAQUE", "size": 7} + + # scalar opaque attribute + attr_name = "opaque_scalar" + value = base64.b64encode(b"OPAQUE0").decode("ascii") + data = {"type": opaque_type, "value": value} + req = self.endpoint + "/groups/" + root_id + "/attributes/" + attr_name + rsp = self.session.put(req, data=json.dumps(data), headers=headers) + self.assertEqual(rsp.status_code, 201) + + rsp = self.session.get(req, headers=headers) + self.assertEqual(rsp.status_code, 200) + rspJson = json.loads(rsp.text) + self.assertTrue("type" in rspJson) + rsp_type = rspJson["type"] + self.assertEqual(rsp_type["class"], "H5T_OPAQUE") + self.assertEqual(rsp_type["size"], 7) + self.assertTrue("shape" in rspJson) + self.assertEqual(rspJson["shape"]["class"], "H5S_SCALAR") + self.assertTrue("value" in rspJson) + self.assertEqual(rspJson["value"], value) + + # 1-D array of opaque values, including an empty (all-zero) element + attr_name = "opaque_array" + values = [ + base64.b64encode(b"OPAQUE0").decode("ascii"), + base64.b64encode(b"OPAQUE1").decode("ascii"), + "", + ] + data = {"type": opaque_type, "shape": 3, "value": values} + req = self.endpoint + "/groups/" + root_id + "/attributes/" + attr_name + rsp = self.session.put(req, data=json.dumps(data), headers=headers) + self.assertEqual(rsp.status_code, 201) + + rsp = self.session.get(req, headers=headers) + self.assertEqual(rsp.status_code, 200) + rspJson = json.loads(rsp.text) + rsp_type = rspJson["type"] + self.assertEqual(rsp_type["class"], "H5T_OPAQUE") + self.assertEqual(rsp_type["size"], 7) + self.assertEqual(rspJson["shape"]["dims"], [3, ]) + self.assertEqual(rspJson["value"], values) + def testPutNoData(self): # Test PUT value for 1d attribute without any data provided print("testPutNoData", self.base_domain) @@ -1530,7 +1829,6 @@ def testPutAttributeWithEncoding(self): self.assertTrue("type" in rspJson) self.assertTrue("shape" in rspJson) self.assertTrue("encoding" not in rspJson) - # self.assertEqual(rspJson["encoding"], "base64") self.assertEqual(rspJson["value"], value) # get the encoded value back params = {"encoding": "base64"} diff --git a/tests/integ/dataset_test.py b/tests/integ/dataset_test.py index 280877cf..d12b8522 100755 --- a/tests/integ/dataset_test.py +++ b/tests/integ/dataset_test.py @@ -13,16 +13,13 @@ import json import time import numpy as np + +from h5json.objid import createObjId +from h5json.filters import getFilterItem + import helper import config -# min/max chunk size - these can be set by config, but -# practially the min config value should be larger than -# CHUNK_MIN and the max config value should less than -# CHUNK_MAX -CHUNK_MIN = 1024 # lower limit (1024b) -CHUNK_MAX = 50 * 1024 * 1024 # upper limit (50M) - class DatasetTest(unittest.TestCase): def __init__(self, *args, **kwargs): @@ -61,31 +58,34 @@ def testScalarDataset(self): rsp = self.session.post(req, data=json.dumps(data), headers=headers) self.assertEqual(rsp.status_code, 201) rspJson = json.loads(rsp.text) - self.assertEqual(rspJson["attributeCount"], 0) - dset_id = rspJson["id"] - self.assertTrue(helper.validateId(dset_id)) - - # read back the obj - req = self.endpoint + "/datasets/" + dset_id - rsp = self.session.get(req, headers=headers) - self.assertEqual(rsp.status_code, 200) - rspJson = json.loads(rsp.text) expected_keys = [ "id", "shape", - "hrefs", - "layout", "creationProperties", "attributeCount", "created", "lastModified", "root", - "domain", ] - for name in expected_keys: self.assertTrue(name in rspJson) + + # additional keys expected for GET response + expected_keys.append("hrefs") + expected_keys.append("creationProperties") + expected_keys.append("domain") + + self.assertEqual(rspJson["attributeCount"], 0) + dset_id = rspJson["id"] + self.assertTrue(helper.validateId(dset_id)) + + # read back the obj + req = self.endpoint + "/datasets/" + dset_id + rsp = self.session.get(req, headers=headers) + self.assertEqual(rsp.status_code, 200) + rspJson = json.loads(rsp.text) + self.assertEqual(rspJson["id"], dset_id) self.assertEqual(rspJson["root"], root_uuid) self.assertEqual(rspJson["domain"], domain) @@ -190,6 +190,182 @@ def testScalarDataset(self): rsp = self.session.get(req, headers=headers) self.assertEqual(rsp.status_code, 410) + def testPostDatasetWithId(self): + # Test creation of a dataset obj with client creating obj id + domain = self.base_domain + "/testPostDatasetWithId.h5" + helper.setupDomain(domain) + print("testPostDatasetWithId", domain) + headers = helper.getRequestHeaders(domain=domain) + req = self.endpoint + "/" + + # Get root uuid + rsp = self.session.get(req, headers=headers) + self.assertEqual(rsp.status_code, 200) + rspJson = json.loads(rsp.text) + root_uuid = rspJson["root"] + helper.validateId(root_uuid) + + # make a new dataset id + dset_id = createObjId("datasets", root_id=root_uuid) + + # create a dataset obj + data = {"id": dset_id, "type": "H5T_IEEE_F32LE", "shape": "H5S_SCALAR"} + req = self.endpoint + "/datasets" + rsp = self.session.post(req, data=json.dumps(data), headers=headers) + self.assertEqual(rsp.status_code, 201) + rspJson = json.loads(rsp.text) + self.assertEqual(rspJson["attributeCount"], 0) + self.assertEqual(rspJson["id"], dset_id) + + # read back the obj + req = self.endpoint + "/datasets/" + dset_id + rsp = self.session.get(req, headers=headers) + self.assertEqual(rsp.status_code, 200) + rspJson = json.loads(rsp.text) + + expected_keys = [ + "id", + "shape", + "hrefs", + "creationProperties", + "attributeCount", + "created", + "lastModified", + "root", + "domain", + ] + + for name in expected_keys: + self.assertTrue(name in rspJson) + self.assertEqual(rspJson["id"], dset_id) + self.assertEqual(rspJson["root"], root_uuid) + self.assertEqual(rspJson["domain"], domain) + self.assertEqual(rspJson["attributeCount"], 0) + shape_json = rspJson["shape"] + self.assertTrue(shape_json["class"], "H5S_SCALAR") + self.assertTrue(rspJson["type"], "H5T_IEEE_F32LE") + + # Get the type + rsp = self.session.get(req + "/type", headers=headers) + self.assertEqual(rsp.status_code, 200) + rspJson = json.loads(rsp.text) + self.assertTrue("type" in rspJson) + self.assertTrue(rspJson["type"], "H5T_IEEE_F32LE") + self.assertTrue("hrefs" in rspJson) + hrefs = rspJson["hrefs"] + self.assertEqual(len(hrefs), 3) + + # Get the shape + rsp = self.session.get(req + "/shape", headers=headers) + self.assertEqual(rsp.status_code, 200) + rspJson = json.loads(rsp.text) + self.assertTrue("created" in rspJson) + self.assertTrue("lastModified" in rspJson) + self.assertTrue("hrefs" in rspJson) + self.assertTrue("shape" in rspJson) + shape_json = rspJson["shape"] + self.assertTrue(shape_json["class"], "H5S_SCALAR") + + # try getting verbose info + params = {"verbose": 1} + rsp = self.session.get(req, params=params, headers=headers) + self.assertEqual(rsp.status_code, 200) + rspJson = json.loads(rsp.text) + + for name in expected_keys: + self.assertTrue(name in rspJson) + + # flush to storage and force an immediate rescan + domain_req = self.endpoint + "/" + domain_params = {"flush": 1, "rescan": 1} + rsp = self.session.put(domain_req, params=domain_params, headers=headers) + # should get a NO_CONTENT code, + self.assertEqual(rsp.status_code, 204) + + # do a get and verify the additional keys are present + expected_keys.append("num_chunks") + expected_keys.append("allocated_size") + + rsp = self.session.get(req, params=params, headers=headers) + self.assertEqual(rsp.status_code, 200) + rspJson = json.loads(rsp.text) + + for name in expected_keys: + self.assertTrue(name in rspJson) + + def testPostDatasetWithAttributes(self): + # test POST with attribute initialization + domain = self.base_domain + "/testPostDatasetWithAttributes.h5" + helper.setupDomain(domain) + print("testPostDatasetWithAttributes", domain) + headers = helper.getRequestHeaders(domain=domain) + + # get root id + req = helper.getEndpoint() + "/" + rsp = self.session.get(req, headers=headers) + self.assertEqual(rsp.status_code, 200) + rspJson = json.loads(rsp.text) + root_uuid = rspJson["root"] + helper.validateId(root_uuid) + + # setup some attributes to include + attr_count = 4 + attributes = {} + extent = 10 + for i in range(attr_count): + value = [i * 10 + j for j in range(extent)] + data = {"type": "H5T_STD_I32LE", "shape": extent, "value": value} + attr_name = f"attr{i + 1:04d}" + attributes[attr_name] = data + + # create new dataset + payload = {"type": "H5T_IEEE_F32LE", "shape": "H5S_SCALAR"} + payload["attributes"] = attributes + payload["link"] = {"id": root_uuid, "name": "linked_datatype"} + + req = helper.getEndpoint() + "/datasets" + rsp = self.session.post(req, data=json.dumps(payload), headers=headers) + self.assertEqual(rsp.status_code, 201) + rspJson = json.loads(rsp.text) + self.assertEqual(rspJson["attributeCount"], 4) + dset_id = rspJson["id"] + self.assertTrue(helper.validateId(dset_id)) + + # fetch the attributes + req = f"{helper.getEndpoint()}/datasets/{dset_id}/attributes" + rsp = self.session.get(req, headers=headers) + self.assertEqual(rsp.status_code, 200) + rspJson = json.loads(rsp.text) + self.assertTrue("hrefs" in rspJson) + self.assertFalse("type" in rspJson) + self.assertFalse("shape" in rspJson) + self.assertTrue("attributes") in rspJson + self.assertEqual(len(rspJson["attributes"]), attr_count) + + # try fetching the objson in domain resp + req = helper.getEndpoint() + "/" + params = {"getobjs": 1} + for i in range(10): + # try a few times to allow for async update of summary info + time.sleep(5) + rsp = self.session.get(req, params=params, headers=headers) + self.assertEqual(rsp.status_code, 200) + rspJson = json.loads(rsp.text) + if "domain_objs" in rspJson: + break + + self.assertTrue("domain_objs" in rspJson) + domain_objs = rspJson["domain_objs"] + self.assertTrue(root_uuid in domain_objs) + self.assertTrue(dset_id in domain_objs) + dset_json = domain_objs[dset_id] + self.assertTrue("attributes" in dset_json) + self.assertEqual(len(dset_json["attributes"]), attr_count) + self.assertTrue("type" in dset_json) + self.assertTrue("shape" in dset_json) + self.assertTrue("creationProperties" in dset_json) + self.assertFalse("value" in dset_json) # no data written yet + def testScalarEmptyDimsDataset(self): # Test creation/deletion of scalar dataset obj domain = self.base_domain + "/testScalarEmptyDimsDataset.h5" @@ -265,7 +441,6 @@ def testGet(self): "id", "shape", "hrefs", - "layout", "creationProperties", "attributeCount", "created", @@ -289,7 +464,8 @@ def testGet(self): self.assertEqual(shape["dims"], [10, 10]) self.assertEqual(shape["maxdims"], [10, 10]) - layout = rspJson["layout"] + cpl = rspJson["creationProperties"] + layout = cpl["layout"] self.assertEqual(layout["class"], "H5D_CHUNKED") self.assertEqual(layout["dims"], [10, 10]) self.assertTrue("partition_count" not in layout) @@ -359,7 +535,6 @@ def testGetByPath(self): "id", "shape", "hrefs", - "layout", "creationProperties", "attributeCount", "created", @@ -381,7 +556,6 @@ def testGetByPath(self): "id", "shape", "hrefs", - "layout", "creationProperties", "attributeCount", "created", @@ -443,7 +617,6 @@ def testGetVerbose(self): "id", "shape", "hrefs", - "layout", "creationProperties", "attributeCount", "created", @@ -637,10 +810,19 @@ def testResizableDataset(self): # create the dataset req = self.endpoint + "/datasets" payload = {"type": "H5T_IEEE_F32LE", "shape": 10, "maxdims": 20} - payload["creationProperties"] = {"fillValue": 3.12} + contiguous_layout = {"class": "H5D_CONTIGUOUS"} + cpl = {"fillValue": 3.12, "layout": contiguous_layout} + payload["creationProperties"] = cpl + req = self.endpoint + "/datasets" rsp = self.session.post(req, data=json.dumps(payload), headers=headers) - self.assertEqual(rsp.status_code, 201) # create dataset + self.assertEqual(rsp.status_code, 400) # need chunk layout for resizable + + # if we remove the layout, HSDS will setup a chunked layout for us + del cpl["layout"] + rsp = self.session.post(req, data=json.dumps(payload), headers=headers) + self.assertEqual(rsp.status_code, 201) + rspJson = json.loads(rsp.text) dset_uuid = rspJson["id"] self.assertTrue(helper.validateId(dset_uuid)) @@ -668,8 +850,16 @@ def testResizableDataset(self): self.assertTrue("maxdims" in shape) self.assertEqual(shape["maxdims"][0], 20) + self.assertTrue("creationProperties" in rspJson) creationProps = rspJson["creationProperties"] + self.assertTrue("fillValue" in creationProps) self.assertEqual(creationProps["fillValue"], 3.12) + self.assertTrue("layout" in creationProps) + layout = creationProps["layout"] + self.assertTrue("class" in layout) + self.assertEqual(layout["class"], "H5D_CHUNKED") + self.assertTrue("dims" in layout) + self.assertEqual(len(layout["dims"]), 1) # verify shape using the GET shape request req = req + "/shape" @@ -691,12 +881,6 @@ def testResizableDataset(self): self.assertEqual(rsp.status_code, 201) rspJson = json.loads(rsp.text) - # reduce the size to 5 elements - # payload = {"shape": 5} - # rsp = self.session.put(req, data=json.dumps(payload), headers=headers) - # self.assertEqual(rsp.status_code, 201) - # rspJson = json.loads(rsp.text) - # verify updated-shape using the GET shape request rsp = self.session.get(req, headers=headers) self.assertEqual(rsp.status_code, 200) @@ -748,7 +932,7 @@ def testResizableUnlimitedDataset(self): # create the dataset req = self.endpoint + "/datasets" - payload = {"type": "H5T_IEEE_F32LE", "shape": [10, 20], "maxdims": [30, 0]} + payload = {"type": "H5T_IEEE_F32LE", "shape": [10, 20], "maxdims": [30, "H5S_UNLIMITED"]} payload["creationProperties"] = {"fillValue": 3.12} req = self.endpoint + "/datasets" rsp = self.session.post(req, data=json.dumps(payload), headers=headers) @@ -780,7 +964,7 @@ def testResizableUnlimitedDataset(self): self.assertEqual(shape["dims"][1], 20) self.assertTrue("maxdims" in shape) self.assertEqual(shape["maxdims"][0], 30) - self.assertEqual(shape["maxdims"][1], 0) + self.assertEqual(shape["maxdims"][1], "H5S_UNLIMITED") # verify shape using the GET shape request req = req + "/shape" @@ -797,7 +981,7 @@ def testResizableUnlimitedDataset(self): self.assertTrue("maxdims" in shape) self.assertEqual(len(shape["maxdims"]), 2) self.assertEqual(shape["maxdims"][0], 30) - self.assertEqual(shape["maxdims"][1], 0) + self.assertEqual(shape["maxdims"][1], "H5S_UNLIMITED") # resize the second dimension to 500 elements payload = {"shape": [10, 500]} @@ -819,7 +1003,7 @@ def testResizableUnlimitedDataset(self): self.assertTrue("maxdims" in shape) self.assertEqual(len(shape["maxdims"]), 2) self.assertEqual(shape["maxdims"][0], 30) - self.assertEqual(shape["maxdims"][1], 0) + self.assertEqual(shape["maxdims"][1], "H5S_UNLIMITED") def testExtendDataset(self): # test extending dataset @@ -969,7 +1153,7 @@ def testExtend2DDataset(self): self.assertEqual(shape["dims"][0], 10) self.assertEqual(shape["dims"][1], 20) self.assertTrue("maxdims" in shape) - self.assertEqual(shape["maxdims"][0], 0) + self.assertEqual(shape["maxdims"], ["H5S_UNLIMITED", "H5S_UNLIMITED"]) # verify shape using the GET shape request req = req + "/shape" @@ -983,7 +1167,7 @@ def testExtend2DDataset(self): self.assertEqual(len(shape["dims"]), 2) self.assertEqual(shape["dims"][0], 10) self.assertTrue("maxdims" in shape) - self.assertEqual(shape["maxdims"][0], 0) + self.assertEqual(shape["maxdims"], ["H5S_UNLIMITED", "H5S_UNLIMITED"]) # extend the dataset by 5 elements in first dimension payload = {"extend": 5, "extend_dim": 0} @@ -1068,7 +1252,7 @@ def testExtend3DDataset(self): self.assertEqual(len(shape["dims"]), 3) self.assertEqual(shape["dims"], [0, 3, 0]) self.assertTrue("maxdims" in shape) - self.assertEqual(shape["maxdims"], [0, 3, 0]) + self.assertEqual(shape["maxdims"], ["H5S_UNLIMITED", 3, "H5S_UNLIMITED"]) # verify shape using the GET shape request req = req + "/shape" @@ -1082,7 +1266,7 @@ def testExtend3DDataset(self): self.assertEqual(len(shape["dims"]), 3) self.assertEqual(shape["dims"], [0, 3, 0]) self.assertTrue("maxdims" in shape) - self.assertEqual(shape["maxdims"], [0, 3, 0]) + self.assertEqual(shape["maxdims"], ["H5S_UNLIMITED", 3, "H5S_UNLIMITED"]) # extend the dataset by 5 elements in first dimension payload = {"extend": 5, "extend_dim": 0} @@ -1107,6 +1291,98 @@ def testExtend3DDataset(self): rsp = self.session.put(req, data=json.dumps(payload), headers=headers) self.assertEqual(rsp.status_code, 409) # tried to extend a non-extensible dimension + def testInvalidDatasetLayout(self): + # test that various invalid layouts fail with a 400 status + domain = self.base_domain + "/testInvalidDatasetLayout.h5" + helper.setupDomain(domain) + + print("testInvalidDatasetLayout", domain) + headers = helper.getRequestHeaders(domain=domain) + # get domain + req = helper.getEndpoint() + "/" + rsp = self.session.get(req, headers=headers) + self.assertEqual(rsp.status_code, 200) + rspJson = json.loads(rsp.text) + self.assertTrue("root" in rspJson) + + # dataset create + req = self.endpoint + "/datasets" + # TBD: the larger dimensions are causing SN to crash + # dims = [365, 780, 1024] + dims = [365, 780, 10] + # maxdims = [0, 780, 1024] + maxdims = [0, 780, 10] + payload = { + "type": "H5T_IEEE_F32LE", + "shape": dims, + "maxdims": maxdims, + } + # bad layout class + payload["creationProperties"] = { + "layout": {"class": "H5D_XYZ", "dims": [1, 390, 512]}, + } + rsp = self.session.post(req, data=json.dumps(payload), headers=headers) + self.assertEqual(rsp.status_code, 400) # create dataset + + # chunked layout with mismatched dimensions + payload["creationProperties"] = { + "layout": {"class": "H5D_XYZ", "dims": [1, 390]}, + } + rsp = self.session.post(req, data=json.dumps(payload), headers=headers) + self.assertEqual(rsp.status_code, 400) # create dataset + + # chunked layout with negative dimensions + payload["creationProperties"] = { + "layout": {"class": "H5D_XYZ", "dims": [100, 200, -300]}, + } + rsp = self.session.post(req, data=json.dumps(payload), headers=headers) + self.assertEqual(rsp.status_code, 400) # create dataset + + file_uri = "s3://a-storage-bucket/some-file.h5" + offset = 1234 + size = dims[0] * dims[1] * dims[2] * 4 + + # H5D_CONTIGUOUS layout missing different required keys... + for key in ("file_uri", "offset", "size"): + layout = {"class": "H5D_CONTIGUOUS_REF"} + if key != "file_uri": + layout["file_uri"] = file_uri + elif key != "offset": + layout["offset"] = offset + elif key != "size": + layout["size"] = size + else: + self.assertTrue(False) # one of the above should be true + + payload["creationProperties"] = { + "layout": layout + } + rsp = self.session.post(req, data=json.dumps(payload), headers=headers) + self.assertEqual(rsp.status_code, 400) # create dataset + + # H5D_CONTIGOUS with a vlen type + type_vstr = { + "charSet": "H5T_CSET_ASCII", + "class": "H5T_STRING", + "strPad": "H5T_STR_NULLTERM", + "length": "H5T_VARIABLE", + } + payload = { + "type": type_vstr, + "shape": dims, + } + layout = { + "class": "H5D_CONTIGUOUS_REF", + "file_uri": file_uri, + "offset": offset, + "size": size + } + payload["creationProperties"] = { + "layout": layout + } + rsp = self.session.post(req, data=json.dumps(payload), headers=headers) + self.assertEqual(rsp.status_code, 400) # create dataset + def testCreationPropertiesLayoutDataset(self): # test Dataset with creation property list domain = self.base_domain + "/testCreationPropertiesLayoutDataset.h5" @@ -1130,6 +1406,7 @@ def testCreationPropertiesLayoutDataset(self): "shape": [365, 780, 1024], "maxdims": [0, 780, 1024], } + # define a chunk layout with 4 chunks per 'slice' # chunk size is 798720 bytes gzip_filter = { @@ -1143,41 +1420,67 @@ def testCreationPropertiesLayoutDataset(self): "id": 3, "name": "fletcher32" } - payload["creationProperties"] = { - "layout": {"class": "H5D_CHUNKED", "dims": [1, 390, 512]}, + contiguous_layout = {"class": "H5D_CONTIGUOUS"} + chunked_layout = {"class": "H5D_CHUNKED", "dims": [1, 390, 512]} + creationProps = { "filters": [ gzip_filter, fletcher32_filter, ], } - req = self.endpoint + "/datasets" + payload["creationProperties"] = creationProps rsp = self.session.post(req, data=json.dumps(payload), headers=headers) self.assertEqual(rsp.status_code, 201) # create dataset rspJson = json.loads(rsp.text) dset_uuid = rspJson["id"] self.assertTrue(helper.validateId(dset_uuid)) - # link new dataset as 'chunktest' - name = "chunktest" - req = self.endpoint + "/groups/" + root_uuid + "/links/" + name - payload = {"id": dset_uuid} - rsp = self.session.put(req, data=json.dumps(payload), headers=headers) - self.assertEqual(rsp.status_code, 201) # verify layout req = helper.getEndpoint() + "/datasets/" + dset_uuid rsp = self.session.get(req, headers=headers) self.assertEqual(rsp.status_code, 200) rspJson = json.loads(rsp.text) - self.assertTrue("layout" in rspJson) - layout_json = rspJson["layout"] + self.assertTrue("creationProperties" in rspJson) + cpl = rspJson["creationProperties"] + self.assertTrue("layout" in cpl) + layout_json = cpl["layout"] + self.assertTrue("class" in layout_json) + self.assertEqual(layout_json["class"], "H5D_CHUNKED") + self.assertTrue("dims" in layout_json) # layout created automatically + + # add an explicit layout to creation props and verify contiguous + creationProps["layout"] = contiguous_layout + payload["creationProperties"] = creationProps + req = self.endpoint + "/datasets" + rsp = self.session.post(req, data=json.dumps(payload), headers=headers) + self.assertEqual(rsp.status_code, 400) + + # use a chunk layout to creation props and verify success + creationProps["layout"] = chunked_layout + payload["creationProperties"] = creationProps + rsp = self.session.post(req, data=json.dumps(payload), headers=headers) + self.assertEqual(rsp.status_code, 201) + rspJson = json.loads(rsp.text) + self.assertTrue("creationProperties" in rspJson) + cpl = rspJson["creationProperties"] + self.assertTrue("layout" in cpl) + layout_json = cpl["layout"] self.assertTrue("class" in layout_json) self.assertEqual(layout_json["class"], "H5D_CHUNKED") self.assertTrue("dims" in layout_json) - self.assertEqual(layout_json["dims"], [1, 390, 1024]) + + self.assertEqual(layout_json["dims"], [1, 390, 512]) if config.get("max_chunks_per_folder") > 0: self.assertTrue("partition_count" in layout_json) self.assertEqual(layout_json["partition_count"], 10) + # link new dataset as 'chunktest' + name = "chunktest" + req = self.endpoint + "/groups/" + root_uuid + "/links/" + name + payload = {"id": dset_uuid} + rsp = self.session.put(req, data=json.dumps(payload), headers=headers) + self.assertEqual(rsp.status_code, 201) + # verify compression self.assertTrue("creationProperties" in rspJson) cpl = rspJson["creationProperties"] @@ -1216,7 +1519,7 @@ def testCreationPropertiesContiguousDataset(self): req = self.endpoint + "/datasets" # Create ~1GB dataset - layout = {"class": "H5D_CONTIGUOUS"} + layout = {"class": "H5D_CHUNKED", "dims": [10, 20]} gzip_filter = { "class": "H5Z_FILTER_DEFLATE", "id": 1, @@ -1249,16 +1552,14 @@ def testCreationPropertiesContiguousDataset(self): rsp = self.session.get(req, headers=headers) self.assertEqual(rsp.status_code, 200) rspJson = json.loads(rsp.text) - self.assertTrue("layout" in rspJson) - layout_json = rspJson["layout"] + self.assertTrue("creationProperties" in rspJson) + cpl = rspJson["creationProperties"] + self.assertTrue("layout" in cpl) + layout_json = cpl["layout"] self.assertTrue("class" in layout_json) self.assertEqual(layout_json["class"], "H5D_CHUNKED") self.assertTrue("dims" in layout_json) self.assertEqual(layout_json["dims"], [10, 20]) - # verify creation properties are preserved - self.assertTrue("creationProperties" in rspJson) - cpl = rspJson["creationProperties"] - self.assertTrue("layout" in cpl) def testCompressionFiltersDataset(self): # test Dataset with creation property list @@ -1283,11 +1584,11 @@ def testCompressionFiltersDataset(self): req = self.endpoint + "/datasets" payload = {"type": "H5T_IEEE_F32LE", "shape": [40, 80]} - payload["creationProperties"] = { - "filters": [ - compressor, - ] - } + filter_item = getFilterItem(compressor) + filters = [filter_item, ] + layout = {"class": "H5D_CHUNKED", "dims": [10, 20]} + cpl = {"filters": filters, "layout": layout} + payload["creationProperties"] = cpl req = self.endpoint + "/datasets" rsp = self.session.post(req, data=json.dumps(payload), headers=headers) self.assertEqual(rsp.status_code, 201) # create dataset @@ -1305,8 +1606,10 @@ def testCompressionFiltersDataset(self): rsp = self.session.get(req, headers=headers) self.assertEqual(rsp.status_code, 200) rspJson = json.loads(rsp.text) - self.assertTrue("layout" in rspJson) - layout_json = rspJson["layout"] + self.assertTrue("creationProperties" in rspJson) + cpl = rspJson["creationProperties"] + self.assertTrue("layout" in cpl) + layout_json = cpl["layout"] self.assertTrue("class" in layout_json) self.assertEqual(layout_json["class"], "H5D_CHUNKED") @@ -1342,14 +1645,14 @@ def testCompressionFilterOptionDataset(self): # create the dataset req = self.endpoint + "/datasets" - compressor = {"class": "H5Z_FILTER_USER", "name": "lz4", "level": 5} + filter_item = getFilterItem("lz4", options={"level": 4}) + filters = [filter_item, ] payload = {"type": "H5T_IEEE_F32LE", "shape": [40, 80]} - payload["creationProperties"] = { - "filters": [ - compressor, - ] - } + layout = {"class": "H5D_CHUNKED", "dims": [10, 20]} + cpl = {"filters": filters, "layout": layout} + payload["creationProperties"] = cpl + req = self.endpoint + "/datasets" rsp = self.session.post(req, data=json.dumps(payload), headers=headers) self.assertEqual(rsp.status_code, 201) # create dataset @@ -1367,10 +1670,13 @@ def testCompressionFilterOptionDataset(self): rsp = self.session.get(req, headers=headers) self.assertEqual(rsp.status_code, 200) rspJson = json.loads(rsp.text) - self.assertTrue("layout" in rspJson) - layout_json = rspJson["layout"] + self.assertTrue("creationProperties" in rspJson) + cpl = rspJson["creationProperties"] + self.assertTrue("layout" in cpl) + layout_json = cpl["layout"] self.assertTrue("class" in layout_json) self.assertEqual(layout_json["class"], "H5D_CHUNKED") + self.assertTrue("dims" in layout_json) # verify compression self.assertTrue("creationProperties" in rspJson) @@ -1381,7 +1687,7 @@ def testCompressionFilterOptionDataset(self): filter = filters[0] self.assertTrue(isinstance(filter, dict)) self.assertTrue("class" in filter) - self.assertEqual(filter["class"], "H5Z_FILTER_USER") + self.assertEqual(filter["class"], "H5Z_FILTER_LZ4") self.assertTrue("id" in filter) self.assertTrue("name" in filter) self.assertEqual(filter["name"], "lz4") @@ -1400,25 +1706,15 @@ def testInvalidCompressionFilter(self): rspJson = json.loads(rsp.text) self.assertTrue("root" in rspJson) - bad_compressors = ("shrink-o-rama") - for compressor_name in bad_compressors: - # create the dataset - req = self.endpoint + "/datasets" - compressor = { - "class": "H5Z_FILTER_USER", - "name": compressor_name, - "level": 5, - } + filter_item = {'class': 'H5Z_FILTER_FOOBAR', 'id': 123, 'name': 'foobar'} + # create the dataset + req = self.endpoint + "/datasets" - payload = {"type": "H5T_IEEE_F32LE", "shape": [40, 80]} - payload["creationProperties"] = { - "filters": [ - compressor, - ] - } - req = self.endpoint + "/datasets" - rsp = self.session.post(req, data=json.dumps(payload), headers=headers) - self.assertEqual(rsp.status_code, 400) # create dataset + payload = {"type": "H5T_IEEE_F32LE", "shape": [40, 80]} + payload["creationProperties"] = {"filters": [filter_item, ]} + req = self.endpoint + "/datasets" + rsp = self.session.post(req, data=json.dumps(payload), headers=headers) + self.assertEqual(rsp.status_code, 400) # create dataset def testInvalidFillValue(self): # test Dataset with simple type and fill value that is incompatible with the type @@ -1604,26 +1900,18 @@ def testAutoChunk1dDataset(self): req = self.endpoint + "/datasets" # 50K x 80K dataset extent = 1000 * 1000 * 1000 - dims = [ - extent, - ] + dims = [extent, ] fields = ( {"name": "x", "type": "H5T_IEEE_F64LE"}, {"name": "y", "type": "H5T_IEEE_F64LE"}, {"name": "z", "type": "H5T_IEEE_F64LE"}, ) datatype = {"class": "H5T_COMPOUND", "fields": fields} + item_size = 12 # 3 fields of 4 bytes each + cpl = {"fillValue": 3.12} # no layout given + + payload = {"type": datatype, "shape": dims, "creationProperties": cpl} - payload = {"type": datatype, "shape": dims} - # the following should get ignored as too small - payload["creationProperties"] = { - "layout": { - "class": "H5D_CHUNKED", - "dims": [ - 10, - ], - } - } req = self.endpoint + "/datasets" rsp = self.session.post(req, data=json.dumps(payload), headers=headers) self.assertEqual(rsp.status_code, 201) # create dataset @@ -1644,17 +1932,23 @@ def testAutoChunk1dDataset(self): rsp = self.session.get(req, headers=headers) self.assertEqual(rsp.status_code, 200) rspJson = json.loads(rsp.text) - self.assertTrue("layout" in rspJson) - layout_json = rspJson["layout"] + self.assertTrue("creationProperties" in rspJson) + cpl = rspJson["creationProperties"] + self.assertTrue("fillValue" in cpl) + self.assertTrue("layout" in cpl) + layout_json = cpl["layout"] self.assertTrue("class" in layout_json) self.assertEqual(layout_json["class"], "H5D_CHUNKED") self.assertTrue("dims" in layout_json) - self.assertTrue("partition_count" not in layout_json) - layout = layout_json["dims"] - self.assertEqual(len(layout), 1) - self.assertTrue(layout[0] < dims[0]) - chunk_size = layout[0] * 8 * 3 # three 64bit - # chunk size should be between chunk min and max + chunk_dims = layout_json["dims"] + self.assertEqual(len(chunk_dims), 1) + self.assertTrue(chunk_dims[0] < dims[0]) + + chunk_size = chunk_dims[0] * item_size + + # chunk size will be based on server config, but assume a min/max of 1MB to 1GB + CHUNK_MIN = 1024 * 1024 + CHUNK_MAX = 1024 * 1024 * 1024 self.assertTrue(chunk_size >= CHUNK_MIN) self.assertTrue(chunk_size <= CHUNK_MAX) @@ -1675,6 +1969,7 @@ def testAutoChunk2dDataset(self): req = self.endpoint + "/datasets" # 50K x 80K dataset dims = [50000, 80000] + item_size = 4 # 4 bytes per float32 payload = {"type": "H5T_IEEE_F32LE", "shape": dims} req = self.endpoint + "/datasets" @@ -1697,74 +1992,22 @@ def testAutoChunk2dDataset(self): rsp = self.session.get(req, headers=headers) self.assertEqual(rsp.status_code, 200) rspJson = json.loads(rsp.text) - self.assertTrue("layout" in rspJson) - layout_json = rspJson["layout"] + self.assertTrue("creationProperties" in rspJson) + cpl = rspJson["creationProperties"] + self.assertTrue("layout" in cpl) + layout_json = cpl["layout"] self.assertTrue("class" in layout_json) self.assertEqual(layout_json["class"], "H5D_CHUNKED") self.assertTrue("dims" in layout_json) - layout = layout_json["dims"] - self.assertEqual(len(layout), 2) - self.assertTrue(layout[0] < dims[0]) - self.assertTrue(layout[1] < dims[1]) - chunk_size = layout[0] * layout[1] * 4 - # chunk size should be between chunk min and max - self.assertTrue(chunk_size >= CHUNK_MIN) - self.assertTrue(chunk_size <= CHUNK_MAX) - - def testMinChunkSizeDataset(self): - # test Dataset where chunk layout is adjusted if provided - # layout is too small - domain = self.base_domain + "/testMinChunkSizeDataset.h5" - helper.setupDomain(domain) - print("testMinChunkSizeDataset", domain) - headers = helper.getRequestHeaders(domain=domain) - # get domain - req = helper.getEndpoint() + "/" - rsp = self.session.get(req, headers=headers) - rspJson = json.loads(rsp.text) - self.assertTrue("root" in rspJson) - root_uuid = rspJson["root"] - - # create the dataset - req = self.endpoint + "/datasets" - # 50K x 80K dataset - dims = [50000, 80000] - payload = {"type": "H5T_IEEE_F32LE", "shape": dims} - # define a chunk layout with lots of small chunks - payload["creationProperties"] = { - "layout": {"class": "H5D_CHUNKED", "dims": [10, 10]} - } - - req = self.endpoint + "/datasets" - rsp = self.session.post(req, data=json.dumps(payload), headers=headers) - self.assertEqual(rsp.status_code, 201) # create dataset - rspJson = json.loads(rsp.text) - dset_uuid = rspJson["id"] - self.assertTrue(helper.validateId(dset_uuid)) - - # link new dataset as 'dset' - name = "dset" + helper.getRandomName() - req = self.endpoint + "/groups/" + root_uuid + "/links/" + name - payload = {"id": dset_uuid} - rsp = self.session.put(req, data=json.dumps(payload), headers=headers) - self.assertEqual(rsp.status_code, 201) + chunk_dims = layout_json["dims"] + self.assertEqual(len(chunk_dims), 2) + self.assertTrue(chunk_dims[0] < dims[0]) + self.assertTrue(chunk_dims[1] < dims[1]) + chunk_size = chunk_dims[0] * chunk_dims[1] * item_size - # verify layout - req = helper.getEndpoint() + "/datasets/" + dset_uuid - rsp = self.session.get(req, headers=headers) - self.assertEqual(rsp.status_code, 200) - rspJson = json.loads(rsp.text) - self.assertTrue("layout" in rspJson) - layout_json = rspJson["layout"] - self.assertTrue("class" in layout_json) - self.assertEqual(layout_json["class"], "H5D_CHUNKED") - self.assertTrue("dims" in layout_json) - layout = layout_json["dims"] - self.assertEqual(len(layout), 2) - self.assertTrue(layout[0] < dims[0]) - self.assertTrue(layout[1] < dims[1]) - chunk_size = layout[0] * layout[1] * 4 - # chunk size should be between chunk min and max + # chunk size will be based on server config, but assume a min/max of 1MB to 1GB + CHUNK_MIN = 1024 * 1024 + CHUNK_MAX = 1024 * 1024 * 1024 self.assertTrue(chunk_size >= CHUNK_MIN) self.assertTrue(chunk_size <= CHUNK_MAX) @@ -2059,7 +2302,7 @@ def testDatasetwithDomainDelete(self): rspJson = json.loads(rsp.text) self.assertEqual(rspJson["linkCount"], 1) - # delete the domain (with the orginal user) + # delete the domain (with the original user) req = helper.getEndpoint() + "/" rsp = self.session.delete(req, headers=headers) self.assertEqual(rsp.status_code, 200) @@ -2149,17 +2392,13 @@ def testContiguousRefDataset(self): rsp = self.session.get(req, headers=headers) self.assertEqual(rsp.status_code, 200) rspJson = json.loads(rsp.text) - self.assertTrue("layout" in rspJson) - layout_json = rspJson["layout"] + self.assertTrue("creationProperties" in rspJson) + cpl = rspJson["creationProperties"] + self.assertTrue("layout" in cpl) + layout_json = cpl["layout"] self.assertTrue("class" in layout_json) - self.assertEqual(layout_json["class"], "H5D_CHUNKED") - self.assertTrue("dims" in layout_json) - chunk_dims = layout_json["dims"] - self.assertEqual(len(chunk_dims), 2) - chunk_size = chunk_dims[0] * chunk_dims[1] * 4 - # chunk size should be between chunk min and max - self.assertTrue(chunk_size >= CHUNK_MIN) - self.assertTrue(chunk_size <= CHUNK_MAX) + self.assertEqual(layout_json["class"], "H5D_CONTIGUOUS_REF") + self.assertFalse("dims" in layout_json) # verify cpl self.assertTrue("creationProperties" in rspJson) @@ -2222,23 +2461,13 @@ def testContiguousRefZeroDimDataset(self): rsp = self.session.put(req, data=json.dumps(payload), headers=headers) self.assertEqual(rsp.status_code, 201) - # verify layout + # get dataset json req = helper.getEndpoint() + "/datasets/" + dset_uuid rsp = self.session.get(req, headers=headers) self.assertEqual(rsp.status_code, 200) rspJson = json.loads(rsp.text) - self.assertTrue("layout" in rspJson) - layout_json = rspJson["layout"] - self.assertTrue("class" in layout_json) - self.assertEqual(layout_json["class"], "H5D_CHUNKED") - self.assertTrue("dims" in layout_json) - chunk_dims = layout_json["dims"] - self.assertEqual(len(chunk_dims), 2) - # layout should be same as the dims - self.assertEqual(chunk_dims[0], dims[0]) - self.assertEqual(chunk_dims[1], dims[1]) - # verify cpl + # verify layout self.assertTrue("creationProperties" in rspJson) cpl = rspJson["creationProperties"] self.assertTrue("layout" in cpl) @@ -2312,13 +2541,6 @@ def testChunkedRefDataset(self): rsp = self.session.get(req, headers=headers) self.assertEqual(rsp.status_code, 200) rspJson = json.loads(rsp.text) - self.assertTrue("layout" in rspJson) - layout_json = rspJson["layout"] - self.assertTrue("class" in layout_json) - self.assertEqual(layout_json["class"], "H5D_CHUNKED") - self.assertTrue("dims" in layout_json) - chunk_dims = layout_json["dims"] - self.assertEqual(len(chunk_dims), 2) self.assertTrue("creationProperties" in rspJson) cpl = rspJson["creationProperties"] self.assertTrue("layout" in cpl) @@ -2391,21 +2613,15 @@ def testChunkedRefIndirectDataset(self): rsp = self.session.put(req, data=json.dumps(payload), headers=headers) self.assertEqual(rsp.status_code, 201) - # verify layout + # get dataset json req = helper.getEndpoint() + "/datasets/" + dset_uuid rsp = self.session.get(req, headers=headers) self.assertEqual(rsp.status_code, 200) rspJson = json.loads(rsp.text) - self.assertTrue("layout" in rspJson) - layout_json = rspJson["layout"] - self.assertTrue("class" in layout_json) - self.assertEqual(layout_json["class"], "H5D_CHUNKED") - self.assertTrue("chunks" not in layout_json) - chunk_dims = layout_json["dims"] - self.assertEqual(len(chunk_dims), 2) self.assertTrue("creationProperties" in rspJson) cpl = rspJson["creationProperties"] + self.assertTrue("layout") self.assertTrue("layout" in cpl) cpl_layout = cpl["layout"] self.assertTrue("class" in cpl_layout) @@ -2487,19 +2703,11 @@ def testChunkedRefIndirectS3UriDataset(self): rsp = self.session.put(req, data=json.dumps(payload), headers=headers) self.assertEqual(rsp.status_code, 201) - # verify layout + # fetch dataset json req = helper.getEndpoint() + "/datasets/" + dset_uuid rsp = self.session.get(req, headers=headers) self.assertEqual(rsp.status_code, 200) rspJson = json.loads(rsp.text) - self.assertTrue("layout" in rspJson) - layout_json = rspJson["layout"] - self.assertTrue("class" in layout_json) - self.assertEqual(layout_json["class"], "H5D_CHUNKED") - self.assertTrue("chunks" not in layout_json) - self.assertTrue("dims" in layout_json) - chunk_dims = layout_json["dims"] - self.assertEqual(len(chunk_dims), 2) self.assertTrue("creationProperties" in rspJson) cpl = rspJson["creationProperties"] @@ -2553,8 +2761,10 @@ def testDatasetChunkPartitioning(self): rsp = self.session.get(req, headers=headers) self.assertEqual(rsp.status_code, 200) rspJson = json.loads(rsp.text) - self.assertTrue("layout" in rspJson) - layout_json = rspJson["layout"] + self.assertTrue("creationProperties" in rspJson) + cpl = rspJson["creationProperties"] + self.assertTrue("layout" in cpl) + layout_json = cpl["layout"] self.assertTrue("class" in layout_json) self.assertEqual(layout_json["class"], "H5D_CHUNKED") self.assertTrue("dims" in layout_json) @@ -2570,10 +2780,6 @@ def testDatasetChunkPartitioning(self): self.assertTrue(layout[0] < dims[0]) self.assertTrue(layout[1] < dims[1]) self.assertTrue(layout[2] < dims[2]) - chunk_size = layout[0] * layout[1] * layout[2] * 4 - # chunk size should be between chunk min and max - self.assertTrue(chunk_size >= CHUNK_MIN) - self.assertTrue(chunk_size <= CHUNK_MAX) def testExtendibleDatasetChunkPartitioning(self): # test Dataset partitioning logic for large datasets @@ -2592,9 +2798,17 @@ def testExtendibleDatasetChunkPartitioning(self): req = self.endpoint + "/datasets" # 50K x 80K x 90K dataset dims = [0, 80000, 90000] - # unlimited extend in dim 0, fixeed in dimension 2, extenbile by 10x in dim 3 + + # unlimited extend in dim 0, fixed in dimension 2, extensible by 10x in dim 3 max_dims = [0, 80000, 900000] + chunk_shape = [1000, 1000, 1000] + layout = { + "class": "H5D_CHUNKED", + "dims": chunk_shape + } + cpl = {"layout": layout} payload = {"type": "H5T_IEEE_F32LE", "shape": dims, "maxdims": max_dims} + payload["creationProperties"] = cpl req = self.endpoint + "/datasets" rsp = self.session.post(req, data=json.dumps(payload), headers=headers) @@ -2616,8 +2830,10 @@ def testExtendibleDatasetChunkPartitioning(self): rsp = self.session.get(req, headers=headers) self.assertEqual(rsp.status_code, 200) rspJson = json.loads(rsp.text) - self.assertTrue("layout" in rspJson) - layout_json = rspJson["layout"] + self.assertTrue("creationProperties" in rspJson) + cpl = rspJson["creationProperties"] + self.assertTrue("layout" in cpl) + layout_json = cpl["layout"] self.assertTrue("class" in layout_json) self.assertEqual(layout_json["class"], "H5D_CHUNKED") self.assertTrue("dims" in layout_json) @@ -2627,10 +2843,6 @@ def testExtendibleDatasetChunkPartitioning(self): layout = layout_json["dims"] self.assertEqual(len(layout), 3) - chunk_size = layout[0] * layout[1] * layout[2] * 4 - # chunk size should be between chunk min and max - self.assertTrue(chunk_size >= CHUNK_MIN) - self.assertTrue(chunk_size <= CHUNK_MAX) def testDatasetEmptyChunkExtent(self): # Attempting to create 0-extent chunks should respond with Bad Request @@ -2656,6 +2868,87 @@ def testDatasetEmptyChunkExtent(self): # Should fail with Bad Request due to invalid layout value self.assertEqual(rsp.status_code, 400) # create dataset + def testDatasetPostMulti(self): + # test POST with multi-object creation + domain = self.base_domain + "/testDatasetPostMulti.h5" + helper.setupDomain(domain) + print("testDatasetPostMulti", domain) + headers = helper.getRequestHeaders(domain=domain) + req = helper.getEndpoint() + "/" + + # get root ids + req = helper.getEndpoint() + "/" + rsp = self.session.get(req, headers=headers) + self.assertEqual(rsp.status_code, 200) + rspJson = json.loads(rsp.text) + root_uuid = rspJson["root"] + helper.validateId(root_uuid) + + # get root group and verify link count is 0 + req = helper.getEndpoint() + "/groups/" + root_uuid + rsp = self.session.get(req, headers=headers) + self.assertEqual(rsp.status_code, 200) + rspJson = json.loads(rsp.text) + self.assertEqual(rspJson["linkCount"], 0) + + dataset_count = 3 + datatype = "H5T_STD_I32LE" + payload = [] + for _ in range(dataset_count): + dataset_args = {"type": datatype} + payload.append(dataset_args) + + req = helper.getEndpoint() + "/datasets" + rsp = self.session.post(req, data=json.dumps(payload), headers=headers) + self.assertEqual(rsp.status_code, 201) + rspJson = json.loads(rsp.text) + self.assertTrue("objects" in rspJson) + rsp_objs = rspJson["objects"] + self.assertEqual(len(rsp_objs), dataset_count) + + expected_keys = [ + "id", + "shape", + "attributeCount", + "created", + "lastModified", + "root", + ] + + for i in range(dataset_count): + obj_json = rsp_objs[i] + self.assertEqual(obj_json["attributeCount"], 0) + dset_id = obj_json["id"] + self.assertTrue(helper.validateId(dset_id)) + self.assertTrue(dset_id.startswith("d-")) + for key in expected_keys: + self.assertTrue(key in obj_json) + + # create a set of linked datasets + for i in range(dataset_count): + item = payload[i] + item["link"] = {"id": root_uuid, "name": f"dset_{i + 1}"} + rsp = self.session.post(req, data=json.dumps(payload), headers=headers) + self.assertEqual(rsp.status_code, 201) + rspJson = json.loads(rsp.text) + self.assertTrue("objects" in rspJson) + rsp_objs = rspJson["objects"] + self.assertEqual(len(rsp_objs), dataset_count) + for i in range(dataset_count): + json_rsp = rsp_objs[i] + self.assertEqual(json_rsp["attributeCount"], 0) + dset_id = json_rsp["id"] + self.assertTrue(helper.validateId(dset_id)) + for key in expected_keys: + self.assertTrue(key in obj_json) + + # get root group and verify link count is dataset_count + req = helper.getEndpoint() + "/groups/" + root_uuid + rsp = self.session.get(req, headers=headers) + self.assertEqual(rsp.status_code, 200) + rspJson = json.loads(rsp.text) + self.assertEqual(rspJson["linkCount"], dataset_count) + if __name__ == "__main__": # setup test files diff --git a/tests/integ/datatype_test.py b/tests/integ/datatype_test.py index f3f2d1a9..7bf90d09 100755 --- a/tests/integ/datatype_test.py +++ b/tests/integ/datatype_test.py @@ -11,6 +11,9 @@ ############################################################################## import unittest import json + +from h5json.objid import createObjId + import helper import config @@ -120,6 +123,108 @@ def testCommittedType(self): rsp = self.session.get(req, headers=headers) self.assertEqual(rsp.status_code, 410) + def testPostTypeWithId(self): + # Test creation/deletion of datatype obj + + print("testPostTypeWithId", self.base_domain) + headers = helper.getRequestHeaders(domain=self.base_domain) + req = self.endpoint + "/" + + # Get root uuid + rsp = self.session.get(req, headers=headers) + self.assertEqual(rsp.status_code, 200) + rspJson = json.loads(rsp.text) + root_uuid = rspJson["root"] + helper.validateId(root_uuid) + + # create a datatype id + ctype_id = createObjId("datatypes", root_id=root_uuid) + + # try creating a committed type without a type in the body + req = self.endpoint + "/datatypes" + data = {"id": ctype_id} + rsp = self.session.post(req, data=json.dumps(data), headers=headers) + self.assertEqual(rsp.status_code, 400) # bad request + + # create a committed type obj + data = {"id": ctype_id, "type": "H5T_IEEE_F32LE"} + rsp = self.session.post(req, data=json.dumps(data), headers=headers) + self.assertEqual(rsp.status_code, 201) + rspJson = json.loads(rsp.text) + self.assertEqual(rspJson["attributeCount"], 0) + self.assertEqual(rspJson["id"], ctype_id) + self.assertTrue("type" in rspJson) + type_json = rspJson["type"] + self.assertEqual(type_json["class"], "H5T_FLOAT") + self.assertEqual(type_json["base"], "H5T_IEEE_F32LE") + + # read back the obj + req = self.endpoint + "/datatypes/" + ctype_id + rsp = self.session.get(req, headers=headers) + self.assertEqual(rsp.status_code, 200) + rspJson = json.loads(rsp.text) + self.assertTrue("id" in rspJson) + self.assertEqual(rspJson["id"], ctype_id) + self.assertTrue("root" in rspJson) + self.assertEqual(rspJson["root"], root_uuid) + self.assertTrue("created" in rspJson) + self.assertTrue("lastModified" in rspJson) + self.assertTrue("attributeCount" in rspJson) + self.assertEqual(rspJson["attributeCount"], 0) + self.assertTrue("type" in rspJson) + type_json = rspJson["type"] + self.assertEqual(type_json["class"], "H5T_FLOAT") + self.assertEqual(type_json["base"], "H5T_IEEE_F32LE") + + def testPostWithAttributes(self): + # test POST with attribute initialization + print("testPostWithAttributes", self.base_domain) + headers = helper.getRequestHeaders(domain=self.base_domain) + + # get root id + req = helper.getEndpoint() + "/" + rsp = self.session.get(req, headers=headers) + self.assertEqual(rsp.status_code, 200) + rspJson = json.loads(rsp.text) + root_uuid = rspJson["root"] + helper.validateId(root_uuid) + + # setup some attributes to include + attr_count = 4 + attributes = {} + extent = 10 + for i in range(attr_count): + value = [i * 10 + j for j in range(extent)] + data = {"type": "H5T_STD_I32LE", "shape": extent, "value": value} + attr_name = f"attr{i + 1:04d}" + attributes[attr_name] = data + + # create new datatype + link = {"id": root_uuid, "name": "linked_datatype"} + payload = {"type": "H5T_IEEE_F32LE", "attributes": attributes, "link": link} + req = helper.getEndpoint() + "/datatypes" + rsp = self.session.post(req, data=json.dumps(payload), headers=headers) + self.assertEqual(rsp.status_code, 201) + rspJson = json.loads(rsp.text) + ctype_id = rspJson["id"] + self.assertTrue(helper.validateId(ctype_id)) + self.assertTrue("type" in rspJson) + type_json = rspJson["type"] + self.assertEqual(type_json["class"], "H5T_FLOAT") + self.assertEqual(type_json["base"], "H5T_IEEE_F32LE") + self.assertEqual(rspJson["attributeCount"], attr_count) + + # fetch the attributes, check count + req = f"{helper.getEndpoint()}/datatypes/{ctype_id}/attributes" + rsp = self.session.get(req, headers=headers) + self.assertEqual(rsp.status_code, 200) + rspJson = json.loads(rsp.text) + self.assertTrue("hrefs" in rspJson) + self.assertFalse("type" in rspJson) + self.assertFalse("shape" in rspJson) + self.assertTrue("attributes") in rspJson + self.assertEqual(len(rspJson["attributes"]), attr_count) + def testPostTypes(self): # Test creation with all primitive types @@ -370,6 +475,7 @@ def testPostWithLink(self): rsp = self.session.get(req, headers=headers) self.assertEqual(rsp.status_code, 200) # link doesn't exist yet rspJson = json.loads(rsp.text) + self.assertTrue("link" in rspJson) link_json = rspJson["link"] self.assertEqual(link_json["collection"], "datatypes") @@ -509,6 +615,80 @@ def testPostWithPath(self): rspJson = json.loads(rsp.text) self.assertEqual(rspJson["id"], new_datatype_id) + def testPostMulti(self): + # test POST with multi-object creation + print("testPostMulti", self.base_domain) + headers = helper.getRequestHeaders(domain=self.base_domain) + + # get root id + req = helper.getEndpoint() + "/" + rsp = self.session.get(req, headers=headers) + self.assertEqual(rsp.status_code, 200) + rspJson = json.loads(rsp.text) + root_uuid = rspJson["root"] + helper.validateId(root_uuid) + + # get root group and verify link count is 0 + req = helper.getEndpoint() + "/groups/" + root_uuid + rsp = self.session.get(req, headers=headers) + self.assertEqual(rsp.status_code, 200) + rspJson = json.loads(rsp.text) + self.assertEqual(rspJson["linkCount"], 0) + + str_type = { + "charSet": "H5T_CSET_ASCII", + "class": "H5T_STRING", + "length": 12, + "strPad": "H5T_STR_NULLPAD", + } + + float_type = "H5T_IEEE_F32LE" + + # create a set of anonymous ctypes + fields = ( + {"name": "temp", "type": "H5T_STD_I32LE"}, + {"name": "pressure", "type": "H5T_IEEE_F32LE"}, + ) + compound_type = {"class": "H5T_COMPOUND", "fields": fields} + + payload = [{"type": str_type}, {"type": float_type}, {"type": compound_type}] + req = helper.getEndpoint() + "/datatypes" + rsp = self.session.post(req, data=json.dumps(payload), headers=headers) + self.assertEqual(rsp.status_code, 201) + rspJson = json.loads(rsp.text) + self.assertTrue("objects" in rspJson) + rsp_objs = rspJson["objects"] + self.assertEqual(len(rsp_objs), 3) + + for i in range(3): + obj_json = rsp_objs[i] + self.assertEqual(obj_json["attributeCount"], 0) + ctype_id = obj_json["id"] + self.assertTrue(helper.validateId(ctype_id)) + + # create a set of linked ctypes + for i in range(3): + item = payload[i] + item["link"] = {"id": root_uuid, "name": f"ctype_{i + 1}"} + rsp = self.session.post(req, data=json.dumps(payload), headers=headers) + self.assertEqual(rsp.status_code, 201) + rspJson = json.loads(rsp.text) + self.assertTrue("objects" in rspJson) + rsp_objs = rspJson["objects"] + self.assertEqual(len(rsp_objs), 3) + for i in range(3): + json_rsp = rsp_objs[i] + self.assertEqual(json_rsp["attributeCount"], 0) + ctype_id = json_rsp["id"] + self.assertTrue(helper.validateId(ctype_id)) + + # get root group and verify link count is 3 + req = helper.getEndpoint() + "/groups/" + root_uuid + rsp = self.session.get(req, headers=headers) + self.assertEqual(rsp.status_code, 200) + rspJson = json.loads(rsp.text) + self.assertEqual(rspJson["linkCount"], 3) + if __name__ == "__main__": # setup test files diff --git a/tests/integ/domain_test.py b/tests/integ/domain_test.py index 4f21d44e..5a64e187 100755 --- a/tests/integ/domain_test.py +++ b/tests/integ/domain_test.py @@ -13,6 +13,9 @@ import time import json from os import path as pp + +from h5json.objid import createObjId, getCollectionForId + import config import helper @@ -113,31 +116,19 @@ def testGetDomain(self): attr_count = 0 for objid in domain_objs: obj_json = domain_objs[objid] - self.assertTrue("id" in obj_json) - self.assertTrue("attributeCount" in obj_json) - attr_count += obj_json["attributeCount"] - self.assertFalse("attributes" in obj_json) - - self.assertEqual(attr_count, 4) + collection_type = getCollectionForId(objid) + if collection_type == "datasets": + self.assertTrue("attributes" in obj_json) + self.assertTrue("type" in obj_json) + self.assertTrue("shape" in obj_json) + self.assertTrue("creationProperties" in obj_json) + elif collection_type == "groups": + self.assertTrue("attributes" in obj_json) + self.assertTrue("links" in obj_json) + else: + self.assertTrue(False) # unexpected type + attr_count += len(obj_json["attributes"]) - # get a dict of all objects in the domain including any attributes - params["include_attrs"] = 1 - rsp = self.session.get(req, headers=headers, params=params) - self.assertEqual(rsp.status_code, 200) - rspJson = json.loads(rsp.text) - self.assertTrue("domain_objs" in rspJson) - domain_objs = rspJson["domain_objs"] - self.assertEqual(len(domain_objs), 10) - attr_count = 0 - for objid in domain_objs: - obj_json = domain_objs[objid] - self.assertTrue("attributeCount" in obj_json) - self.assertTrue("attributes" in obj_json) - attributes = obj_json["attributes"] - for attr_name in attributes: - # only the names "attr1" and "attr2" are used in this domain - self.assertTrue(attr_name in ("attr1", "attr2")) - attr_count += 1 self.assertEqual(attr_count, 4) # passing domain via the host header is deprecated @@ -489,6 +480,94 @@ def testCreateDomain(self): self.assertTrue(k in rspJson) # we should get the same value for root id self.assertEqual(root_id, rspJson["root"]) + + def testCreateDomainWithId(self): + domain = self.base_domain + "/newdomainwithid.h5" + print("testCreateDomainWithId", domain) + headers = helper.getRequestHeaders(domain=domain) + + root_id = createObjId("groups") + body = {"root_id": root_id} + req = helper.getEndpoint() + "/" + + rsp = self.session.put(req, data=json.dumps(body), headers=headers) + self.assertEqual(rsp.status_code, 201) + rspJson = json.loads(rsp.text) + for k in ( + "root", + "owner", + "acls", + "created", + "lastModified", + "version", + "limits", + "compressors", + ): + self.assertTrue(k in rspJson) + + self.assertEqual(rspJson["root"], root_id) + + limit_keys = ("min_chunk_size", "max_chunk_size", "max_request_size") + limits = rspJson["limits"] + for k in limit_keys: + self.assertTrue(k in limits) + limit = limits[k] + self.assertTrue(isinstance(limit, int)) + self.assertTrue(limit > 0) + compressors = rspJson["compressors"] + for compressor in EXPECTED_COMPRESSORS: + self.assertTrue(compressor in compressors) + + # do a get on the new domain + rsp = self.session.get(req, headers=headers) + self.assertEqual(rsp.status_code, 200) + rspJson = json.loads(rsp.text) + for k in ( + "root", + "owner", + "class", + "created", + "lastModified", + "limits", + "version", + ): + self.assertTrue(k in rspJson) + # we should get the same value for root id + self.assertEqual(root_id, rspJson["root"]) + # should get limits here too + limits = rspJson["limits"] + for k in limit_keys: + self.assertTrue(k in limits) + limit = limits[k] + self.assertTrue(isinstance(limit, int)) + self.assertTrue(limit > 0) + + # verify we can access root groups + root_req = helper.getEndpoint() + "/groups/" + root_id + headers = helper.getRequestHeaders(domain=domain) + rsp = self.session.get(root_req, headers=headers) + self.assertEqual(rsp.status_code, 200) + + # verify that putting the same domain again fails with a 409 error + rsp = self.session.put(req, headers=headers) + self.assertEqual(rsp.status_code, 409) + + # PUT with a different domain name should also give a 409 + # (due to the root_id conflicting) + domain2 = self.base_domain + "/newdomainwithid2.h5" + headers2 = helper.getRequestHeaders(domain=domain2) + rsp = self.session.put(req, data=json.dumps(body), headers=headers2) + self.assertEqual(rsp.status_code, 409) + + # Delete the original domain + headers = helper.getRequestHeaders(domain=domain) + rsp = self.session.delete(req, headers=headers) + self.assertEqual(rsp.status_code, 200) + + # re-create the domain with the same root id + rsp = self.session.put(req, data=json.dumps(body), headers=headers) + self.assertEqual(rsp.status_code, 201) + """ def testCreateDomainWithCustomClass(self): domain = self.base_domain + "/newclassdomain.h6" @@ -683,6 +762,13 @@ def testCreateFolder(self): rsp = self.session.put(req, data=json.dumps(body), headers=headers) self.assertEqual(rsp.status_code, 409) + # rescan doesn't make sense for a folder domain (no root group to + # scan) - should get a 400 rather than falling through to domain + # creation logic (which would 409 as above) + params = {"rescan": 1} + rsp = self.session.put(req, params=params, headers=headers) + self.assertEqual(rsp.status_code, 400) + # do a get on the new folder rsp = self.session.get(req, headers=headers) self.assertEqual(rsp.status_code, 200) @@ -921,7 +1007,7 @@ def testDeleteDomain(self): else: print("user2_name not set") - # delete the domain (with the orginal user) + # delete the domain (with the original user) headers = helper.getRequestHeaders(domain=domain) rsp = self.session.delete(req, headers=headers) self.assertEqual(rsp.status_code, 200) @@ -1362,12 +1448,24 @@ def testGetDomains(self): domains = rspJson["domains"] self.assertEqual(len(domains), 3) - # bad query expression - query = "atttr1 > 7 AND" + # bad query expression (dangling AND with no right-hand operand - + # use a real attribute name here so this tests the grammar error + # specifically, not just a missing/unknown attribute) + query = "attr1 > 7 AND" params = {"domain": folder + "/", "query": query} rsp = self.session.get(req, params=params, headers=headers) self.assertEqual(rsp.status_code, 400) + # query referencing an attribute that doesn't exist on any domain + # is not an error - it just matches nothing + query = "no_such_attr > 7" + params = {"domain": folder + "/", "query": query} + rsp = self.session.get(req, params=params, headers=headers) + self.assertEqual(rsp.status_code, 200) + rspJson = json.loads(rsp.text) + self.assertTrue("domains" in rspJson) + self.assertEqual(len(rspJson["domains"]), 0) + # empty sub-domains domain = helper.getTestDomain("tall.h5") + "/" params = {"domain": domain} diff --git a/tests/integ/filter_test.py b/tests/integ/filter_test.py index ea2df637..4dee49c7 100755 --- a/tests/integ/filter_test.py +++ b/tests/integ/filter_test.py @@ -58,6 +58,8 @@ def testDeflateCompression(self): # Create ~1MB dataset payload = {"type": "H5T_STD_I8LE", "shape": [1024, 1024]} + # use a chunked layout for compression + layout = {"class": "H5D_CHUNKED", "dims": [64, 64]} # define deflate compression gzip_filter = { "class": "H5Z_FILTER_DEFLATE", @@ -65,7 +67,7 @@ def testDeflateCompression(self): "level": 9, "name": "deflate", } - payload["creationProperties"] = {"filters": [gzip_filter]} + payload["creationProperties"] = {"layout": layout, "filters": [gzip_filter]} req = self.endpoint + "/datasets" rsp = self.session.post(req, data=json.dumps(payload), headers=headers) self.assertEqual(rsp.status_code, 201) # create dataset @@ -120,7 +122,9 @@ def testShuffleFilter(self): payload = {"type": "H5T_STD_I32LE", "shape": [1024, 1024]} # define sshufle compression shuffle_filter = {"class": "H5Z_FILTER_SHUFFLE", "id": 2, "name": "shuffle"} - payload["creationProperties"] = {"filters": [shuffle_filter]} + # use chunked layout for compression + layout = {"class": "H5D_CHUNKED", "dims": [64, 64]} + payload["creationProperties"] = {"filters": [shuffle_filter], "layout": layout} req = self.endpoint + "/datasets" rsp = self.session.post(req, data=json.dumps(payload), headers=headers) self.assertEqual(rsp.status_code, 201) # create dataset @@ -182,7 +186,11 @@ def testShuffleAndDeflate(self): } # and shuffle compression shuffle_filter = {"class": "H5Z_FILTER_SHUFFLE", "id": 2, "name": "shuffle"} - payload["creationProperties"] = {"filters": [shuffle_filter, gzip_filter]} + filters = [shuffle_filter, gzip_filter] + # use chunked layout + layout = {"class": "H5D_CHUNKED", "dims": [64, 64]} + payload["creationProperties"] = {"layout": layout, "filters": filters} + req = self.endpoint + "/datasets" rsp = self.session.post(req, data=json.dumps(payload), headers=headers) self.assertEqual(rsp.status_code, 201) # create dataset @@ -237,7 +245,9 @@ def testBitShuffle(self): # bit shuffle bitshuffle_filter = {"class": "H5Z_FILTER_BITSHUFFLE", "id": 32008, "name": "bitshuffle"} - payload["creationProperties"] = {"filters": [bitshuffle_filter, ]} + # use chunked layout + layout = {"class": "H5D_CHUNKED", "dims": [64, 64]} + payload["creationProperties"] = {"filters": [bitshuffle_filter], "layout": layout} req = self.endpoint + "/datasets" rsp = self.session.post(req, data=json.dumps(payload), headers=headers) self.assertEqual(rsp.status_code, 201) # create dataset @@ -299,7 +309,68 @@ def testBitShuffleAndDeflate(self): } # and bit shuffle bitshuffle_filter = {"class": "H5Z_FILTER_BITSHUFFLE", "id": 32008, "name": "bitshuffle"} - payload["creationProperties"] = {"filters": [bitshuffle_filter, gzip_filter]} + filters = [bitshuffle_filter, gzip_filter] + # use chunked layout + layout = {"class": "H5D_CHUNKED", "dims": [64, 64]} + payload["creationProperties"] = {"filters": filters, "layout": layout} + req = self.endpoint + "/datasets" + rsp = self.session.post(req, data=json.dumps(payload), headers=headers) + self.assertEqual(rsp.status_code, 201) # create dataset + rspJson = json.loads(rsp.text) + dset_uuid = rspJson["id"] + self.assertTrue(helper.validateId(dset_uuid)) + + # link new dataset as 'dset' + name = "dset" + helper.getRandomName() + req = self.endpoint + "/groups/" + root_uuid + "/links/" + name + payload = {"id": dset_uuid} + rsp = self.session.put(req, data=json.dumps(payload), headers=headers) + self.assertEqual(rsp.status_code, 201) + + # write a horizontal strip of 22s + req = self.endpoint + "/datasets/" + dset_uuid + "/value" + data = [22] * 1024 + payload = {"start": [512, 0], "stop": [513, 1024], "value": data} + rsp = self.session.put(req, data=json.dumps(payload), headers=headers) + self.assertEqual(rsp.status_code, 200) + + # read back the 512,512 element + req = self.endpoint + "/datasets/" + dset_uuid + "/value" # test + params = {"select": "[512:513,512:513]"} # read 1 element + rsp = self.session.get(req, params=params, headers=headers) + self.assertEqual(rsp.status_code, 200) + rspJson = json.loads(rsp.text) + self.assertTrue("hrefs" in rspJson) + self.assertTrue("value" in rspJson) + value = rspJson["value"] + self.assertEqual(len(value), 1) + row = value[0] + self.assertEqual(len(row), 1) + self.assertEqual(row[0], 22) + + def testBloscCompression(self): + # test Dataset with creation property list + print("testBloscCompression", self.base_domain) + headers = helper.getRequestHeaders(domain=self.base_domain) + # get domain + req = helper.getEndpoint() + "/" + rsp = self.session.get(req, headers=headers) + rspJson = json.loads(rsp.text) + self.assertTrue("root" in rspJson) + root_uuid = rspJson["root"] + + # create the dataset + req = self.endpoint + "/datasets" + + # Create ~4MB dataset + + payload = {"type": "H5T_STD_I32LE", "shape": [1024, 1024]} + # define blosc compression (blosclz is one of the compressors + # available via the blosc filter, as defined in h5json's FILTER_DEFS) + blosc_filter = {"class": "H5Z_FILTER_BLOSC", "id": 32001, "name": "blosclz"} + # use chunked layout for compression + layout = {"class": "H5D_CHUNKED", "dims": [64, 64]} + payload["creationProperties"] = {"filters": [blosc_filter], "layout": layout} req = self.endpoint + "/datasets" rsp = self.session.post(req, data=json.dumps(payload), headers=headers) self.assertEqual(rsp.status_code, 201) # create dataset @@ -461,6 +532,12 @@ def testBitDeshuffling(self): print("hdf5_sample_bucket config not set, skipping testShuffleFilter") return + else: + msg = "TBD: testBitDeshuffling - need to verify that the bitshuffle " + msg += "filter is working correctly" + print(msg) + return + # Get domain req = helper.getEndpoint() + "/" rsp = self.session.get(req, headers=headers) diff --git a/tests/integ/group_test.py b/tests/integ/group_test.py index 7a832271..ce617e4e 100755 --- a/tests/integ/group_test.py +++ b/tests/integ/group_test.py @@ -13,6 +13,9 @@ import time import json import uuid + +from h5json.objid import createObjId + import helper import config @@ -220,6 +223,22 @@ def testPost(self): self.assertTrue("alias" in rspJson) self.assertEqual(rspJson["alias"], []) + # try with an empty body + payload = {} + req = endpoint + "/groups" + rsp = self.session.post(req, data=json.dumps(payload), headers=headers) + self.assertEqual(rsp.status_code, 201) + rspJson = json.loads(rsp.text) + self.assertEqual(rspJson["linkCount"], 0) + self.assertEqual(rspJson["attributeCount"], 0) + group_id = rspJson["id"] + self.assertTrue(helper.validateId(group_id)) + + # try with a type in body (as if we were trying to create a committed type) + payload["type"] = "H5T_IEEE_F32LE" + rsp = self.session.post(req, data=json.dumps(payload), headers=headers) + self.assertEqual(rsp.status_code, 400) + # try POST with user who doesn't have create permission on this domain test_user2 = config.get("user2_name") # some tests will be skipped if not set if not test_user2: @@ -233,6 +252,39 @@ def testPost(self): rsp = self.session.post(req, headers=headers) self.assertEqual(rsp.status_code, 403) # forbidden + def testPostWithId(self): + # test POST group with a client-generated id + print("testPostWithId", self.base_domain) + endpoint = helper.getEndpoint() + headers = helper.getRequestHeaders(domain=self.base_domain) + req = endpoint + "/groups" + + # get root id + req = helper.getEndpoint() + "/" + rsp = self.session.get(req, headers=headers) + self.assertEqual(rsp.status_code, 200) + rspJson = json.loads(rsp.text) + root_uuid = rspJson["root"] + helper.validateId(root_uuid) + + # create a group id + grp_id = createObjId("groups", root_id=root_uuid) + + # create a new group using the grp_id + payload = {"id": grp_id} + req = helper.getEndpoint() + "/groups" + rsp = self.session.post(req, data=json.dumps(payload), headers=headers) + + self.assertEqual(rsp.status_code, 201) + rspJson = json.loads(rsp.text) + self.assertEqual(rspJson["linkCount"], 0) + self.assertEqual(rspJson["attributeCount"], 0) + self.assertEqual(grp_id, rspJson["id"]) + + # try sending the same request again + rsp = self.session.post(req, data=json.dumps(payload), headers=headers) + self.assertEqual(rsp.status_code, 400) # bad request + def testPostWithLink(self): # test POST with link creation print("testPostWithLink", self.base_domain) @@ -279,7 +331,7 @@ def testPostWithLink(self): self.assertEqual(rspJson["linkCount"], 0) self.assertEqual(rspJson["attributeCount"], 0) new_group_id = rspJson["id"] - self.assertTrue(helper.validateId(rspJson["id"])) + self.assertTrue(helper.validateId(new_group_id)) self.assertTrue(new_group_id != root_uuid) # get root group and verify link count is 1 @@ -310,6 +362,145 @@ def testPostWithLink(self): self.assertTrue("alias" in rspJson) self.assertEqual(rspJson["alias"], ["/linked_group",]) + def testPostIdWithLink(self): + # test POST with link creation + print("testPostIdWithLink", self.base_domain) + headers = helper.getRequestHeaders(domain=self.base_domain) + + # get root id + req = helper.getEndpoint() + "/" + rsp = self.session.get(req, headers=headers) + self.assertEqual(rsp.status_code, 200) + rspJson = json.loads(rsp.text) + root_uuid = rspJson["root"] + helper.validateId(root_uuid) + + grp_count = 3 + req = helper.getEndpoint() + "/groups" + + for i in range(grp_count): + # create a group id + grp_id = createObjId("groups", root_id=root_uuid) + + # create new group + payload = {"id": grp_id, "link": {"id": root_uuid, "name": f"g{i:04d}"}} + rsp = self.session.post(req, data=json.dumps(payload), headers=headers) + self.assertEqual(rsp.status_code, 201) + rspJson = json.loads(rsp.text) + self.assertEqual(rspJson["linkCount"], 0) + self.assertEqual(rspJson["attributeCount"], 0) + self.assertEqual(grp_id, rspJson["id"]) + + # get root group and verify number of links + req = helper.getEndpoint() + "/groups/" + root_uuid + rsp = self.session.get(req, headers=headers) + self.assertEqual(rsp.status_code, 200) + rspJson = json.loads(rsp.text) + self.assertEqual(rspJson["linkCount"], grp_count) + + def testPostWithAttributes(self): + # test POST with attribute initialization + print("testPostWithAttributes", self.base_domain) + headers = helper.getRequestHeaders(domain=self.base_domain) + + # get root id + req = helper.getEndpoint() + "/" + rsp = self.session.get(req, headers=headers) + self.assertEqual(rsp.status_code, 200) + rspJson = json.loads(rsp.text) + root_uuid = rspJson["root"] + helper.validateId(root_uuid) + + # setup some attributes to include + attr_count = 4 + attributes = {} + extent = 10 + for i in range(attr_count): + value = [i * 10 + j for j in range(extent)] + data = {"type": "H5T_STD_I32LE", "shape": extent, "value": value} + attr_name = f"attr{i + 1:04d}" + attributes[attr_name] = data + + # create new group + payload = {"attributes": attributes, "link": {"id": root_uuid, "name": "linked_group"}} + req = helper.getEndpoint() + "/groups" + rsp = self.session.post(req, data=json.dumps(payload), headers=headers) + self.assertEqual(rsp.status_code, 201) + rspJson = json.loads(rsp.text) + self.assertEqual(rspJson["linkCount"], 0) + self.assertEqual(rspJson["attributeCount"], attr_count) + grp_id = rspJson["id"] + self.assertTrue(helper.validateId(grp_id)) + + # fetch the attributes, check count + req = f"{helper.getEndpoint()}/groups/{grp_id}/attributes" + rsp = self.session.get(req, headers=headers) + self.assertEqual(rsp.status_code, 200) + rspJson = json.loads(rsp.text) + self.assertTrue("hrefs" in rspJson) + self.assertFalse("type" in rspJson) + self.assertFalse("shape" in rspJson) + self.assertTrue("attributes") in rspJson + self.assertEqual(len(rspJson["attributes"]), attr_count) + + def testPostWithLinks(self): + # test POST with attribute initialization + print("testPostWithLinks", self.base_domain) + headers = helper.getRequestHeaders(domain=self.base_domain) + + # get root id + req = helper.getEndpoint() + "/" + rsp = self.session.get(req, headers=headers) + self.assertEqual(rsp.status_code, 200) + rspJson = json.loads(rsp.text) + root_uuid = rspJson["root"] + helper.validateId(root_uuid) + + # some objects to link + link_count = 4 + links = {} + req = helper.getEndpoint() + "/groups" + + for i in range(link_count): + rsp = self.session.post(req, headers=headers) + self.assertEqual(rsp.status_code, 201) + rspJson = json.loads(rsp.text) + group_id = rspJson["id"] + self.assertTrue(helper.validateId(group_id)) + links[f"obj_{i}"] = {"id": group_id} + + # create new group + payload = {"links": links, "link": {"id": root_uuid, "name": "g1"}} + req = helper.getEndpoint() + "/groups" + rsp = self.session.post(req, data=json.dumps(payload), headers=headers) + self.assertEqual(rsp.status_code, 201) + rspJson = json.loads(rsp.text) + self.assertEqual(rspJson["linkCount"], link_count) + self.assertEqual(rspJson["attributeCount"], 0) + grp_id = rspJson["id"] + helper.validateId(grp_id) + + # fetch all the links + req = helper.getEndpoint() + "/groups/" + grp_id + "/links" + rsp = self.session.get(req, headers=headers) + self.assertEqual(rsp.status_code, 200) + rspJson = json.loads(rsp.text) + + self.assertTrue("links" in rspJson) + links_rsp = rspJson["links"] + self.assertEqual(len(links_rsp), link_count) + for i in range(link_count): + link_rsp = links_rsp[i] + self.assertTrue("class" in link_rsp) + self.assertEqual(link_rsp["class"], "H5L_TYPE_HARD") + self.assertTrue("id" in link_rsp) + self.assertTrue("title" in link_rsp) + self.assertEqual(link_rsp["title"], f"obj_{i}") + self.assertTrue("collection" in link_rsp) + self.assertEqual(link_rsp["collection"], "groups") + self.assertTrue("target" in link_rsp) + self.assertTrue("href" in link_rsp) + def testPostWithPath(self): # test POST with implicit parent group creation print("testPostWithPath", self.base_domain) @@ -427,9 +618,92 @@ def testPostWithPath(self): rsp = self.session.get(req, headers=headers, params=params) self.assertEqual(rsp.status_code, 200) + def testPostIdWithPath(self): + # test POST with implicit parent group creation + print("testPostIdWithPath", self.base_domain) + headers = helper.getRequestHeaders(domain=self.base_domain) + + # get root id + req = helper.getEndpoint() + "/" + rsp = self.session.get(req, headers=headers) + self.assertEqual(rsp.status_code, 200) + rspJson = json.loads(rsp.text) + root_uuid = rspJson["root"] + helper.validateId(root_uuid) + + # get root group and verify link count is 0 + req = helper.getEndpoint() + "/groups/" + root_uuid + rsp = self.session.get(req, headers=headers) + self.assertEqual(rsp.status_code, 200) + rspJson = json.loads(rsp.text) + self.assertEqual(rspJson["linkCount"], 0) + + # create new group with link path: /g1 + g1_id = createObjId("groups", root_id=root_uuid) + payload = {"id": g1_id, "h5path": "g1"} + req = helper.getEndpoint() + "/groups" + rsp = self.session.post(req, data=json.dumps(payload), headers=headers) + self.assertEqual(rsp.status_code, 201) + rspJson = json.loads(rsp.text) + self.assertEqual(rspJson["linkCount"], 0) + self.assertEqual(rspJson["attributeCount"], 0) + self.assertEqual(rspJson["id"], g1_id) + + # get root group and verify link count is 1 + req = helper.getEndpoint() + "/groups/" + root_uuid + rsp = self.session.get(req, headers=headers) + self.assertEqual(rsp.status_code, 200) + rspJson = json.loads(rsp.text) + self.assertEqual(rspJson["linkCount"], 1) + + # get the group at "g1" + req = helper.getEndpoint() + "/groups/" + params = {"h5path": "/g1"} + rsp = self.session.get(req, headers=headers, params=params) + self.assertEqual(rsp.status_code, 200) + + # try creating new group with link path: /g2/g2.1 + g21_id = createObjId("groups", root_id=root_uuid) + payload = {"id": g21_id, "h5path": "g2/g2.1"} + req = helper.getEndpoint() + "/groups" + rsp = self.session.post(req, data=json.dumps(payload), headers=headers) + self.assertEqual(rsp.status_code, 404) # g2 not found + + # try again with implicit creation set + params = {"implicit": 1} + g21_id = createObjId("groups", root_id=root_uuid) + payload = {"id": g21_id, "h5path": "g2/g2.1"} + rsp = self.session.post(req, data=json.dumps(payload), params=params, headers=headers) + self.assertEqual(rsp.status_code, 201) # g2 and g2.1 created + rspJson = json.loads(rsp.text) + self.assertEqual(rspJson["linkCount"], 0) + self.assertEqual(rspJson["attributeCount"], 0) + self.assertEqual(rspJson["id"], g21_id) + + # get root group and verify link count is 2 + req = helper.getEndpoint() + "/groups/" + root_uuid + rsp = self.session.get(req, headers=headers) + self.assertEqual(rsp.status_code, 200) + rspJson = json.loads(rsp.text) + self.assertEqual(rspJson["linkCount"], 2) + + # get the group at "/g2" + req = helper.getEndpoint() + "/groups/" + params = {"h5path": "/g2"} + rsp = self.session.get(req, headers=headers, params=params) + self.assertEqual(rsp.status_code, 200) + rspJson = json.loads(rsp.text) + self.assertEqual(rspJson["linkCount"], 1) # group g2.1 + + # get the group at "/g2/g2.1" + req = helper.getEndpoint() + "/groups/" + params = {"h5path": "/g2/g2.1"} + rsp = self.session.get(req, headers=headers, params=params) + self.assertEqual(rsp.status_code, 200) + def testPostWithCreationProps(self): # test POST group with creation properties - print("testPost", self.base_domain) + print("testPostWithCreationProps", self.base_domain) endpoint = helper.getEndpoint() headers = helper.getRequestHeaders(domain=self.base_domain) req = endpoint + "/groups" @@ -470,6 +744,69 @@ def testPostWithCreationProps(self): self.assertTrue("alias" in rspJson) self.assertEqual(rspJson["alias"], []) + def testPostMulti(self): + # test POST with multi-object creation + print("testPostMulti", self.base_domain) + headers = helper.getRequestHeaders(domain=self.base_domain) + + # get root id + req = helper.getEndpoint() + "/" + rsp = self.session.get(req, headers=headers) + self.assertEqual(rsp.status_code, 200) + rspJson = json.loads(rsp.text) + root_uuid = rspJson["root"] + helper.validateId(root_uuid) + + # get root group and verify link count is 0 + req = helper.getEndpoint() + "/groups/" + root_uuid + rsp = self.session.get(req, headers=headers) + self.assertEqual(rsp.status_code, 200) + rspJson = json.loads(rsp.text) + self.assertEqual(rspJson["linkCount"], 0) + + # create a set of anonymous groups + grp_count = 3 + req = helper.getEndpoint() + "/groups" + + payload = [{},] * grp_count + rsp = self.session.post(req, data=json.dumps(payload), headers=headers) + self.assertEqual(rsp.status_code, 201) + rspJson = json.loads(rsp.text) + self.assertTrue("objects" in rspJson) + rsp_objs = rspJson["objects"] + self.assertEqual(len(rsp_objs), grp_count) + for i in range(grp_count): + grp_rsp = rsp_objs[i] + self.assertEqual(grp_rsp["linkCount"], 0) + self.assertEqual(grp_rsp["attributeCount"], 0) + group_id = grp_rsp["id"] + self.assertTrue(helper.validateId(group_id)) + + # create a set of linked groups + grp_count = 3 + payload = [] + for i in range(grp_count): + payload.append({"link": {"id": root_uuid, "name": f"g{i}"}}) + rsp = self.session.post(req, data=json.dumps(payload), headers=headers) + self.assertEqual(rsp.status_code, 201) + rspJson = json.loads(rsp.text) + self.assertTrue("objects" in rspJson) + rsp_objs = rspJson["objects"] + self.assertEqual(len(rsp_objs), grp_count) + for i in range(grp_count): + grp_rsp = rsp_objs[i] + self.assertEqual(grp_rsp["linkCount"], 0) + self.assertEqual(grp_rsp["attributeCount"], 0) + group_id = grp_rsp["id"] + self.assertTrue(helper.validateId(group_id)) + + # get root group and verify link count is grp_count + req = helper.getEndpoint() + "/groups/" + root_uuid + rsp = self.session.get(req, headers=headers) + self.assertEqual(rsp.status_code, 200) + rspJson = json.loads(rsp.text) + self.assertEqual(rspJson["linkCount"], grp_count) + def testDelete(self): # test Delete print("testDelete", self.base_domain) diff --git a/tests/integ/helper.py b/tests/integ/helper.py index e8cfa9f0..e51b22ab 100644 --- a/tests/integ/helper.py +++ b/tests/integ/helper.py @@ -98,8 +98,6 @@ def getRequestHeaders(domain=None, username=None, bucket=None, password=None, ** password = config.get("user2_password") headers = dict() if domain is not None: - # if config.get("bucket_name"): - # domain = config.get("bucket_name") + domain headers['X-Hdf-domain'] = domain.encode('utf-8') if username and password: auth_string = username + ':' + password @@ -230,6 +228,10 @@ def getUUIDByPath(domain, path, username=None, password=None, session=None): def getHDF5JSON(filename): """Get HDF5 JSON dump for chunk locations""" + if not op.isabs(filename): + # resolve relative to this file's directory so callers work + # regardless of the current working directory + filename = op.join(op.dirname(op.abspath(__file__)), filename) if not op.isfile(filename): return None hdf5_json = None diff --git a/tests/integ/link_test.py b/tests/integ/link_test.py index a6f72aeb..38444c3f 100755 --- a/tests/integ/link_test.py +++ b/tests/integ/link_test.py @@ -68,6 +68,14 @@ def testHardLink(self): rsp = self.session.get(req, headers=headers) self.assertEqual(rsp.status_code, 404) # link doesn't exist yet + # try creating link with no body + rsp = self.session.put(req, headers=headers) + self.assertEqual(rsp.status_code, 400) + + # try creating link with no items + rsp = self.session.put(req, headers=headers, data=json.dumps({})) + self.assertEqual(rsp.status_code, 400) + # try creating a link with a different user (should fail) if test_user2: headers = helper.getRequestHeaders(domain=domain, username=test_user2) @@ -83,7 +91,7 @@ def testHardLink(self): rsp = self.session.put(req, data=json.dumps(payload), headers=headers) self.assertEqual(rsp.status_code, 201) # created - # now gettting the link should succeed + # now getting the link should succeed rsp = self.session.get(req, headers=headers) self.assertEqual(rsp.status_code, 200) # should get link now rspJson = json.loads(rsp.text) @@ -262,7 +270,7 @@ def testExternalLink(self): target_path = "somewhere" link_title = "external_link" req = helper.getEndpoint() + "/groups/" + root_id + "/links/" + link_title - payload = {"h5path": target_path, "h5domain": target_domain} + payload = {"h5path": target_path, "file": target_domain} rsp = self.session.put(req, data=json.dumps(payload), headers=headers) self.assertEqual(rsp.status_code, 201) # created @@ -286,7 +294,7 @@ def testExternalLink(self): self.assertEqual(rspLink["title"], link_title) self.assertEqual(rspLink["class"], "H5L_TYPE_EXTERNAL") self.assertEqual(rspLink["h5path"], target_path) - self.assertEqual(rspLink["h5domain"], target_domain) + self.assertEqual(rspLink["file"], target_domain) def testGetLinks(self): domain = self.base_domain + "/testGetLinks.h5" @@ -516,10 +524,10 @@ def testGet(self): self.assertTrue(link["created"] < now - 10) else: self.assertEqual(link_class, "H5L_TYPE_EXTERNAL") - for name in ("created", "class", "h5domain", "h5path", "title", "href"): + for name in ("created", "class", "file", "h5path", "title", "href"): self.assertTrue(name in link) self.assertEqual(link["title"], "extlink") - extlink_file = link["h5domain"] + extlink_file = link["file"] self.assertEqual(extlink_file, "somefile") self.assertEqual(link["h5path"], "somepath") self.assertTrue(link["created"] < now - 10) @@ -547,7 +555,8 @@ def testGet(self): self.assertTrue(name in link) self.assertEqual(link["class"], "H5L_TYPE_SOFT") - self.assertFalse("h5domain" in link) # only for external links + self.assertFalse("h5domain" in link) # deprecated name + self.assertFalse("file" in link) # only for external links self.assertEqual(link["title"], "slink") self.assertEqual(link["h5path"], "somevalue") @@ -610,12 +619,14 @@ def testGetRecursive(self): softlink_count += 1 self.assertTrue("h5path" in link) self.assertFalse("h5domain" in link) + self.assertFalse("file" in link) self.assertFalse("id" in link) self.assertTrue(link_title in expected_soft_links) elif link_class == "H5L_TYPE_EXTERNAL": extlink_count += 1 self.assertTrue("h5path" in link) - self.assertTrue("h5domain" in link) + self.assertTrue("file" in link) + self.assertFalse("h5domain" in link) # deprecated name self.assertFalse("id" in link) self.assertTrue(link_title in expected_external_links) else: @@ -682,7 +693,7 @@ def testGetPattern(self): self.assertEqual(len(links), 1) # only extlink should be returned link = links[0] - for name in ("created", "class", "h5domain", "h5path", "title"): + for name in ("created", "class", "file", "h5path", "title"): self.assertTrue(name in link) if use_post: pass # no href with post @@ -690,7 +701,7 @@ def testGetPattern(self): self.assertTrue("href" in link) self.assertEqual(link["class"], "H5L_TYPE_EXTERNAL") self.assertEqual(link["title"], "extlink") - self.assertEqual(link["h5domain"], "somefile") + self.assertEqual(link["file"], "somefile") self.assertEqual(link["h5path"], "somepath") self.assertTrue(link["created"] < now - 10) @@ -918,7 +929,7 @@ def testExternalLinkTraversal(self): target_path = "/external_group" link_title = "external_link_to_group" req = helper.getEndpoint() + "/groups/" + root_id + "/links/" + link_title - payload = {"h5path": target_path, "h5domain": second_domain} + payload = {"h5path": target_path, "file": second_domain} headers = helper.getRequestHeaders(domain=domain) rsp = self.session.put(req, data=json.dumps(payload), headers=headers) self.assertEqual(rsp.status_code, 201) @@ -951,7 +962,7 @@ def testExternalLinkTraversal(self): target_path = "/external_group" link_title = "external_link_to_group_prefix" req = helper.getEndpoint() + "/groups/" + root_id + "/links/" + link_title - payload = {"h5path": target_path, "h5domain": f"hdf5:/{second_domain}"} + payload = {"h5path": target_path, "file": f"hdf5:/{second_domain}"} headers = helper.getRequestHeaders(domain=domain) rsp = self.session.put(req, data=json.dumps(payload), headers=headers) self.assertEqual(rsp.status_code, 201) @@ -1214,10 +1225,10 @@ def testPostLinkSingle(self): self.assertTrue(link["created"] < now - 10) else: self.assertEqual(link_class, "H5L_TYPE_EXTERNAL") - for name in ("created", "class", "h5domain", "h5path", "title"): + for name in ("created", "class", "file", "h5path", "title"): self.assertTrue(name in link) self.assertEqual(link["title"], "extlink") - extlink_file = link["h5domain"] + extlink_file = link["file"] self.assertEqual(extlink_file, "somefile") self.assertEqual(link["h5path"], "somepath") self.assertTrue(link["created"] < now - 10) @@ -1288,7 +1299,7 @@ def testPostLinkMultiple(self): # soft or external link self.assertEqual(link["h5path"], expected["h5path"]) if link_class == "H5L_TYPE_EXTERNAL": - self.assertEqual(link["h5domain"], expected["h5domain"]) + self.assertEqual(link["file"], expected["file"]) # get just the requested links for each group req = helper.getEndpoint() + "/groups/" + root_id + "/links" @@ -1481,12 +1492,211 @@ def testPutLinkMultiple(self): links = {} for i in range(grp_count): title = grp_names[i] - links[title] = {"id": grp_ids[i]} + if i % 2 == 0: + # create a hardlink implicitly + links[title] = {"id": grp_ids[i]} + else: + # for variety, create a hardlink by providing full link json + links[title] = {"class": "H5L_TYPE_HARD", "id": grp_ids[i]} + + # add a soft and external link as well + links["softlink"] = {"h5path": "a_path"} + links["extlink"] = {"h5path": "another_path", "file": "/a_domain"} + link_count = len(links) + + # write links to the grpA + data = {"links": links} + req = self.endpoint + "/groups/" + grpA_id + "/links" + rsp = self.session.put(req, data=json.dumps(data), headers=headers) + self.assertEqual(rsp.status_code, 201) + + # do a get on the links + rsp = self.session.get(req, headers=headers) + self.assertEqual(rsp.status_code, 200) + rspJson = json.loads(rsp.text) + self.assertTrue("links" in rspJson) + ret_links = rspJson["links"] + self.assertEqual(len(ret_links), link_count) + for link in ret_links: + self.assertTrue("title" in link) + title = link["title"] + self.assertTrue("class" in link) + link_class = link["class"] + if link_class == "H5L_TYPE_HARD": + self.assertTrue("id" in link) + self.assertTrue(link["id"] in grp_ids) + self.assertTrue(title in grp_names) + elif link_class == "H5L_TYPE_SOFT": + self.assertTrue("h5path" in link) + h5path = link["h5path"] + self.assertEqual(h5path, "a_path") + elif link_class == "H5L_TYPE_EXTERNAL": + self.assertTrue("h5path" in link) + h5path = link["h5path"] + self.assertEqual(h5path, "another_path") + self.assertTrue("file" in link) + h5domain = link["file"] + self.assertEqual(h5domain, "/a_domain") + else: + self.assertTrue(False) # unexpected + + # try writing again, should get 200 (no new links) + rsp = self.session.put(req, data=json.dumps(data), headers=headers) + self.assertEqual(rsp.status_code, 200) + + # write some links to three group objects + links = {} + links["hardlink_multicast"] = {"id": root_id} + links["softlink_multicast"] = {"h5path": "multi_path"} + links["extlink_multicast"] = {"h5path": "multi_path", "file": "/another_domain"} + link_count = len(links) + data = {"links": links, "grp_ids": grp_ids} + req = self.endpoint + "/groups/" + root_id + "/links" + rsp = self.session.put(req, data=json.dumps(data), headers=headers) + self.assertEqual(rsp.status_code, 201) + + # check that the links got created + for grp_id in grp_ids: + req = self.endpoint + "/groups/" + grp_id + "/links" + rsp = self.session.get(req, headers=headers) + self.assertEqual(rsp.status_code, 200) + rspJson = json.loads(rsp.text) + self.assertTrue("links" in rspJson) + ret_links = rspJson["links"] + self.assertEqual(len(ret_links), 3) + for ret_link in ret_links: + self.assertTrue("class" in ret_link) + link_class = ret_link["class"] + if link_class == "H5L_TYPE_HARD": + self.assertTrue("id" in ret_link) + self.assertEqual(ret_link["id"], root_id) + elif link_class == "H5L_TYPE_SOFT": + self.assertTrue("h5path" in ret_link) + self.assertEqual(ret_link["h5path"], "multi_path") + elif link_class == "H5L_TYPE_EXTERNAL": + self.assertTrue("h5path" in ret_link) + self.assertEqual(ret_link["h5path"], "multi_path") + self.assertTrue("file" in ret_link) + self.assertEqual(ret_link["file"], "/another_domain") + else: + self.assertTrue(False) # unexpected + + # write different links to three group objects + link_data = {} + for i in range(grp_count): + grp_id = grp_ids[i] + links = {} + links[f"hardlink_{i}"] = {"id": root_id} + links[f"softlink_{i}"] = {"h5path": f"multi_path_{i}"} + ext_link = {"h5path": f"multi_path_{i}", "file": f"/another_domain/{i}"} + links[f"extlink_{i}"] = ext_link + link_data[grp_id] = {"links": links} + + data = {"grp_ids": link_data} + req = self.endpoint + "/groups/" + root_id + "/links" + rsp = self.session.put(req, data=json.dumps(data), headers=headers) + self.assertEqual(rsp.status_code, 201) + + # check that the new links got created + for i in range(grp_count): + grp_id = grp_ids[i] + titles = [f"hardlink_{i}", f"softlink_{i}", f"extlink_{i}", ] + data = {"titles": titles} + # do a post to just return the links we are interested in + req = self.endpoint + "/groups/" + grp_id + "/links" + rsp = self.session.post(req, data=json.dumps(data), headers=headers) + self.assertEqual(rsp.status_code, 200) + rspJson = json.loads(rsp.text) + self.assertTrue("links" in rspJson) + ret_links = rspJson["links"] + self.assertEqual(len(ret_links), len(titles)) + for j in range(len(titles)): + ret_link = ret_links[j] + self.assertTrue("class" in ret_link) + link_class = ret_link["class"] + self.assertTrue("title" in ret_link) + link_title = ret_link["title"] + if link_class == "H5L_TYPE_HARD": + self.assertEqual(link_title, f"hardlink_{i}") + self.assertTrue("id" in ret_link) + self.assertEqual(ret_link["id"], root_id) + elif link_class == "H5L_TYPE_SOFT": + self.assertEqual(link_title, f"softlink_{i}") + self.assertTrue("h5path" in ret_link) + self.assertEqual(ret_link["h5path"], f"multi_path_{i}") + elif link_class == "H5L_TYPE_EXTERNAL": + self.assertEqual(link_title, f"extlink_{i}") + self.assertTrue("h5path" in ret_link) + self.assertEqual(ret_link["h5path"], f"multi_path_{i}") + self.assertTrue("file" in ret_link) + self.assertEqual(ret_link["file"], f"/another_domain/{i}") + else: + self.assertTrue(False) # unexpected + + def testPutLinkMultipleWithTimestamps(self): + domain = self.base_domain + "/testPutLinkMultipleWithTImestamps.h5" + helper.setupDomain(domain) + print("testPutLinkMultipleWithTimestamps", domain) + headers = helper.getRequestHeaders(domain=domain) + req = self.endpoint + "/" + + rsp = self.session.get(req, headers=headers) + self.assertEqual(rsp.status_code, 200) + rspJson = json.loads(rsp.text) + root_id = rspJson["root"] + + # create a group + req = self.endpoint + "/groups" + rsp = self.session.post(req, headers=headers) + self.assertEqual(rsp.status_code, 201) + rspJson = json.loads(rsp.text) + grpA_id = rspJson["id"] + self.assertTrue(helper.validateId(grpA_id)) + + # link new obj as '/grpA' + req = self.endpoint + "/groups/" + root_id + "/links/grpA" + payload = {"id": grpA_id} + rsp = self.session.put(req, data=json.dumps(payload), headers=headers) + self.assertEqual(rsp.status_code, 201) # created + + # create some groups under grp1 + grp_count = 3 + + grp_names = [f"grp{(i + 1):04d}" for i in range(grp_count)] + grp_ids = [] + + for grp_name in grp_names: + # create sub_groups + req = self.endpoint + "/groups" + rsp = self.session.post(req, headers=headers) + self.assertEqual(rsp.status_code, 201) + rspJson = json.loads(rsp.text) + grp_id = rspJson["id"] + self.assertTrue(helper.validateId(grp_id)) + grp_ids.append(grp_id) + + # create some links + links = {} + for i in range(grp_count): + title = grp_names[i] + if i % 2 == 0: + # create a hardlink implicitly + links[title] = {"id": grp_ids[i]} + else: + # for variety, create a hardlink by providing full link json + links[title] = {"class": "H5L_TYPE_HARD", "id": grp_ids[i]} # add a soft and external link as well links["softlink"] = {"h5path": "a_path"} - links["extlink"] = {"h5path": "another_path", "h5domain": "/a_domain"} + links["extlink"] = {"h5path": "another_path", "file": "/a_domain"} link_count = len(links) + # add timestamp + timestamps = set() + for title in links: + link = links[title] + now = time.time() + link["created"] = now + timestamps.add(now) # write links to the grpA data = {"links": links} @@ -1518,11 +1728,13 @@ def testPutLinkMultiple(self): self.assertTrue("h5path" in link) h5path = link["h5path"] self.assertEqual(h5path, "another_path") - self.assertTrue("h5domain" in link) - h5domain = link["h5domain"] + self.assertTrue("file" in link) + h5domain = link["file"] self.assertEqual(h5domain, "/a_domain") else: self.assertTrue(False) # unexpected + self.assertTrue("created" in link) + self.assertTrue(link["created"] in timestamps) # try writing again, should get 200 (no new links) rsp = self.session.put(req, data=json.dumps(data), headers=headers) @@ -1532,8 +1744,15 @@ def testPutLinkMultiple(self): links = {} links["hardlink_multicast"] = {"id": root_id} links["softlink_multicast"] = {"h5path": "multi_path"} - links["extlink_multicast"] = {"h5path": "multi_path", "h5domain": "/another_domain"} + links["extlink_multicast"] = {"h5path": "multi_path", "file": "/another_domain"} link_count = len(links) + timestamps = set() + for title in links: + link = links[title] + now = time.time() + link["created"] = now + timestamps.add(now) + data = {"links": links, "grp_ids": grp_ids} req = self.endpoint + "/groups/" + root_id + "/links" rsp = self.session.put(req, data=json.dumps(data), headers=headers) @@ -1560,20 +1779,28 @@ def testPutLinkMultiple(self): elif link_class == "H5L_TYPE_EXTERNAL": self.assertTrue("h5path" in ret_link) self.assertEqual(ret_link["h5path"], "multi_path") - self.assertTrue("h5domain" in ret_link) - self.assertEqual(ret_link["h5domain"], "/another_domain") + self.assertTrue("file" in ret_link) + self.assertEqual(ret_link["file"], "/another_domain") else: self.assertTrue(False) # unexpected + self.assertTrue("created" in ret_link) + self.assertTrue(ret_link["created"] in timestamps) # write different links to three group objects link_data = {} + timestamps = set() for i in range(grp_count): grp_id = grp_ids[i] links = {} links[f"hardlink_{i}"] = {"id": root_id} links[f"softlink_{i}"] = {"h5path": f"multi_path_{i}"} - ext_link = {"h5path": f"multi_path_{i}", "h5domain": f"/another_domain/{i}"} + ext_link = {"h5path": f"multi_path_{i}", "file": f"/another_domain/{i}"} links[f"extlink_{i}"] = ext_link + for title in links: + link = links[title] + now = time.time() + link["created"] = now + timestamps.add(now) link_data[grp_id] = {"links": links} data = {"grp_ids": link_data} @@ -1612,10 +1839,12 @@ def testPutLinkMultiple(self): self.assertEqual(link_title, f"extlink_{i}") self.assertTrue("h5path" in ret_link) self.assertEqual(ret_link["h5path"], f"multi_path_{i}") - self.assertTrue("h5domain" in ret_link) - self.assertEqual(ret_link["h5domain"], f"/another_domain/{i}") + self.assertTrue("file" in ret_link) + self.assertEqual(ret_link["file"], f"/another_domain/{i}") else: self.assertTrue(False) # unexpected + self.assertTrue("created" in ret_link) + self.assertTrue(ret_link["created"] in timestamps) def testDeleteLinkMultiple(self): domain = self.base_domain + "/testDeleteLinkMultiple.h5" @@ -1660,7 +1889,7 @@ def testDeleteLinkMultiple(self): links[title] = {"h5path": "a_path"} titles.append(title) title = "extlink" - links[title] = {"h5path": "another_path", "h5domain": "/a_domain"} + links[title] = {"h5path": "another_path", "file": "/a_domain"} titles.append(title) link_count = len(links) @@ -1767,7 +1996,6 @@ def testLinkCreationOrder(self): self.assertEqual(rsp.status_code, 200) rspJson = json.loads(rsp.text) links_json = rspJson["links"] - print("params:", params) # verify the links are in order for i in range(link_count - 1): @@ -1793,6 +2021,58 @@ def testLinkCreationOrder(self): self.assertEqual(prev_link['title'], sorted(link_names)[i]) self.assertEqual(link['title'], sorted(link_names)[i + 1]) + def testUseTimestamp(self): + # Test PUT value for link with timestamp included + domain = self.base_domain + "/testLinkUseTimestamp.h5" + + helper.setupDomain(domain) + print("testUseTimestamp", domain) + headers = helper.getRequestHeaders(domain=domain) + req = helper.getEndpoint() + "/" + + # Get root uuid + rsp = self.session.get(req, headers=headers) + self.assertEqual(rsp.status_code, 200) + rspJson = json.loads(rsp.text) + root_uuid = rspJson["root"] + helper.validateId(root_uuid) + + def _create_link(title, ts=None): + # create link + req = helper.getEndpoint() + f"/groups/{root_uuid}/links/{title}" + body = {"h5path": "some_path"} + if ts: + body["created"] = ts + rsp = self.session.put(req, data=json.dumps(body), headers=headers) + self.assertEqual(rsp.status_code, 201) + + def _check_link_ts(title, min_ts=None, max_ts=None): + # read link + req = helper.getEndpoint() + f"/groups/{root_uuid}/links/{title}" + rsp = self.session.get(req, headers=headers) + self.assertEqual(rsp.status_code, 200) + rspJson = json.loads(rsp.text) + self.assertTrue("hrefs" in rspJson) + self.assertTrue("created" in rspJson) + if min_ts: + self.assertGreaterEqual(rspJson["created"], min_ts) + if max_ts: + self.assertLessEqual(rspJson["created"], max_ts) + + now = time.time() + # server-based timestamp + _create_link("a1", ts=None) + _check_link_ts("a1", min_ts=(now - 1), max_ts=(now + 1)) + # client assigned timestamp + _create_link("a2", ts=now) + _check_link_ts("a2", min_ts=now, max_ts=now) + # client assigned with small time-skew, ok + _create_link("a3", ts=int(now)) + _check_link_ts("a3", min_ts=int(now), max_ts=int(now)) + # client assigned with large time-skew, ignored + _create_link("a4", ts=999) + _check_link_ts("a4", min_ts=(now - 1), max_ts=(now + 1)) + if __name__ == "__main__": # setup test files diff --git a/tests/integ/pointsel_test.py b/tests/integ/pointsel_test.py index 194eb2ce..bd6246f2 100755 --- a/tests/integ/pointsel_test.py +++ b/tests/integ/pointsel_test.py @@ -76,7 +76,7 @@ def testPost1DDataset(self): helper.validateId(root_uuid) # create dataset - # pass in layout specification so that we can test selection across chunk boundries + # pass in layout specification so that we can test selection across chunk boundaries data = {"type": "H5T_STD_I32LE", "shape": (100,)} data["creationProperties"] = { "layout": { @@ -181,7 +181,7 @@ def testPost2DDataset(self): helper.validateId(root_uuid) # create dataset - # pass in layout specification so that we can test selection across chunk boundries + # pass in layout specification so that we can test selection across chunk boundaries data = {"type": "H5T_STD_I32LE", "shape": [20, 30]} data["creationProperties"] = { "layout": {"class": "H5D_CHUNKED", "dims": [10, 10]} @@ -268,7 +268,7 @@ def testPost1DDatasetBinary(self): helper.validateId(root_uuid) # create dataset - # pass in layout specification so that we can test selection across chunk boundries + # pass in layout specification so that we can test selection across chunk boundaries data = {"type": "H5T_STD_I32LE", "shape": (100,)} data["creationProperties"] = { "layout": { @@ -392,7 +392,7 @@ def testPost2DDatasetBinary(self): helper.validateId(root_uuid) # create dataset - # pass in layout specification so that we can test selection across chunk boundries + # pass in layout specification so that we can test selection across chunk boundaries data = {"type": "H5T_STD_I32LE", "shape": [20, 30]} data["creationProperties"] = { "layout": {"class": "H5D_CHUNKED", "dims": [10, 10]} @@ -567,7 +567,7 @@ def testPostContiguousDataset(self): points = [2, 3, 5, 7, 11, 13, 17, 19] body = {"points": points} # add nonstrict - params = {"nonstrict": 1} # enable SN to invoke lambda func + params = {"nonstrict": 1} # enable nonstrict mode rsp = self.session.post( req, params=params, data=json.dumps(body), headers=headers @@ -697,7 +697,7 @@ def testPostChunkedRefDataset(self): body = {"points": points} rsp = self.session.post(req, data=json.dumps(body), headers=headers) if rsp.status_code == 404: - msg = "s3object: {s3path} not found, skipping point chunk ref test" + msg = f"s3object: {s3path} not found, skipping point chunk ref test" print(msg) else: self.assertEqual(rsp.status_code, 200) @@ -878,7 +878,7 @@ def testPut1DDataset(self): helper.validateId(root_uuid) # create dataset - # pass in layout specification so that we can test selection across chunk boundries + # pass in layout specification so that we can test selection across chunk boundaries data = {"type": "H5T_STD_I8LE", "shape": (100,)} data["creationProperties"] = { "layout": { @@ -982,7 +982,7 @@ def testPut2DDataset(self): helper.validateId(root_uuid) # create dataset - # pass in layout specification so that we can test selection across chunk boundries + # pass in layout specification so that we can test selection across chunk boundaries data = {"type": "H5T_STD_I32LE", "shape": [20, 30]} data["creationProperties"] = { "layout": {"class": "H5D_CHUNKED", "dims": [10, 10]} @@ -1021,7 +1021,6 @@ def testPut2DDataset(self): self.assertTrue("value" in rspJson) # verify the correct elements got set value = rspJson["value"] - # print("value:", value) for x in range(20): row = value[x] for y in range(30): @@ -1045,7 +1044,7 @@ def testPut1DDatasetBinary(self): helper.validateId(root_uuid) # create dataset - # pass in layout specification so that we can test selection across chunk boundries + # pass in layout specification so that we can test selection across chunk boundaries data = {"type": "H5T_STD_I8LE", "shape": (100,)} data["creationProperties"] = { "layout": { @@ -1153,7 +1152,7 @@ def testPut1DDatasetVlenBinary(self): helper.validateId(root_uuid) # create dataset - # pass in layout specification so that we can test selection across chunk boundries + # pass in layout specification so that we can test selection across chunk boundaries vlen_type = {"class": "H5T_VLEN", "base": "H5T_STD_I8LE"} data = {"type": vlen_type, "shape": (100,)} data["creationProperties"] = { @@ -1275,7 +1274,7 @@ def testPut2DDatasetBinary(self): helper.validateId(root_uuid) # create dataset - # pass in layout specification so that we can test selection across chunk boundries + # pass in layout specification so that we can test selection across chunk boundaries data = {"type": "H5T_STD_I32LE", "shape": [20, 30]} data["creationProperties"] = { "layout": {"class": "H5D_CHUNKED", "dims": [10, 10]} @@ -1323,7 +1322,6 @@ def testPut2DDatasetBinary(self): self.assertTrue("value" in rspJson) # verify the correct elements got set value = rspJson["value"] - # print("value:", value) for x in range(20): row = value[x] for y in range(30): @@ -1370,8 +1368,10 @@ def testDatasetChunkPartitioning(self): rsp = self.session.get(req, headers=headers) self.assertEqual(rsp.status_code, 200) rspJson = json.loads(rsp.text) - self.assertTrue("layout" in rspJson) - layout_json = rspJson["layout"] + self.assertTrue("creationProperties" in rspJson) + creation_props = rspJson["creationProperties"] + self.assertTrue("layout" in creation_props) + layout_json = creation_props["layout"] self.assertTrue("class" in layout_json) self.assertEqual(layout_json["class"], "H5D_CHUNKED") self.assertTrue("dims" in layout_json) @@ -1703,7 +1703,7 @@ def testPostCompoundDataset(self): helper.validateId(root_uuid) # create dataset - # pass in layout specification so that we can test selection across chunk boundries + # pass in layout specification so that we can test selection across chunk boundaries data = {"type": "H5T_STD_I32LE", "shape": (100,)} # # create 1d dataset diff --git a/tests/integ/query_test.py b/tests/integ/query_test.py index 9025c78b..850c0f3e 100644 --- a/tests/integ/query_test.py +++ b/tests/integ/query_test.py @@ -11,6 +11,7 @@ ############################################################################## import unittest import json + import helper import config @@ -79,7 +80,7 @@ def testSimpleQuery(self): dset_uuid = rspJson["id"] self.assertTrue(helper.validateId(dset_uuid)) - # link new dataset as 'dset1' + # link new dataset name = "dset" + helper.getRandomName() req = self.endpoint + "/groups/" + root_uuid + "/links/" + name payload = {"id": dset_uuid} @@ -107,8 +108,8 @@ def testSimpleQuery(self): rsp = self.session.put(req, data=json.dumps(payload), headers=headers) self.assertEqual(rsp.status_code, 200) # write value - def verifyQueryRsp(rsp, expected_indices=None, expect_bin=None): - ROW_BYTES = 28 # 8 + 4 + 8 + 4 + 4 + def verifyValueRsp(rsp, expected_indices=None, expect_bin=None): + ROW_BYTES = 20 # 4 + 8 + 4 + 4 self.assertEqual(rsp.status_code, 200) data = None if rsp.headers["Content-Type"] == "application/octet-stream": @@ -123,42 +124,85 @@ def verifyQueryRsp(rsp, expected_indices=None, expect_bin=None): index_start = i * ROW_BYTES index_end = (i + 1) * ROW_BYTES x = bin_data[index_start:index_end] - index = int.from_bytes(x[0:8], "little", signed=False) - symbol = x[8:12].decode("ascii") - date_str = x[12:20].decode("ascii") - open = int.from_bytes(x[20:24], "little") - close = int.from_bytes(x[24:28], "little") - row = [index, symbol, date_str, open, close] + symbol = x[0:4].decode("ascii") + date_str = x[4:12].decode("ascii") + open = int.from_bytes(x[12:16], "little") + close = int.from_bytes(x[16:20], "little") + row = [symbol, date_str, open, close] data.append(row) else: if expect_bin is True: self.assertTrue(False) + self.assertTrue(len(rsp.text) > 0) rspJson = json.loads(rsp.text) self.assertTrue("hrefs" in rspJson) self.assertTrue("value" in rspJson) + self.assertFalse("indices" in rspJson) data = rspJson["value"] - index_set = set() expected_count = None if expected_indices: - for index in expected_indices: - index_set.add(index) expected_count = len(expected_indices) - for item in data: - self.assertEqual(len(item), 5) # index + 4 fields - index = item[0] - index_set.add(index) - expected = value[index] - for i in range(4): - self.assertEqual(item[i + 1], expected[i]) - # indices should be unique - self.assertEqual(len(index_set), len(data)) # check we got the expected number of results if expected_count is not None: self.assertEqual(len(data), expected_count) - # end verifyQueryRsp + for j in range(len(data)): + item = data[j] + self.assertEqual(len(item), 4) # fields + if expected_indices is not None: + index = expected_indices[j] + expected = value[index] + for i in range(4): + self.assertEqual(item[i], expected[i]) + # indices should be unique - req = self.endpoint + "/datasets/" + dset_uuid + "/value" + # end verifyValueRsp + + def verifyQueryRsp(rsp, expected_indices=None, expect_bin=None): + self.assertEqual(rsp.status_code, 200) + INDEX_SIZE = 8 # 8 bytes for each index (int64) + if rsp.headers["Content-Type"] == "application/octet-stream": + if expect_bin is False: + self.assertTrue(False) + bin_data = rsp.content + self.assertEqual(len(bin_data) % INDEX_SIZE, 0) + # assemble binary response to a python list - this dataset + # is rank-1, so each match is a single 8-byte coordinate + count = len(bin_data) // INDEX_SIZE + indices = [] + for i in range(count): + index_start = i * INDEX_SIZE + index_end = (i + 1) * INDEX_SIZE + index = int.from_bytes(bin_data[index_start:index_end], "little") + indices.append(index) + else: + if expect_bin is True: + self.assertTrue(False) + self.assertTrue(len(rsp.text) > 0) + rspJson = json.loads(rsp.text) + self.assertTrue("hrefs" in rspJson) + self.assertTrue("indices" in rspJson) + self.assertFalse("values" in rspJson) + # each match is a (rank,) coordinate tuple - flatten to a + # plain index since this dataset is rank-1 + indices = [coord[0] for coord in rspJson["indices"]] + + expected_count = None + if expected_indices: + expected_count = len(expected_indices) + # check we got the expected number of results + if expected_count is not None: + self.assertEqual(len(indices), expected_count) + for j in range(len(indices)): + index = indices[j] + if expected_indices is not None: + expected = expected_indices[j] + self.assertEqual(index, expected) + # indices should be unique + # end of verifyQueryRsp + + value_req = self.endpoint + "/datasets/" + dset_uuid + "/value" + query_req = self.endpoint + "/datasets/" + dset_uuid + "/query" for query_headers in (headers, headers_bin_rsp): kwargs = {} @@ -169,67 +213,94 @@ def verifyQueryRsp(rsp, expected_indices=None, expect_bin=None): kwargs["expect_bin"] = False # items in list - params = {"query": "open < 4000 where stock_symbol in (b'AAPL', b'EBAY')"} - rsp = self.session.get(req, params=params, headers=query_headers) + params = {"query": "open < 3000 AND stock_symbol IN (AAPL, EBAY)"} + rsp = self.session.get(value_req, params=params, headers=query_headers) self.assertEqual(rsp.status_code, 200) - kwargs["expected_indices"] = [0, 1, 3, 4, 6, 7, 9, 10] + kwargs["expected_indices"] = [6, 7, 9, 10] + verifyValueRsp(rsp, **kwargs) + rsp = self.session.get(query_req, params=params, headers=query_headers) verifyQueryRsp(rsp, **kwargs) # read first row with AAPL - params = {"query": "stock_symbol == b'AAPL'", "Limit": 1} - rsp = self.session.get(req, params=params, headers=query_headers) + params = {"query": "stock_symbol == AAPL", "Limit": 1} + rsp = self.session.get(value_req, params=params, headers=query_headers) kwargs["expected_indices"] = (1,) - + verifyValueRsp(rsp, **kwargs) + rsp = self.session.get(query_req, params=params, headers=query_headers) verifyQueryRsp(rsp, **kwargs) # read all rows with APPL - params = {"query": "stock_symbol == b'AAPL'"} - rsp = self.session.get(req, params=params, headers=query_headers) + params = {"query": "stock_symbol == AAPL"} + rsp = self.session.get(value_req, params=params, headers=query_headers) expected_indices = (1, 4, 7, 10) kwargs["expected_indices"] = expected_indices + verifyValueRsp(rsp, **kwargs) + rsp = self.session.get(query_req, params=params, headers=query_headers) verifyQueryRsp(rsp, **kwargs) # return just open and close fields - params = {"query": "stock_symbol == b'AAPL'", "fields": "open:close"} + params = {"query": "stock_symbol == AAPL", "fields": "open:close"} # just do json to keep the verification simple - rsp = self.session.get(req, params=params, headers=headers) + rsp = self.session.get(value_req, params=params, headers=headers) # need to check this one by hand self.assertEqual(rsp.status_code, 200) rspJson = json.loads(rsp.text) - query_rsp = rspJson["value"] - self.assertEqual(len(query_rsp), 4) + value_rsp = rspJson["value"] + self.assertEqual(len(value_rsp), 4) for i in range(4): - item = query_rsp[i] - self.assertEqual(len(item), 3) - self.assertEqual(item[0], expected_indices[i]) + item = value_rsp[i] + self.assertEqual(len(item), 2) + expected_index = expected_indices[i] + row = value[expected_index] + self.assertEqual(item[0], row[2]) + self.assertEqual(item[1], row[3]) # expected_indices will be the same params["select"] = "[2:12]" del params["fields"] # remove key from last test - rsp = self.session.get(req, params=params, headers=query_headers) + rsp = self.session.get(value_req, params=params, headers=query_headers) kwargs["expected_indices"] = (4, 7, 10) + verifyValueRsp(rsp, **kwargs) + rsp = self.session.get(query_req, params=params, headers=query_headers) verifyQueryRsp(rsp, **kwargs) - params = {"query": "where stock_symbol in (b'AAPL', b'EBAY')"} - rsp = self.session.get(req, params=params, headers=query_headers) + params = {"query": "stock_symbol IN (AAPL, EBAY)"} + rsp = self.session.get(value_req, params=params, headers=query_headers) self.assertEqual(rsp.status_code, 200) kwargs["expected_indices"] = [0, 1, 3, 4, 6, 7, 9, 10] + verifyValueRsp(rsp, **kwargs) + rsp = self.session.get(query_req, params=params, headers=query_headers) verifyQueryRsp(rsp, **kwargs) - params = {"query": "open < 3000 where stock_symbol in (b'AAPL', b'EBAY')"} + + params = {"query": "open < 3000 AND stock_symbol IN (AAPL, EBAY)"} rsp = self.session.get(req, params=params, headers=query_headers) self.assertEqual(rsp.status_code, 200) kwargs["expected_indices"] = [6, 7, 9, 10] + verifyValueRsp(rsp, **kwargs) + rsp = self.session.get(query_req, params=params, headers=query_headers) + verifyQueryRsp(rsp, **kwargs) + + params = {"query": "open < 4000 AND stock_symbol IN (AAPL, EBAY)"} + rsp = self.session.get(value_req, params=params, headers=query_headers) + self.assertEqual(rsp.status_code, 200) + kwargs["expected_indices"] = [0, 1, 3, 4, 6, 7, 9, 10] + verifyValueRsp(rsp, **kwargs) + rsp = self.session.get(query_req, params=params, headers=query_headers) verifyQueryRsp(rsp, **kwargs) # combine with Limit params["Limit"] = 2 - rsp = self.session.get(req, params=params, headers=query_headers) - kwargs["expected_indices"] = (6, 7) + rsp = self.session.get(value_req, params=params, headers=query_headers) + kwargs["expected_indices"] = [0, 1] + verifyValueRsp(rsp, **kwargs) + rsp = self.session.get(query_req, params=params, headers=query_headers) verifyQueryRsp(rsp, **kwargs) # try bad Limit params["Limit"] = "abc" - rsp = self.session.get(req, params=params, headers=query_headers) + rsp = self.session.get(value_req, params=params, headers=query_headers) + self.assertEqual(rsp.status_code, 400) + rsp = self.session.get(query_req, params=params, headers=query_headers) self.assertEqual(rsp.status_code, 400) # try invalid query strings @@ -244,22 +315,28 @@ def verifyQueryRsp(rsp, expected_indices=None, expect_bin=None): ) for query in queries: params = {"query": query} - rsp = self.session.get(req, params=params, headers=query_headers) + rsp = self.session.get(value_req, params=params, headers=query_headers) + self.assertEqual(rsp.status_code, 400) + rsp = self.session.get(query_req, params=params, headers=query_headers) self.assertEqual(rsp.status_code, 400) # try boolean query - params = {"query": "(open > 3000) & (open < 3100)"} - rsp = self.session.get(req, params=params, headers=query_headers) + params = {"query": "(open > 3000) AND (open < 3100)"} + rsp = self.session.get(value_req, params=params, headers=query_headers) self.assertEqual(rsp.status_code, 200) kwargs["expected_indices"] = (0, 1, 3, 5, 11) + verifyValueRsp(rsp, **kwargs) + rsp = self.session.get(query_req, params=params, headers=query_headers) verifyQueryRsp(rsp, **kwargs) # query for a zero sector field (should return none) params = {"query": "open == 0"} # query for zero sector - rsp = self.session.get(req, params=params, headers=headers) + rsp = self.session.get(value_req, params=params, headers=headers) self.assertEqual(rsp.status_code, 200) kwargs["expected_indices"] = () kwargs["expect_bin"] = False # will always get json for null response + verifyValueRsp(rsp, **kwargs) + rsp = self.session.get(query_req, params=params, headers=headers) verifyQueryRsp(rsp, **kwargs) def testChunkedRefIndirectDataset(self): @@ -396,7 +473,7 @@ def testChunkedRefIndirectDataset(self): # read a selection req = self.endpoint + "/datasets/" + dset_id + "/value" - params = {"query": "stock_symbol == b'AAPL'"} # query for AAPL + params = {"query": "stock_symbol == AAPL"} # query for AAPL params["select"] = "[0:1000000]" # search over just first 1MM rows rsp = self.session.get(req, params=params, headers=headers) @@ -407,11 +484,19 @@ def testChunkedRefIndirectDataset(self): readData = rspJson["value"] self.assertEqual(len(readData), 3902) item = readData[0] - self.assertEqual(item[0], 128912) - self.assertEqual(item[1], "1980.12.12") - self.assertEqual(item[2], "AAPL") + self.assertEqual(item[0], "1980.12.12") + self.assertEqual(item[1], "AAPL") def testPutQuery(self): + """ + PUT .../value with a `query` parameter performs a query-based + conditional update: rows matching `query` have the given field(s) + set to the given value(s) (query_update is only allowed when the + value is one element, which gets broadcast across all matching + elements). The response's `value` is the list of global dataset + indices that matched (and were updated), not the row data itself - + e.g. `[[1], [4], [7], [10]]` for four single-dimension matches. + """ # Test PUT query for 1d dataset print("testPutQuery", self.base_domain) @@ -458,7 +543,7 @@ def testPutQuery(self): dset_uuid = rspJson["id"] self.assertTrue(helper.validateId(dset_uuid)) - # link new dataset as 'dset1' + # link new dataset name = "dset" + helper.getRandomName() req = self.endpoint + "/groups/" + root_uuid + "/links/" + name payload = {"id": dset_uuid} @@ -487,7 +572,7 @@ def testPutQuery(self): self.assertEqual(rsp.status_code, 200) # write value # set any rows with AAPL to have open of 999 - params = {"query": "stock_symbol == b'AAPL'"} + params = {"query": "stock_symbol == AAPL"} update_value = {"open": 999} payload = {"value": update_value} rsp = self.session.put( @@ -496,14 +581,11 @@ def testPutQuery(self): self.assertEqual(rsp.status_code, 200) rspJson = json.loads(rsp.text) self.assertTrue("hrefs" in rspJson) - self.assertTrue("value" in rspJson) - readData = rspJson["value"] - self.assertEqual(len(readData), 4) - indicies = [] - for item in readData: - indicies.append(item[0]) - self.assertEqual(item[1], "AAPL") - self.assertEqual(indicies, [1, 4, 7, 10]) + self.assertTrue("value" not in rspJson) + self.assertTrue("indices" in rspJson) + indices = rspJson["indices"] + self.assertEqual(len(indices), 4) + self.assertEqual(indices, [[1,], [4,], [7,], [10,]]) # read values and verify the expected changes where made req = self.endpoint + "/datasets/" + dset_uuid + "/value" @@ -531,7 +613,7 @@ def testPutQuery(self): self.assertEqual(rsp.status_code, 200) # write value # set just one row with AAPL to have open of 42 - params = {"query": "stock_symbol == b'AAPL'"} + params = {"query": "stock_symbol == AAPL"} params["Limit"] = 1 update_value = {"open": 999} payload = {"value": update_value} @@ -540,10 +622,10 @@ def testPutQuery(self): self.assertEqual(rsp.status_code, 200) rspJson = json.loads(rsp.text) self.assertTrue("hrefs" in rspJson) - self.assertTrue("value" in rspJson) - readData = rspJson["value"] - self.assertEqual(len(readData), 1) - self.assertEqual(readData[0], [1, "AAPL", "20170102", 999, 2933]) + self.assertTrue("indices" in rspJson) + indices = rspJson["indices"] + self.assertEqual(len(indices), 1) + self.assertEqual(indices[0], [1,]) # read values and verify the expected changes where made req = self.endpoint + "/datasets/" + dset_uuid + "/value" @@ -572,9 +654,201 @@ def testPutQuery(self): rsp = self.session.put(req, params=params, data=json.dumps(update_value), headers=headers) self.assertEqual(rsp.status_code, 200) rspJson = json.loads(rsp.text) + self.assertTrue("indices" in rspJson) + indices = rspJson["indices"] + self.assertEqual(len(indices), 0) + + # create a new dataset with an explicit chunk layout + num_elements = 12 + payload = {"type": datatype, "shape": num_elements} + chunked_layout = {"class": "H5D_CHUNKED", "dims": [6,]} + cpl = {"layout": chunked_layout} + payload["creationProperties"] = cpl + req = self.endpoint + "/datasets" + rsp = self.session.post(req, data=json.dumps(payload), headers=headers) + self.assertEqual(rsp.status_code, 201) # create dataset + + rspJson = json.loads(rsp.text) + dset_uuid = rspJson["id"] + self.assertTrue(helper.validateId(dset_uuid)) + + # confirm that we got the requested chunked layout + req = self.endpoint + "/datasets/" + dset_uuid + rsp = self.session.get(req, headers=headers) + self.assertEqual(rsp.status_code, 200) + rspJson = json.loads(rsp.text) + self.assertTrue("creationProperties" in rspJson) + cpl = rspJson["creationProperties"] + self.assertTrue("layout" in cpl) + layout = cpl["layout"] + self.assertEqual(layout["class"], "H5D_CHUNKED") + self.assertEqual(layout["dims"], [6,]) + + # link new dataset + name = "dset" + helper.getRandomName() + req = self.endpoint + "/groups/" + root_uuid + "/links/" + name + payload = {"id": dset_uuid} + rsp = self.session.put(req, data=json.dumps(payload), headers=headers) + self.assertEqual(rsp.status_code, 201) + + # write entire array + payload = {"value": value} + req = self.endpoint + "/datasets/" + dset_uuid + "/value" + rsp = self.session.put(req, data=json.dumps(payload), headers=headers) + self.assertEqual(rsp.status_code, 200) # write value + + # set just one row with AAPL to have open of 42 + params = {"query": "stock_symbol == AAPL"} + params["Limit"] = 1 + update_value = {"open": 999} + payload = {"value": update_value} + + rsp = self.session.put(req, params=params, data=json.dumps(update_value), headers=headers) + self.assertEqual(rsp.status_code, 200) + rspJson = json.loads(rsp.text) + self.assertTrue("hrefs" in rspJson) + self.assertTrue("indices" in rspJson) + indices = rspJson["indices"] + self.assertEqual(len(indices), 1) + self.assertEqual(indices[0], [1,]) + + # read values and verify the expected changes where made + req = self.endpoint + "/datasets/" + dset_uuid + "/value" + rsp = self.session.get(req, headers=headers) + self.assertEqual(rsp.status_code, 200) + rspJson = json.loads(rsp.text) + read_values = rspJson["value"] + self.assertEqual(len(read_values), len(value)) + for i in range(len(value)): + orig_item = value[i] + mod_item = read_values[i] + self.assertEqual(orig_item[0], mod_item[0]) + self.assertEqual(orig_item[1], mod_item[1]) + self.assertEqual(orig_item[3], mod_item[3]) + + if orig_item[0] == "AAPL" and i == 1: + self.assertEqual(mod_item[2], 999) + else: + self.assertEqual(orig_item[2], mod_item[2]) + + def testQueryNoneCompound(self): + + headers = helper.getRequestHeaders(domain=self.base_domain) + req = self.endpoint + "/" + + # Get root uuid + rsp = self.session.get(req, headers=headers) + self.assertEqual(rsp.status_code, 200) + rspJson = json.loads(rsp.text) + root_uuid = rspJson["root"] + helper.validateId(root_uuid) + + # create a simple type/2d dataset + nrows = 10 + ncols = 10 + shape = (nrows, ncols) + payload = {"type": "H5T_STD_I32LE", "shape": shape} + req = self.endpoint + "/datasets" + rsp = self.session.post(req, data=json.dumps(payload), headers=headers) + self.assertEqual(rsp.status_code, 201) # create dataset + + rspJson = json.loads(rsp.text) + dset_uuid = rspJson["id"] + self.assertTrue(helper.validateId(dset_uuid)) + + # link new dataset + name = "dset" + helper.getRandomName() + req = self.endpoint + "/groups/" + root_uuid + "/links/" + name + payload = {"id": dset_uuid} + rsp = self.session.put(req, data=json.dumps(payload), headers=headers) + self.assertEqual(rsp.status_code, 201) + + values = [] + for i in range(nrows): + row = [] + for j in range(ncols): + row.append(i * j) + values.append(row) + payload = {"value": values} + value_req = self.endpoint + "/datasets/" + dset_uuid + "/value" + query_req = self.endpoint + "/datasets/" + dset_uuid + "/query" + rsp = self.session.put(value_req, data=json.dumps(payload), headers=headers) + self.assertEqual(rsp.status_code, 200) # write value + + expected_indices = [] + for i in range(nrows + ncols - 1): + if i < nrows: + index = [0, i] + else: + index = [i - ncols + 1, 0] + expected_indices.append(index) + + # query for all zero values + params = {"query": "_ == 0"} + rsp = self.session.get(value_req, params=params, headers=headers) + self.assertEqual(rsp.status_code, 200) + rspJson = json.loads(rsp.text) self.assertTrue("value" in rspJson) - readData = rspJson["value"] - self.assertEqual(len(readData), 0) + data_arr = rspJson["value"] + self.assertTrue(isinstance(data_arr, list)) + self.assertEqual(len(data_arr), 19) + self.assertTrue(all(index == 0 for index in data_arr)) + + # do a query request to get the indices of all zero values + + rsp = self.session.get(query_req, params=params, headers=headers) + self.assertEqual(rsp.status_code, 200) + rspJson = json.loads(rsp.text) + self.assertTrue("indices" in rspJson) + indices = rspJson["indices"] + self.assertTrue(isinstance(indices, list)) + self.assertEqual(len(indices), nrows + ncols - 1) # 19 + self.assertEqual(indices, expected_indices) + + # update all zero values to 999 + update_value = {"value": 999} + data = json.dumps(update_value) + rsp = self.session.put(value_req, params=params, data=data, headers=headers) + self.assertEqual(rsp.status_code, 200) + rspJson = json.loads(rsp.text) + self.assertTrue("hrefs" in rspJson) + self.assertTrue("value" not in rspJson) + self.assertTrue("indices" in rspJson) + indices = rspJson["indices"] + self.assertEqual(len(indices), nrows + ncols - 1) # 19 + for i in range(nrows + ncols - 1): + index = indices[i] + if i < nrows: + self.assertEqual(index[0], 0) + self.assertEqual(index[1], i) + else: + self.assertEqual(index[0], i - ncols + 1) + self.assertEqual(index[1], 0) + + # read values and verify the expected changes where made + rsp = self.session.get(value_req, headers=headers) + self.assertEqual(rsp.status_code, 200) + rspJson = json.loads(rsp.text) + read_values = rspJson["value"] + self.assertEqual(len(read_values), nrows) + for i in range(nrows): + row = read_values[i] + self.assertEqual(len(row), ncols) + for j in range(ncols): + n = row[j] + if i == 0 or j == 0: + self.assertEqual(n, 999) + else: + self.assertEqual(n, i * j) + + # do a query request again. Should get an empty list back + rsp = self.session.get(query_req, params=params, headers=headers) + self.assertEqual(rsp.status_code, 200) + rspJson = json.loads(rsp.text) + self.assertTrue("indices" in rspJson) + indices = rspJson["indices"] + self.assertTrue(isinstance(indices, list)) + self.assertEqual(len(indices), 0) if __name__ == "__main__": diff --git a/tests/integ/value_test.py b/tests/integ/value_test.py index 66287caf..6dc669c0 100755 --- a/tests/integ/value_test.py +++ b/tests/integ/value_test.py @@ -12,6 +12,7 @@ import unittest import json import numpy as np +import base64 import helper import config import time @@ -111,9 +112,7 @@ def testPut1DDataset(self): rspJson = json.loads(rsp.text) self.assertTrue("hrefs" in rspJson) self.assertTrue("value" in rspJson) - expect_value = [ - 0, - ] + expect_value = [0, ] expect_value *= data["shape"] self.assertEqual(rspJson["value"], expect_value) @@ -594,7 +593,7 @@ def testPutSelection2DDataset(self): helper.validateId(root_uuid) # create dataset - # pass in layout specification so that we can test selection across chunk boundries + # pass in layout specification so that we can test selection across chunk boundaries data = {"type": "H5T_STD_I32LE", "shape": [45, 54]} data["creationProperties"] = { "layout": {"class": "H5D_CHUNKED", "dims": [10, 10]} @@ -798,7 +797,7 @@ def testPutNullPadStringBinary(self): if j < len(string): data[offset] = ord(string[j]) else: - data[offset] = 0 # null padd rest of the element + data[offset] = 0 # null pad rest of the element payload = {"value": data} rsp = self.session.put(req, data=data, headers=headers_bin_req) @@ -928,7 +927,7 @@ def testPutScalarDataset(self): rsp = self.session.put(req, data=json.dumps(payload), headers=headers) self.assertEqual(rsp.status_code, 201) - # read unintialized value from dataset + # read uninitialized value from dataset req = self.endpoint + "/datasets/" + dset_id + "/value" rsp = self.session.get(req, headers=headers) self.assertEqual(rsp.status_code, 200) @@ -951,6 +950,145 @@ def testPutScalarDataset(self): self.assertTrue("value" in rspJson) self.assertEqual(rspJson["value"], "Hello, world") + def testScalarDatasetInitData(self): + # Test creation/deletion of scalar dataset obj along with initial data + print("testScalarDatasetInitData", self.base_domain) + headers = helper.getRequestHeaders(domain=self.base_domain) + req = self.endpoint + "/" + + # Get root uuid + rsp = self.session.get(req, headers=headers) + self.assertEqual(rsp.status_code, 200) + rspJson = json.loads(rsp.text) + root_uuid = rspJson["root"] + helper.validateId(root_uuid) + + # create a dataset obj + data = {"type": "H5T_STD_I32LE", "shape": "H5S_SCALAR", "value": 42} + req = self.endpoint + "/datasets" + rsp = self.session.post(req, data=json.dumps(data), headers=headers) + self.assertEqual(rsp.status_code, 201) + rspJson = json.loads(rsp.text) + self.assertEqual(rspJson["attributeCount"], 0) + dset_id = rspJson["id"] + self.assertTrue(helper.validateId(dset_id)) + + # read back the obj + req = self.endpoint + "/datasets/" + dset_id + rsp = self.session.get(req, headers=headers) + self.assertEqual(rsp.status_code, 200) + rspJson = json.loads(rsp.text) + + expected_keys = [ + "id", + "shape", + "hrefs", + "creationProperties", + "attributeCount", + "created", + "lastModified", + "root", + "domain", + ] + + for name in expected_keys: + self.assertTrue(name in rspJson) + self.assertEqual(rspJson["id"], dset_id) + self.assertEqual(rspJson["root"], root_uuid) + self.assertEqual(rspJson["domain"], self.base_domain) + self.assertEqual(rspJson["attributeCount"], 0) + shape_json = rspJson["shape"] + self.assertTrue(shape_json["class"], "H5S_SCALAR") + self.assertTrue(rspJson["type"], "H5T_STD_I32LE") + + # read the data back + req += "/value" + rsp = self.session.get(req, headers=headers) + self.assertEqual(rsp.status_code, 200) + rspJson = json.loads(rsp.text) + self.assertTrue("hrefs" in rspJson) + self.assertTrue("value" in rspJson) + self.assertEqual(rspJson["value"], 42) + + def testScalarDatasetInitDataMulti(self): + # Test creation/deletion of multiple scalar dataset obj along with initial data + print("testScalarDatasetInitDataMulti", self.base_domain) + headers = helper.getRequestHeaders(domain=self.base_domain) + req = self.endpoint + "/" + + # Get root uuid + rsp = self.session.get(req, headers=headers) + self.assertEqual(rsp.status_code, 200) + rspJson = json.loads(rsp.text) + root_uuid = rspJson["root"] + helper.validateId(root_uuid) + + dataset_count = 3 + datatype = "H5T_STD_I32LE" + payload = [] + for i in range(dataset_count): + dataset_args = {"type": datatype} + dataset_args["value"] = i + payload.append(dataset_args) + + # create dataset objects + req = self.endpoint + "/datasets" + rsp = self.session.post(req, data=json.dumps(payload), headers=headers) + self.assertEqual(rsp.status_code, 201) + rspJson = json.loads(rsp.text) + + self.assertTrue("objects" in rspJson) + rsp_objs = rspJson["objects"] + self.assertEqual(len(rsp_objs), dataset_count) + + for i in range(dataset_count): + obj_json = rsp_objs[i] + self.assertEqual(obj_json["attributeCount"], 0) + dset_id = obj_json["id"] + self.assertTrue(helper.validateId(dset_id)) + self.assertTrue(dset_id.startswith("d-")) + + # read back the obj + for i in range(dataset_count): + dset_id = rsp_objs[i]["id"] + req = self.endpoint + "/datasets/" + dset_id + rsp = self.session.get(req, headers=headers) + self.assertEqual(rsp.status_code, 200) + rspJson = json.loads(rsp.text) + + expected_keys = [ + "id", + "shape", + "hrefs", + "creationProperties", + "attributeCount", + "created", + "lastModified", + "root", + "domain", + ] + + for name in expected_keys: + self.assertTrue(name in rspJson) + self.assertEqual(rspJson["id"], dset_id) + self.assertEqual(rspJson["root"], root_uuid) + self.assertEqual(rspJson["domain"], self.base_domain) + self.assertEqual(rspJson["attributeCount"], 0) + shape_json = rspJson["shape"] + self.assertTrue(shape_json["class"], "H5S_SCALAR") + self.assertTrue(rspJson["type"], "H5T_STD_I32LE") + + # read the data back + for i in range(dataset_count): + dset_id = rsp_objs[i]["id"] + req = self.endpoint + "/datasets/" + dset_id + "/value" + rsp = self.session.get(req, headers=headers) + self.assertEqual(rsp.status_code, 200) + rspJson = json.loads(rsp.text) + self.assertTrue("hrefs" in rspJson) + self.assertTrue("value" in rspJson) + self.assertEqual(rspJson["value"], i) + def testNullSpaceDataset(self): # Test attempted read/write to null space dataset print("testNullSpaceDataset", self.base_domain) @@ -1107,7 +1245,7 @@ def testPutCompound(self): self.assertEqual(len(item), 2) expected = (i * 10, i * 10 + i / 10.0) if i > 0 else (42, 0.42) self.assertEqual(item[0], expected[0]) - tol = 0.1 # tbd: investiage why results need such a high tolerance + tol = 0.1 # tbd: investigate why results need such a high tolerance self.assertTrue(abs(item[1] - expected[1]) < tol) # read back just the "temp" field of the compound type @@ -1150,6 +1288,54 @@ def testPutCompound(self): tol = 0.1 self.assertTrue(abs(item[1] - expected[1]) < tol) + # selection write with field selection + payload = {"start": 9, "stop": 10, "fields": "temp", "value": 84} + req = self.endpoint + "/datasets/" + dset1d_uuid + "/value" + rsp = self.session.put(req, data=json.dumps(payload), headers=headers) + self.assertEqual(rsp.status_code, 200) # write value + + # read back the data again + rsp = self.session.get(req, headers=headers) + self.assertEqual(rsp.status_code, 200) + rspJson = json.loads(rsp.text) + + readData = rspJson["value"] + self.assertEqual(len(readData), num_elements) + for i in range(num_elements): + item = readData[i] + self.assertEqual(len(item), 2) + x = (num_elements - i - 1) * 10 if i < 9 else 84 + y = i * 10 + i / 10 if i > 0 else 0.42 + expected = (x, y) + self.assertEqual(item[0], expected[0]) + tol = 0.1 + self.assertTrue(abs(item[1] - expected[1]) < tol) + + # selection write with field selection and base64 encoding + + # "qAAAAA==" is 168 as a 4-byte int base64 encoded + payload = {"start": 9, "stop": 10, "fields": "temp", "value_base64": "qAAAAA=="} + req = self.endpoint + "/datasets/" + dset1d_uuid + "/value" + rsp = self.session.put(req, data=json.dumps(payload), headers=headers) + self.assertEqual(rsp.status_code, 200) # write value + + # read back the data again + rsp = self.session.get(req, headers=headers) + self.assertEqual(rsp.status_code, 200) + rspJson = json.loads(rsp.text) + + readData = rspJson["value"] + self.assertEqual(len(readData), num_elements) + for i in range(num_elements): + item = readData[i] + self.assertEqual(len(item), 2) + x = (num_elements - i - 1) * 10 if i < 9 else 168 + y = i * 10 + i / 10 if i > 0 else 0.42 + expected = (x, y) + self.assertEqual(item[0], expected[0]) + tol = 0.1 + self.assertTrue(abs(item[1] - expected[1]) < tol) + # # create 2d dataset # @@ -1212,6 +1398,171 @@ def testPutCompound(self): self.assertEqual(len(item), 1) self.assertEqual(item[0], i * 10) + def testPutCompoundPointsFields(self): + # Test PUT points value update with a "fields" selection on a + # compound-type dataset - only the selected field(s) should be + # updated at each point, other fields should be left untouched. + print("testPutCompoundPointsFields", self.base_domain) + headers = helper.getRequestHeaders(domain=self.base_domain) + req = self.endpoint + "/" + + rsp = self.session.get(req, headers=headers) + self.assertEqual(rsp.status_code, 200) + rspJson = json.loads(rsp.text) + root_uuid = rspJson["root"] + + str4_type = { + "class": "H5T_STRING", "length": 4, + "charSet": "H5T_CSET_ASCII", "strPad": "H5T_STR_NULLPAD", + } + str8_type = { + "class": "H5T_STRING", "length": 8, + "charSet": "H5T_CSET_ASCII", "strPad": "H5T_STR_NULLPAD", + } + fields = ( + {"name": "symbol", "type": str4_type}, + {"name": "date", "type": str8_type}, + {"name": "open", "type": "H5T_STD_I32LE"}, + {"name": "close", "type": "H5T_STD_I32LE"}, + ) + datatype = {"class": "H5T_COMPOUND", "fields": fields} + + num_elements = 5 + payload = {"type": datatype, "shape": num_elements} + req = self.endpoint + "/datasets" + rsp = self.session.post(req, data=json.dumps(payload), headers=headers) + self.assertEqual(rsp.status_code, 201) # create dataset + rspJson = json.loads(rsp.text) + dset_uuid = rspJson["id"] + self.assertTrue(helper.validateId(dset_uuid)) + + name = "dset_compound_points" + helper.getRandomName() + req = self.endpoint + "/groups/" + root_uuid + "/links/" + name + payload = {"id": dset_uuid} + rsp = self.session.put(req, data=json.dumps(payload), headers=headers) + self.assertEqual(rsp.status_code, 201) + + # initialize full records for every element + value = [] + for i in range(num_elements): + value.append((f"SYM{i}", f"20200{i:03d}", i * 100, i * 100 + 1)) + payload = {"value": value} + req = self.endpoint + "/datasets/" + dset_uuid + "/value" + rsp = self.session.put(req, data=json.dumps(payload), headers=headers) + self.assertEqual(rsp.status_code, 200) # write value + + # update just the "open" field at points 1 and 3 + points = [1, 3] + new_open_values = [111, 333] + payload = {"points": points, "value": new_open_values} + params = {"fields": "open"} + req = self.endpoint + "/datasets/" + dset_uuid + "/value" + rsp = self.session.put(req, data=json.dumps(payload), params=params, headers=headers) + self.assertEqual(rsp.status_code, 200) # write value + + # read back full records at those points and verify only "open" changed + body = {"points": points} + rsp = self.session.post(req, data=json.dumps(body), headers=headers) + self.assertEqual(rsp.status_code, 200) + rspJson = json.loads(rsp.text) + self.assertTrue("value" in rspJson) + ret_value = rspJson["value"] + self.assertEqual(len(ret_value), 2) + + for i, pt in enumerate(points): + item = ret_value[i] + self.assertEqual(len(item), 4) + expected_symbol = f"SYM{pt}" + expected_date = f"20200{pt:03d}" + expected_close = pt * 100 + 1 + self.assertEqual(item[0], expected_symbol) + self.assertEqual(item[1], expected_date) + self.assertEqual(item[2], new_open_values[i]) # updated field + self.assertEqual(item[3], expected_close) # untouched field + + def testPutCompoundInitData(self): + headers = helper.getRequestHeaders(domain=self.base_domain) + req = self.endpoint + "/" + + # Get root uuid + rsp = self.session.get(req, headers=headers) + self.assertEqual(rsp.status_code, 200) + rspJson = json.loads(rsp.text) + root_uuid = rspJson["root"] + helper.validateId(root_uuid) + + str_type = { + "charSet": "H5T_CSET_ASCII", + "class": "H5T_STRING", + "strPad": "H5T_STR_NULLPAD", + "length": 5, + } + + fields = ( + {"name": "temp", "type": "H5T_STD_I32LE"}, + {"name": "unit", "type": str_type}, + ) + datatype = {"class": "H5T_COMPOUND", "fields": fields} + + # + # create compound scalar dataset + # + value = (42, 'C') + payload = {"type": datatype, "value": value} + req = self.endpoint + "/datasets" + rsp = self.session.post(req, data=json.dumps(payload), headers=headers) + self.assertEqual(rsp.status_code, 201) # create dataset + + rspJson = json.loads(rsp.text) + dset0d_uuid = rspJson["id"] + self.assertTrue(helper.validateId(dset0d_uuid)) + + # verify the shape of the dataset + req = self.endpoint + "/datasets/" + dset0d_uuid + rsp = self.session.get(req, headers=headers) + self.assertEqual(rsp.status_code, 200) # get dataset + rspJson = json.loads(rsp.text) + shape = rspJson["shape"] + self.assertEqual(shape["class"], "H5S_SCALAR") + + # read back the value + req = self.endpoint + "/datasets/" + dset0d_uuid + "/value" + rsp = self.session.get(req, headers=headers) + self.assertEqual(rsp.status_code, 200) + rspJson = json.loads(rsp.text) + self.assertTrue("hrefs" in rspJson) + self.assertTrue("value" in rspJson) + self.assertEqual(rspJson["value"], [42, 'C']) + + # + # create 1d dataset + # + + # make up some data + num_elements = 10 + value = [] + for i in range(num_elements): + item = (i * 10, chr(ord('A') + i)) + value.append(item) + payload = {"type": datatype, "shape": num_elements, "value": value} + req = self.endpoint + "/datasets" + rsp = self.session.post(req, data=json.dumps(payload), headers=headers) + self.assertEqual(rsp.status_code, 201) # create dataset + + rspJson = json.loads(rsp.text) + dset1d_uuid = rspJson["id"] + self.assertTrue(helper.validateId(dset1d_uuid)) + + # read back the value + req = self.endpoint + "/datasets/" + dset1d_uuid + "/value" + rsp = self.session.get(req, headers=headers) + self.assertEqual(rsp.status_code, 200) + rspJson = json.loads(rsp.text) + self.assertTrue("hrefs" in rspJson) + self.assertTrue("value" in rspJson) + self.assertEqual(len(rspJson["value"]), num_elements) + self.assertEqual(rspJson["value"][2], [20, 'C']) + def testSimpleTypeFillValue(self): # test Dataset with simple type and fill value print("testSimpleTypeFillValue", self.base_domain) @@ -1641,7 +1992,446 @@ def testPutObjRefDatasetBinary(self): data[offset] = ord(ref_value[j]) req = self.endpoint + "/datasets/" + dset_id + "/value" - rsp = self.session.put(req, data=data, headers=headers_bin_req) + rsp = self.session.put(req, data=data, headers=headers_bin_req) + self.assertEqual(rsp.status_code, 200) + + rsp = self.session.get(req, headers=headers) + self.assertEqual(rsp.status_code, 200) + rspJson = json.loads(rsp.text) + self.assertTrue("hrefs" in rspJson) + self.assertTrue("value" in rspJson) + ret_values = rspJson["value"] + self.assertEqual(ret_values[0], "groups/" + root_uuid) + self.assertEqual(ret_values[1], "") + self.assertEqual(ret_values[2], "groups/" + g1_uuid) + + def testPutRegionRefDataset(self): + # Test PUT region ref values for 1d dataset + print("testPutRegionRefDataset", self.base_domain) + + headers = helper.getRequestHeaders(domain=self.base_domain) + req = self.endpoint + "/" + + # Get root uuid + rsp = self.session.get(req, headers=headers) + self.assertEqual(rsp.status_code, 200) + rspJson = json.loads(rsp.text) + root_uuid = rspJson["root"] + helper.validateId(root_uuid) + + # create target dataset that the region refs will point into + target_data = {"type": "H5T_STD_I32LE", "shape": [10, 10]} + req = self.endpoint + "/datasets" + rsp = self.session.post(req, data=json.dumps(target_data), headers=headers) + self.assertEqual(rsp.status_code, 201) + rspJson = json.loads(rsp.text) + target_id = rspJson["id"] + self.assertTrue(helper.validateId(target_id)) + + # link target dataset as 'target' + name = "target" + req = self.endpoint + "/groups/" + root_uuid + "/links/" + name + payload = {"id": target_id} + rsp = self.session.put(req, data=json.dumps(payload), headers=headers) + self.assertEqual(rsp.status_code, 201) + + # create a dataset of region references + ref_type = {"class": "H5T_REFERENCE", "base": "H5T_STD_REF_DSETREG"} + data = {"type": ref_type, "shape": 3} + + req = self.endpoint + "/datasets" + rsp = self.session.post(req, data=json.dumps(data), headers=headers) + self.assertEqual(rsp.status_code, 201) + rspJson = json.loads(rsp.text) + dset_id = rspJson["id"] + self.assertTrue(helper.validateId(dset_id)) + + # link new dataset as 'regionrefs' + name = "regionrefs" + req = self.endpoint + "/groups/" + root_uuid + "/links/" + name + payload = {"id": dset_id} + rsp = self.session.put(req, data=json.dumps(payload), headers=headers) + self.assertEqual(rsp.status_code, 201) + + req = self.endpoint + "/datasets/" + dset_id + "/value" + + # write a hyperslab region ref, a points region ref, and a null ref + ref_values = [ + { + "id": target_id[2:], # bare uuid, no 'd-' prefix + "select_type": "H5S_SEL_HYPERSLABS", + "selection": [[[1, 2], [3, 5]]], + }, + { + "id": target_id[2:], + "select_type": "H5S_SEL_POINTS", + "selection": [[0, 0], [2, 2], [4, 4]], + }, + None, + ] + payload = {"value": ref_values} + rsp = self.session.put(req, data=json.dumps(payload), headers=headers) + self.assertEqual(rsp.status_code, 200) + + rsp = self.session.get(req, headers=headers) + self.assertEqual(rsp.status_code, 200) + rspJson = json.loads(rsp.text) + self.assertTrue("hrefs" in rspJson) + self.assertTrue("value" in rspJson) + ret_values = rspJson["value"] + + self.assertEqual(ret_values[0]["id"], target_id[2:]) + self.assertEqual(ret_values[0]["select_type"], "H5S_SEL_HYPERSLABS") + self.assertEqual(ret_values[0]["selection"], [[[1, 2], [3, 5]]]) + + self.assertEqual(ret_values[1]["id"], target_id[2:]) + self.assertEqual(ret_values[1]["select_type"], "H5S_SEL_POINTS") + self.assertEqual(ret_values[1]["selection"], [[0, 0], [2, 2], [4, 4]]) + + self.assertTrue(ret_values[2] is None) + + def testGetValueRegionRefDataset(self): + # Test GET /datasets/{id}/value?regionref=... where the regionref + # source is itself a dataset of region references + print("testGetValueRegionRefDataset", self.base_domain) + + headers = helper.getRequestHeaders(domain=self.base_domain) + req = self.endpoint + "/" + + rsp = self.session.get(req, headers=headers) + self.assertEqual(rsp.status_code, 200) + rspJson = json.loads(rsp.text) + root_uuid = rspJson["root"] + + # target dataset the region refs will point into, with real data + target_data = {"type": "H5T_STD_I32LE", "shape": [10, 10]} + req = self.endpoint + "/datasets" + rsp = self.session.post(req, data=json.dumps(target_data), headers=headers) + self.assertEqual(rsp.status_code, 201) + target_id = json.loads(rsp.text)["id"] + + req = self.endpoint + "/datasets/" + target_id + "/value" + target_values = [[i * 10 + j for j in range(10)] for i in range(10)] + rsp = self.session.put(req, data=json.dumps({"value": target_values}), headers=headers) + self.assertEqual(rsp.status_code, 200) + + # region ref (rows 1:4, cols 2:6 inclusive-end -> a 3x4 block) + hyperslab_ref = { + "id": target_id[2:], + "select_type": "H5S_SEL_HYPERSLABS", + "selection": [[[1, 2], [3, 5]]], + } + points_ref = { + "id": target_id[2:], + "select_type": "H5S_SEL_POINTS", + "selection": [[0, 0], [2, 2], [4, 4]], + } + + # dataset of 3 region refs: hyperslab, points, null + ref_type = {"class": "H5T_REFERENCE", "base": "H5T_STD_REF_DSETREG"} + data = {"type": ref_type, "shape": 3} + req = self.endpoint + "/datasets" + rsp = self.session.post(req, data=json.dumps(data), headers=headers) + self.assertEqual(rsp.status_code, 201) + ref_dset_id = json.loads(rsp.text)["id"] + + req = self.endpoint + "/groups/" + root_uuid + "/links/refs1d" + rsp = self.session.put(req, data=json.dumps({"id": ref_dset_id}), headers=headers) + self.assertEqual(rsp.status_code, 201) + + req = self.endpoint + "/datasets/" + ref_dset_id + "/value" + payload = {"value": [hyperslab_ref, points_ref, None]} + rsp = self.session.put(req, data=json.dumps(payload), headers=headers) + self.assertEqual(rsp.status_code, 200) + + # scalar dataset holding just the hyperslab ref (for the + # no-explicit-select case) + data = {"type": ref_type} + req = self.endpoint + "/datasets" + rsp = self.session.post(req, data=json.dumps(data), headers=headers) + self.assertEqual(rsp.status_code, 201) + scalar_ref_dset_id = json.loads(rsp.text)["id"] + req = self.endpoint + "/datasets/" + scalar_ref_dset_id + "/value" + rsp = self.session.put(req, data=json.dumps({"value": hyperslab_ref}), headers=headers) + self.assertEqual(rsp.status_code, 200) + + # --- happy path: hyperslab ref, explicit select --- + req = self.endpoint + "/datasets/" + target_id + "/value" + params = {"regionref": f"/datasets/{ref_dset_id}", "select": "[0]"} + rsp = self.session.get(req, params=params, headers=headers) + self.assertEqual(rsp.status_code, 200) + rspJson = json.loads(rsp.text) + expected = [row[2:6] for row in target_values[1:4]] + self.assertEqual(rspJson["value"], expected) + + # --- happy path: points ref --- + params = {"regionref": f"/datasets/{ref_dset_id}", "select": "[1]"} + rsp = self.session.get(req, params=params, headers=headers) + self.assertEqual(rsp.status_code, 200) + rspJson = json.loads(rsp.text) + expected_points = [target_values[0][0], target_values[2][2], target_values[4][4]] + self.assertEqual(rspJson["value"], expected_points) + + # --- null ref -> 204 --- + params = {"regionref": f"/datasets/{ref_dset_id}", "select": "[2]"} + rsp = self.session.get(req, params=params, headers=headers) + self.assertEqual(rsp.status_code, 204) + + # --- happy path: scalar source, no select needed --- + params = {"regionref": f"/datasets/{scalar_ref_dset_id}"} + rsp = self.session.get(req, params=params, headers=headers) + self.assertEqual(rsp.status_code, 200) + rspJson = json.loads(rsp.text) + self.assertEqual(rspJson["value"], expected) + + # --- rank mismatch: 2-D ref selection against a 1-D target -> 400 --- + data1d = {"type": "H5T_STD_I32LE", "shape": [10]} + req1d = self.endpoint + "/datasets" + rsp = self.session.post(req1d, data=json.dumps(data1d), headers=headers) + self.assertEqual(rsp.status_code, 201) + target1d_id = json.loads(rsp.text)["id"] + req1d = self.endpoint + "/datasets/" + target1d_id + "/value" + params = {"regionref": f"/datasets/{ref_dset_id}", "select": "[0]"} + rsp = self.session.get(req1d, params=params, headers=headers) + self.assertEqual(rsp.status_code, 400) + + # --- out of bounds: ref selection exceeds a smaller target -> 400 --- + small_data = {"type": "H5T_STD_I32LE", "shape": [2, 2]} + req_small = self.endpoint + "/datasets" + rsp = self.session.post(req_small, data=json.dumps(small_data), headers=headers) + self.assertEqual(rsp.status_code, 201) + small_id = json.loads(rsp.text)["id"] + req_small = self.endpoint + "/datasets/" + small_id + "/value" + params = {"regionref": f"/datasets/{ref_dset_id}", "select": "[0]"} + rsp = self.session.get(req_small, params=params, headers=headers) + self.assertEqual(rsp.status_code, 400) + + # --- wrong-type source (not a region ref dataset) -> 400 --- + params = {"regionref": f"/datasets/{target_id}", "select": "[0]"} + rsp = self.session.get(req, params=params, headers=headers) + self.assertEqual(rsp.status_code, 400) + + # --- regionref combined with query -> 400 --- + params = {"regionref": f"/datasets/{ref_dset_id}", "select": "[0]", "query": "_ > 0"} + rsp = self.session.get(req, params=params, headers=headers) + self.assertEqual(rsp.status_code, 400) + + # --- invalid regionref path form -> 400 --- + params = {"regionref": f"datasets/{ref_dset_id}"} # missing leading slash + rsp = self.session.get(req, params=params, headers=headers) + self.assertEqual(rsp.status_code, 400) + + def testGetValueRegionRefAttribute(self): + # Test GET /datasets/{id}/value?regionref=... where the regionref + # source is a scalar region-ref attribute on a group + print("testGetValueRegionRefAttribute", self.base_domain) + + headers = helper.getRequestHeaders(domain=self.base_domain) + req = self.endpoint + "/" + + rsp = self.session.get(req, headers=headers) + self.assertEqual(rsp.status_code, 200) + rspJson = json.loads(rsp.text) + root_uuid = rspJson["root"] + + target_data = {"type": "H5T_STD_I32LE", "shape": [10, 10]} + req = self.endpoint + "/datasets" + rsp = self.session.post(req, data=json.dumps(target_data), headers=headers) + self.assertEqual(rsp.status_code, 201) + target_id = json.loads(rsp.text)["id"] + + req = self.endpoint + "/datasets/" + target_id + "/value" + target_values = [[i * 10 + j for j in range(10)] for i in range(10)] + rsp = self.session.put(req, data=json.dumps({"value": target_values}), headers=headers) + self.assertEqual(rsp.status_code, 200) + + payload = {"link": {"id": root_uuid, "name": "g1_regionref_attr"}} + req = self.endpoint + "/groups" + rsp = self.session.post(req, data=json.dumps(payload), headers=headers) + self.assertEqual(rsp.status_code, 201) + group_id = json.loads(rsp.text)["id"] + + ref_type = {"class": "H5T_REFERENCE", "base": "H5T_STD_REF_DSETREG"} + attr_name = "region_ref" + value = { + "id": target_id[2:], + "select_type": "H5S_SEL_HYPERSLABS", + "selection": [[[1, 2], [3, 5]]], + } + req = self.endpoint + "/groups/" + group_id + "/attributes/" + attr_name + payload = {"type": ref_type, "value": value} + rsp = self.session.put(req, data=json.dumps(payload), headers=headers) + self.assertEqual(rsp.status_code, 201) + + # happy path - scalar attribute source, no select needed + req = self.endpoint + "/datasets/" + target_id + "/value" + params = {"regionref": f"/groups/{group_id}/attributes/{attr_name}"} + rsp = self.session.get(req, params=params, headers=headers) + self.assertEqual(rsp.status_code, 200) + rspJson = json.loads(rsp.text) + expected = [row[2:6] for row in target_values[1:4]] + self.assertEqual(rspJson["value"], expected) + + # missing attribute -> 404 + params = {"regionref": f"/groups/{group_id}/attributes/does_not_exist"} + rsp = self.session.get(req, params=params, headers=headers) + self.assertEqual(rsp.status_code, 404) + + def testPutValueRegionRefDataset(self): + # Test PUT /datasets/{id}/value?regionref=... - the selection for + # the write is taken from a stored region reference + print("testPutValueRegionRefDataset", self.base_domain) + + headers = helper.getRequestHeaders(domain=self.base_domain) + req = self.endpoint + "/" + + rsp = self.session.get(req, headers=headers) + self.assertEqual(rsp.status_code, 200) + rspJson = json.loads(rsp.text) + root_uuid = rspJson["root"] + + # target dataset the region refs will point into, initially all zero + target_data = {"type": "H5T_STD_I32LE", "shape": [10, 10]} + req = self.endpoint + "/datasets" + rsp = self.session.post(req, data=json.dumps(target_data), headers=headers) + self.assertEqual(rsp.status_code, 201) + target_id = json.loads(rsp.text)["id"] + target_value_req = self.endpoint + "/datasets/" + target_id + "/value" + + # region ref (rows 1:4, cols 2:6 inclusive-end -> a 3x4 block) + hyperslab_ref = { + "id": target_id[2:], + "select_type": "H5S_SEL_HYPERSLABS", + "selection": [[[1, 2], [3, 5]]], + } + points_ref = { + "id": target_id[2:], + "select_type": "H5S_SEL_POINTS", + "selection": [[0, 0], [2, 2], [4, 4]], + } + + # dataset of 3 region refs: hyperslab, points, null + ref_type = {"class": "H5T_REFERENCE", "base": "H5T_STD_REF_DSETREG"} + data = {"type": ref_type, "shape": 3} + req = self.endpoint + "/datasets" + rsp = self.session.post(req, data=json.dumps(data), headers=headers) + self.assertEqual(rsp.status_code, 201) + ref_dset_id = json.loads(rsp.text)["id"] + + req = self.endpoint + "/groups/" + root_uuid + "/links/put_refs1d" + rsp = self.session.put(req, data=json.dumps({"id": ref_dset_id}), headers=headers) + self.assertEqual(rsp.status_code, 201) + + req = self.endpoint + "/datasets/" + ref_dset_id + "/value" + payload = {"value": [hyperslab_ref, points_ref, None]} + rsp = self.session.put(req, data=json.dumps(payload), headers=headers) + self.assertEqual(rsp.status_code, 200) + + # --- happy path: write a 3x4 block via the hyperslab ref --- + block = [[100 + i * 10 + j for j in range(4)] for i in range(3)] + params = {"regionref": f"/datasets/{ref_dset_id}", "select": "[0]"} + rsp = self.session.put( + target_value_req, params=params, data=json.dumps({"value": block}), headers=headers + ) + self.assertEqual(rsp.status_code, 200) + + # verify it landed at rows 1:4, cols 2:6 + rsp = self.session.get(target_value_req, headers=headers) + self.assertEqual(rsp.status_code, 200) + target_values = json.loads(rsp.text)["value"] + for i in range(3): + self.assertEqual(target_values[1 + i][2:6], block[i]) + + # --- happy path: write via the points ref (3 scattered points) --- + params = {"regionref": f"/datasets/{ref_dset_id}", "select": "[1]"} + rsp = self.session.put( + target_value_req, params=params, data=json.dumps({"value": [7, 8, 9]}), headers=headers + ) + self.assertEqual(rsp.status_code, 200) + rsp = self.session.get(target_value_req, headers=headers) + target_values = json.loads(rsp.text)["value"] + self.assertEqual(target_values[0][0], 7) + self.assertEqual(target_values[2][2], 8) + self.assertEqual(target_values[4][4], 9) + + # --- shape mismatch: input doesn't match the ref's selection shape -> 400 --- + params = {"regionref": f"/datasets/{ref_dset_id}", "select": "[0]"} + bad_block = [[1, 2], [3, 4]] # 2x2, but ref selects a 3x4 block + rsp = self.session.put( + target_value_req, params=params, data=json.dumps({"value": bad_block}), headers=headers + ) + self.assertEqual(rsp.status_code, 400) + + # --- null ref -> 400 --- + params = {"regionref": f"/datasets/{ref_dset_id}", "select": "[2]"} + rsp = self.session.put( + target_value_req, params=params, data=json.dumps({"value": block}), headers=headers + ) + self.assertEqual(rsp.status_code, 400) + + # --- regionref combined with query -> 400 --- + params = {"regionref": f"/datasets/{ref_dset_id}", "select": "[0]", "query": "_ > 0"} + rsp = self.session.put( + target_value_req, params=params, data=json.dumps({"value": block}), headers=headers + ) + self.assertEqual(rsp.status_code, 400) + + # --- regionref combined with points -> 400 --- + params = {"regionref": f"/datasets/{ref_dset_id}"} + rsp = self.session.put( + target_value_req, + params=params, + data=json.dumps({"points": [[0, 0]], "value": [1]}), + headers=headers, + ) + self.assertEqual(rsp.status_code, 400) + + def testPutOpaqueDataset(self): + # Test PUT/GET opaque dataset values + print("testPutOpaqueDataset", self.base_domain) + + headers = helper.getRequestHeaders(domain=self.base_domain) + req = self.endpoint + "/" + + rsp = self.session.get(req, headers=headers) + self.assertEqual(rsp.status_code, 200) + rspJson = json.loads(rsp.text) + root_uuid = rspJson["root"] + + opaque_type = {"class": "H5T_OPAQUE", "size": 7} + data = {"type": opaque_type, "shape": 4} + req = self.endpoint + "/datasets" + rsp = self.session.post(req, data=json.dumps(data), headers=headers) + self.assertEqual(rsp.status_code, 201) + rspJson = json.loads(rsp.text) + dset_id = rspJson["id"] + self.assertTrue(helper.validateId(dset_id)) + + name = "opaque_dset" + req = self.endpoint + "/groups/" + root_uuid + "/links/" + name + payload = {"id": dset_id} + rsp = self.session.put(req, data=json.dumps(payload), headers=headers) + self.assertEqual(rsp.status_code, 201) + + req = self.endpoint + "/datasets/" + dset_id + "/value" + + # before any write, values should default to all-zero -> "" + rsp = self.session.get(req, headers=headers) + self.assertEqual(rsp.status_code, 200) + rspJson = json.loads(rsp.text) + self.assertTrue("hrefs" in rspJson) + self.assertTrue("value" in rspJson) + self.assertEqual(rspJson["value"], [""] * data["shape"]) + + # write opaque values, including an empty (all-zero) element + values = [ + base64.b64encode(b"OPAQUE0").decode("ascii"), + base64.b64encode(b"OPAQUE1").decode("ascii"), + base64.b64encode(b"OPAQUE2").decode("ascii"), + "", + ] + payload = {"value": values} + rsp = self.session.put(req, data=json.dumps(payload), headers=headers) self.assertEqual(rsp.status_code, 200) rsp = self.session.get(req, headers=headers) @@ -1649,10 +2439,7 @@ def testPutObjRefDatasetBinary(self): rspJson = json.loads(rsp.text) self.assertTrue("hrefs" in rspJson) self.assertTrue("value" in rspJson) - ret_values = rspJson["value"] - self.assertEqual(ret_values[0], "groups/" + root_uuid) - self.assertEqual(ret_values[1], "") - self.assertEqual(ret_values[2], "groups/" + g1_uuid) + self.assertEqual(rspJson["value"], values) def testGet(self): domain = helper.getTestDomain("tall.h5") @@ -1911,7 +2698,7 @@ def testResizable1DValue(self): # read values from the extended region req = self.endpoint + "/datasets/" + dset_uuid + "/value" - params = {"select": "[{}:{}]".format(0, num_elements)} + params = {"select": f"[0:{num_elements}]"} rsp = self.session.get(req, params=params, headers=headers) self.assertEqual(rsp.status_code, 200) rspJson = json.loads(rsp.text) @@ -2038,7 +2825,7 @@ def testAppend1DJson(self): data = rspJson["value"] self.assertEqual(data, list(range(num_elements * 2))) - # test mis-match of append value and data + # test mismatch of append value and data value = list(range(num_elements, num_elements * 2)) payload = {"value": value, "append": num_elements + 1} req = self.endpoint + "/datasets/" + dset_uuid + "/value" @@ -2139,7 +2926,7 @@ def testAppend1DBinary(self): read_values = rspJson["value"] self.assertEqual(read_values, list(range(num_elements * 2))) - # test mis-match of append value and data + # test mismatch of append value and data req = self.endpoint + "/datasets/" + dset_uuid + "/value" params = {"append": num_elements + 1} rsp = self.session.put(req, data=data, params=params, headers=headers_bin_req) @@ -2262,7 +3049,7 @@ def testAppend2DJson(self): self.assertEqual(data[0], list(range(num_elements))) self.assertEqual(data[1], list(range(num_elements))) - # test mis-match of append value and data + # test mismatch of append value and data value = list(range(num_elements, num_elements * 2)) payload = {"value": value, "append": num_elements + 1} req = self.endpoint + "/datasets/" + dset_uuid + "/value" @@ -2496,7 +3283,7 @@ def testGetSelectionChunkedRefDataset(self): # read a selection req = self.endpoint + "/datasets/" + dset_id + "/value" params = {"select": "[1234567:1234568]"} # read 1 element, starting at index 1234567 - params["nonstrict"] = 1 # allow use of aws lambda if configured + params["nonstrict"] = 1 # enable nonstrict mode rsp = self.session.get(req, params=params, headers=headers) if rsp.status_code == 404: print(f"s3object: {s3path} not found, skipping {test_name}") @@ -2661,7 +3448,7 @@ def testChunkedRefIndirectDataset(self): params = { "select": "[1234567:1234568]" } # read 1 element, starting at index 1234567 - params["nonstrict"] = 1 # enable SN to invoke lambda func + params["nonstrict"] = 1 # enable nonstrict mode rsp = self.session.get(req, params=params, headers=headers) self.assertEqual(rsp.status_code, 200) @@ -2843,7 +3630,7 @@ def testChunkedRefIndirectS3UriDataset(self): params = { "select": "[1234567:1234568]" } # read 1 element, starting at index 1234567 - params["nonstrict"] = 1 # enable SN to invoke lambda func + params["nonstrict"] = 1 # enable nonstrict mode rsp = self.session.get(req, params=params, headers=headers) if rsp.status_code == 404: @@ -2979,7 +3766,7 @@ def testChunkInitializerDataset(self): params = { "select": "[1234567:1234568]" } # read 1 element, starting at index 1234567 - params["nonstrict"] = 1 # enable SN to invoke lambda func + params["nonstrict"] = 1 # enable nonstrict mode # read the selection rsp = self.session.get(req, params=params, headers=headers) @@ -3014,7 +3801,7 @@ def testARangeInitializerDataset(self): extent = 1_000_000_000 # one billion elements dset_dims = [extent, ] layout = {"class": "H5D_CHUNKED"} - layout["dims"] = dset_dims + layout["dims"] = [1_000, ] range_start = 0 # -0.25 range_step = 1 @@ -3046,7 +3833,7 @@ def testARangeInitializerDataset(self): sel_start = 19_531_260 # 20_000_000 # 123_456_789 sel_stop = sel_start + count params = {"select": f"[{sel_start}:{sel_stop}]"} # read 10 elements - params["nonstrict"] = 1 # enable SN to invoke lambda func + params["nonstrict"] = 1 # enable nonstrict mode rsp = self.session.get(req, params=params, headers=headers) self.assertEqual(rsp.status_code, 200) @@ -3064,6 +3851,7 @@ def testARangeInitializerDataset(self): def testIntelligentRangeGet1D(self): test_name = "testIntelligentRangeGet1D" + print(test_name, self.base_domain) headers = helper.getRequestHeaders(domain=self.base_domain) @@ -3161,7 +3949,7 @@ def testIntelligentRangeGet1D(self): start = 1234567 stop = start + 10 params = {"select": f"[{start}:{stop}]"} # read 10 element, starting at index 1234567 - params["nonstrict"] = 1 # enable SN to invoke lambda func + params["nonstrict"] = 1 # enable nonstrict mode # read the selection rsp = self.session.get(req, params=params, headers=headers) @@ -3276,7 +4064,7 @@ def testIntelligentRangeGet2D(self): col_index = 123 # read 10 element, starting at index 1234 params = {"select": f"[{start}:{stop}, {col_index}]"} - params["nonstrict"] = 1 # enable SN to invoke lambda func + params["nonstrict"] = 1 # enable nonstrict mode # read the selection rsp = self.session.get(req, params=params, headers=headers) @@ -3395,7 +4183,7 @@ def testIntelligentRangeGet3D(self): count = 10 # read 10 element, starting at index (x,y,z) params = {"select": f"[{x}, {y}, {z}:{z + count}]"} - params["nonstrict"] = 1 # enable SN to invoke lambda func + params["nonstrict"] = 1 # enable nonstrict mode # read the selection rsp = self.session.get(req, params=params, headers=headers) @@ -3504,7 +4292,7 @@ def testIntelligentRangeGetFillValue(self): start = 1234567 stop = start + 10 params = {"select": f"[{start}:{stop}]"} # read 10 element, starting at index 1234567 - params["nonstrict"] = 1 # enable SN to invoke lambda func + params["nonstrict"] = 1 # enable nonstrict mode # read the selection rsp = self.session.get(req, params=params, headers=headers) @@ -3519,7 +4307,7 @@ def testIntelligentRangeGetFillValue(self): self.assertEqual(value, [42,] * 10) def testLargeCreationProperties(self): - # test Dataset with artifically large creation_properties data + # test Dataset with artificially large creation_properties data print("testLargeCreationProperties", self.base_domain) headers = helper.getRequestHeaders(domain=self.base_domain) @@ -4166,6 +4954,290 @@ def testCreateArrayDatasetBinary(self): self.assertEqual(rsp.status_code, 200) self.assertEqual(rsp.content, data) + def testArrayFieldPartialSelection(self): + # partial (hyperslab) read/write for a compound dtype with an + # array-typed field - testCreateArrayDataset/ + # testCreateArrayDatasetBinary above only exercise a full-value + # write/read, this fills that gap + headers = helper.getRequestHeaders(domain=self.base_domain) + req = self.endpoint + "/" + + rsp = self.session.get(req, headers=headers) + self.assertEqual(rsp.status_code, 200) + + array_dims = [5] + num_arrays = 4 + + array_datatype = { + "class": "H5T_ARRAY", + "base": { + "class": "H5T_INTEGER", + "base": "H5T_STD_I64LE" + }, + "dims": array_dims + } + fields = ( + {"name": "temp", "type": array_datatype}, + {"name": "pressure", "type": "H5T_IEEE_F32LE"}, + ) + datatype = {"class": "H5T_COMPOUND", "fields": fields} + + payload = {"type": datatype, "shape": num_arrays} + req = self.endpoint + "/datasets" + rsp = self.session.post(req, data=json.dumps(payload), headers=headers) + self.assertEqual(rsp.status_code, 201) # create dataset + + rspJson = json.loads(rsp.text) + array_dset_uuid = rspJson["id"] + self.assertTrue(helper.validateId(array_dset_uuid)) + + # write the full array + value = [] + for i in range(num_arrays): + value.append([[i * 10 + j for j in range(5)], i + 0.5]) + payload = {"value": value} + req = self.endpoint + "/datasets/" + array_dset_uuid + "/value" + rsp = self.session.put(req, data=json.dumps(payload), headers=headers) + self.assertEqual(rsp.status_code, 200) # write value + + # partial read of just elements [1:3] + params = {"select": "[1:3]"} + rsp = self.session.get(req, params=params, headers=headers) + self.assertEqual(rsp.status_code, 200) + rspJson = json.loads(rsp.text) + self.assertEqual(rspJson["value"], value[1:3]) + + # partial write to just elements [1:3] + new_value = [[[100, 101, 102, 103, 104], 9.5], [[200, 201, 202, 203, 204], 19.5]] + payload = {"value": new_value, "start": 1, "stop": 3} + rsp = self.session.put(req, data=json.dumps(payload), headers=headers) + self.assertEqual(rsp.status_code, 200) # write value + + # read back the whole array and confirm just [1:3] changed + rsp = self.session.get(req, headers=headers) + self.assertEqual(rsp.status_code, 200) + rspJson = json.loads(rsp.text) + expected = [value[0]] + new_value + [value[3]] + self.assertEqual(rspJson["value"], expected) + + def testArrayFieldResize(self): + # resizing a resizable dataset whose dtype is a compound with an + # array-typed field - not covered by the other array-field tests + headers = helper.getRequestHeaders(domain=self.base_domain) + req = self.endpoint + "/" + + rsp = self.session.get(req, headers=headers) + self.assertEqual(rsp.status_code, 200) + + array_dims = [3] + num_arrays = 2 + + array_datatype = { + "class": "H5T_ARRAY", + "base": { + "class": "H5T_INTEGER", + "base": "H5T_STD_I32LE" + }, + "dims": array_dims + } + fields = ( + {"name": "vec", "type": array_datatype}, + {"name": "scale", "type": "H5T_IEEE_F32LE"}, + ) + datatype = {"class": "H5T_COMPOUND", "fields": fields} + + payload = {"type": datatype, "shape": [num_arrays], "maxdims": [0]} + req = self.endpoint + "/datasets" + rsp = self.session.post(req, data=json.dumps(payload), headers=headers) + self.assertEqual(rsp.status_code, 201) # create dataset + + rspJson = json.loads(rsp.text) + dset_uuid = rspJson["id"] + self.assertTrue(helper.validateId(dset_uuid)) + + # write the initial elements + value = [[[1, 2, 3], 1.5], [[4, 5, 6], 2.5]] + payload = {"value": value} + req = self.endpoint + "/datasets/" + dset_uuid + "/value" + rsp = self.session.put(req, data=json.dumps(payload), headers=headers) + self.assertEqual(rsp.status_code, 200) # write value + + # resize to double the extent + num_arrays *= 2 + req = self.endpoint + "/datasets/" + dset_uuid + "/shape" + payload = {"shape": [num_arrays]} + rsp = self.session.put(req, data=json.dumps(payload), headers=headers) + self.assertEqual(rsp.status_code, 201) + + # original elements should be preserved + req = self.endpoint + "/datasets/" + dset_uuid + "/value" + params = {"select": "[0:2]"} + rsp = self.session.get(req, params=params, headers=headers) + self.assertEqual(rsp.status_code, 200) + rspJson = json.loads(rsp.text) + self.assertEqual(rspJson["value"], value) + + # write to the newly extended region + new_value = [[[7, 8, 9], 3.5], [[10, 11, 12], 4.5]] + payload = {"value": new_value, "start": 2, "stop": 4} + rsp = self.session.put(req, data=json.dumps(payload), headers=headers) + self.assertEqual(rsp.status_code, 200) # write value + + # read back the whole array and confirm the new region is correct + rsp = self.session.get(req, headers=headers) + self.assertEqual(rsp.status_code, 200) + rspJson = json.loads(rsp.text) + self.assertEqual(rspJson["value"], value + new_value) + + def testArrayFieldFieldSelection(self): + # field-restricted read/write on a compound dataset that has an + # array-typed field - field selection is otherwise only tested + # against plain (all-scalar-field) compounds, and array-field + # selection is otherwise only tested for whole-record access + # (testCreateArrayDataset/testArrayFieldPartialSelection above). + # The write case specifically must not disturb the untouched field. + headers = helper.getRequestHeaders(domain=self.base_domain) + req = self.endpoint + "/" + + rsp = self.session.get(req, headers=headers) + self.assertEqual(rsp.status_code, 200) + + array_dims = [3] + num_elements = 4 + + array_datatype = { + "class": "H5T_ARRAY", + "base": { + "class": "H5T_INTEGER", + "base": "H5T_STD_I32LE" + }, + "dims": array_dims + } + fields = ( + {"name": "vec", "type": array_datatype}, + {"name": "scale", "type": "H5T_IEEE_F32LE"}, + ) + datatype = {"class": "H5T_COMPOUND", "fields": fields} + + payload = {"type": datatype, "shape": num_elements} + req = self.endpoint + "/datasets" + rsp = self.session.post(req, data=json.dumps(payload), headers=headers) + self.assertEqual(rsp.status_code, 201) # create dataset + + rspJson = json.loads(rsp.text) + dset_uuid = rspJson["id"] + self.assertTrue(helper.validateId(dset_uuid)) + + # write the full array + value = [] + for i in range(num_elements): + value.append([[i, i + 1, i + 2], i + 0.5]) + payload = {"value": value} + req = self.endpoint + "/datasets/" + dset_uuid + "/value" + rsp = self.session.put(req, data=json.dumps(payload), headers=headers) + self.assertEqual(rsp.status_code, 200) # write value + + # read just the scalar "scale" field + params = {"fields": "scale"} + rsp = self.session.get(req, params=params, headers=headers) + self.assertEqual(rsp.status_code, 200) + rspJson = json.loads(rsp.text) + expected_scale = [[v[1]] for v in value] + self.assertEqual(rspJson["value"], expected_scale) + + # field-restricted write to just "scale" for elements [1:3] + new_scale = [77.5, 88.5] + payload = {"start": 1, "stop": 3, "fields": "scale", "value": new_scale} + rsp = self.session.put(req, data=json.dumps(payload), headers=headers) + self.assertEqual(rsp.status_code, 200) # write value + + # the array-typed "vec" field must be completely untouched, for + # every element (not just the ones the "scale" write targeted) + rsp = self.session.get(req, headers=headers) + self.assertEqual(rsp.status_code, 200) + rspJson = json.loads(rsp.text) + out_value = rspJson["value"] + for i in range(num_elements): + self.assertEqual(out_value[i][0], value[i][0]) # vec unchanged + self.assertEqual(out_value[0][1], value[0][1]) + self.assertEqual(out_value[1][1], new_scale[0]) + self.assertEqual(out_value[2][1], new_scale[1]) + self.assertEqual(out_value[3][1], value[3][1]) + + # field-restricted write to just the array-typed "vec" field, for + # element 0 only + new_vec = [[100, 101, 102]] + payload = {"start": 0, "stop": 1, "fields": "vec", "value": new_vec} + rsp = self.session.put(req, data=json.dumps(payload), headers=headers) + self.assertEqual(rsp.status_code, 200) # write value + + # "scale" must be untouched by the "vec"-only write, for every element + rsp = self.session.get(req, headers=headers) + self.assertEqual(rsp.status_code, 200) + rspJson = json.loads(rsp.text) + out_value = rspJson["value"] + self.assertEqual(out_value[0][0], new_vec[0]) + self.assertEqual(out_value[0][1], value[0][1]) + self.assertEqual(out_value[1][1], new_scale[0]) + self.assertEqual(out_value[2][1], new_scale[1]) + self.assertEqual(out_value[3][1], value[3][1]) + for i in range(1, num_elements): + self.assertEqual(out_value[i][0], value[i][0]) # vec unchanged + + def testArrayFieldSingleFieldReadArrayField(self): + # reading a SINGLE field of a compound dataset, where that one + # field is itself array-typed - see util/chunkUtil.py's + # chunkReadSelection() and tests/unit/chunk_util_test.py's + # testChunkReadSelectionSingleArrayField for the unit-level fix + # this was covering. + headers = helper.getRequestHeaders(domain=self.base_domain) + req = self.endpoint + "/" + + rsp = self.session.get(req, headers=headers) + self.assertEqual(rsp.status_code, 200) + + array_dims = [3] + num_elements = 4 + + array_datatype = { + "class": "H5T_ARRAY", + "base": { + "class": "H5T_INTEGER", + "base": "H5T_STD_I32LE" + }, + "dims": array_dims + } + fields = ( + {"name": "vec", "type": array_datatype}, + {"name": "scale", "type": "H5T_IEEE_F32LE"}, + ) + datatype = {"class": "H5T_COMPOUND", "fields": fields} + + payload = {"type": datatype, "shape": num_elements} + req = self.endpoint + "/datasets" + rsp = self.session.post(req, data=json.dumps(payload), headers=headers) + self.assertEqual(rsp.status_code, 201) # create dataset + + rspJson = json.loads(rsp.text) + dset_uuid = rspJson["id"] + self.assertTrue(helper.validateId(dset_uuid)) + + value = [] + for i in range(num_elements): + value.append([[i, i + 1, i + 2], i + 0.5]) + payload = {"value": value} + req = self.endpoint + "/datasets/" + dset_uuid + "/value" + rsp = self.session.put(req, data=json.dumps(payload), headers=headers) + self.assertEqual(rsp.status_code, 200) # write value + + # read just the array-typed "vec" field + params = {"fields": "vec"} + rsp = self.session.get(req, params=params, headers=headers) + self.assertEqual(rsp.status_code, 200) + rspJson = json.loads(rsp.text) + expected_vec = [[v[0]] for v in value] + self.assertEqual(rspJson["value"], expected_vec) + def testCreateNestedArrayDataset(self): headers = helper.getRequestHeaders(domain=self.base_domain) req = self.endpoint + "/" @@ -4216,6 +5288,66 @@ def testCreateNestedArrayDataset(self): self.assertEqual(shape["class"], "H5S_SIMPLE") self.assertEqual(shape["dims"], [num_nested_arrays]) + def testCreateBareArrayDataset(self): + # unlike testCreateArrayDataset/testCreateArrayDatasetBinary above + # (where the H5T_ARRAY type is one field of a compound), this + # exercises a dataset whose *own* top-level type is H5T_ARRAY - + # see tests/unit/chunk_util_test.py's + # testChunkWriteSelectionArrayDtype for the underlying fix. + headers = helper.getRequestHeaders(domain=self.base_domain) + headers_bin_req = helper.getRequestHeaders(domain=self.base_domain) + headers_bin_req["Content-Type"] = "application/octet-stream" + headers_bin_rsp = helper.getRequestHeaders(domain=self.base_domain) + headers_bin_rsp["accept"] = "application/octet-stream" + + req = self.endpoint + "/" + rsp = self.session.get(req, headers=headers) + self.assertEqual(rsp.status_code, 200) + + array_dims = [3] + num_elements = 2 + + array_datatype = { + "class": "H5T_ARRAY", + "base": { + "class": "H5T_INTEGER", + "base": "H5T_STD_I8LE" + }, + "dims": array_dims + } + + payload = {"type": array_datatype, "shape": num_elements} + req = self.endpoint + "/datasets" + rsp = self.session.post(req, data=json.dumps(payload), headers=headers) + self.assertEqual(rsp.status_code, 201) # create dataset + + rspJson = json.loads(rsp.text) + array_dset_uuid = rspJson["id"] + self.assertTrue(helper.validateId(array_dset_uuid)) + + # verify the shape of the dataset + req = self.endpoint + "/datasets/" + array_dset_uuid + rsp = self.session.get(req, headers=headers) + self.assertEqual(rsp.status_code, 200) # get dataset + rspJson = json.loads(rsp.text) + shape = rspJson["shape"] + self.assertEqual(shape["class"], "H5S_SIMPLE") + self.assertEqual(shape["dims"], [num_elements]) + + # write dataset values + dt = np.dtype(("i1", tuple(array_dims))) + data = np.frombuffer(bytes([1, 2, 3, 4, 5, 6]), dtype=dt) + self.assertEqual(data.shape, (num_elements,) + tuple(array_dims)) + + req = self.endpoint + "/datasets/" + array_dset_uuid + "/value" + rsp = self.session.put(req, data=data.tobytes(), headers=headers_bin_req) + self.assertEqual(rsp.status_code, 200) # write value + + # read the dataset values back + rsp = self.session.get(req, headers=headers_bin_rsp) + self.assertEqual(rsp.status_code, 200) + self.assertEqual(rsp.content, data.tobytes()) + if __name__ == "__main__": # setup test files diff --git a/tests/integ/vlen_test.py b/tests/integ/vlen_test.py index e45504e6..28bb3e90 100755 --- a/tests/integ/vlen_test.py +++ b/tests/integ/vlen_test.py @@ -13,18 +13,15 @@ import json import helper import numpy as np -import sys -sys.path.append("../..") -from hsds.util.arrayUtil import arrayToBytes, bytesToArray -from hsds.util.hdf5dtype import createDataType +from h5json.hdf5dtype import createDataType +from h5json.array_util import arrayToBytes, bytesToArray class VlenTest(unittest.TestCase): def __init__(self, *args, **kwargs): super(VlenTest, self).__init__(*args, **kwargs) self.base_domain = helper.getTestDomainName(self.__class__.__name__) - print(self.base_domain) helper.setupDomain(self.base_domain) self.endpoint = helper.getEndpoint() @@ -38,7 +35,7 @@ def tearDown(self): # main def testPutVLenInt(self): - # Test PUT value for 1d attribute with variable length int types + # Test PUT value for 1d dataset with variable length int types print("testPutVLenInt", self.base_domain) headers = helper.getRequestHeaders(domain=self.base_domain) @@ -123,7 +120,7 @@ def testPutVLenInt(self): self.assertEqual(value[1], [1, 2, 3, 4]) def testPutVLenIntBinary(self): - # Test PUT value for 1d attribute with variable length int types using binary transfer + # Test PUT value for 1d dataset with variable length int types using binary transfer print("testPutVLenIntBinary", self.base_domain) count = 4 @@ -220,7 +217,7 @@ def testPutVLenIntBinary(self): self.assertEqual(value[0], [1, 2, 3]) def testPutVLen2DInt(self): - # Test PUT value for 1d attribute with variable length int types + # Test PUT value for 1d dataset with variable length int types print("testPutVLen2DInt", self.base_domain) nrow = 2 ncol = 2 @@ -297,7 +294,7 @@ def testPutVLen2DInt(self): self.assertEqual(value[0][1], [1, 2]) def testPutVLenString(self): - # Test PUT value for 1d attribute with variable length string types + # Test PUT value for 1d dataset with variable length string types print("testPutVLenString", self.base_domain) headers = helper.getRequestHeaders(domain=self.base_domain) @@ -367,7 +364,7 @@ def testPutVLenString(self): self.assertEqual(value[1], data[3]) def testPutVLenStringBinary(self): - # Test PUT value for 1d attribute with variable length string types + # Test PUT value for 1d dataset with variable length string types print("testPutVLenStringBinary", self.base_domain) headers = helper.getRequestHeaders(domain=self.base_domain) diff --git a/tests/lambda/config.yml b/tests/lambda/config.yml deleted file mode 100644 index d3691404..00000000 --- a/tests/lambda/config.yml +++ /dev/null @@ -1,18 +0,0 @@ -# config values for HSDS unit tests -root_dir: null -bucket_name: "hsdstest" -sn_port: 0 -hsds_unit_test_bucket: null -azure_connection_string: null -aws_s3_gateway: null -log_level: ERROR -cors_domain: "*" -metadata_mem_cache_size: 128m # 128 MB - metadata cache size per DN node -metadata_mem_cache_expire: 3600 # expire cache items after one hour -chunk_mem_cache_size: 128m # 128 MB - chunk cache size per DN node -chunk_mem_cache_expire: 3600 # expire cache items after one hour -allow_noauth: True -aws_dynamodb_gateway: null -password_salt: null -password_file: null -groups_file: null diff --git a/tests/lambda/lambdaTest.py b/tests/lambda/lambdaTest.py deleted file mode 100644 index da2a7e24..00000000 --- a/tests/lambda/lambdaTest.py +++ /dev/null @@ -1,37 +0,0 @@ -############################################################################## -# Copyright by The HDF Group. # -# All rights reserved. # -# # -# This file is part of HSDS (HDF5 Scalable Data Service), Libraries and # -# Utilities. The full HSDS copyright notice, including # -# terms governing use, modification, and redistribution, is contained in # -# the file COPYING, which can be found at the root of the source code # -# distribution tree. If you do not have access to this file, you may # -# request a copy from help@hdfgroup.org. # -############################################################################## -import sys -import unittest - -sys.path.append("../..") - -from hsds.lambda_function import lambda_handler - - -class LambdaTest(unittest.TestCase): - def __init__(self, *args, **kwargs): - super(LambdaTest, self).__init__(*args, **kwargs) - # main - - def testInvoke(self): - event = {"method": "GET", "route": "/about"} - context = {} - rsp = lambda_handler(event, context) - print("rsp:", rsp) - self.assertTrue("statusCode" in rsp) - self.assertEqual(rsp["statusCode"], 200) - - -if __name__ == "__main__": - # setup test files - - unittest.main() diff --git a/tests/load/kubeptwrite/setup.py b/tests/load/kubeptwrite/setup.py index e7844478..704632c9 100644 --- a/tests/load/kubeptwrite/setup.py +++ b/tests/load/kubeptwrite/setup.py @@ -80,7 +80,7 @@ def getRequestHeaders(domain=None, username=None, password=None, **kwargs): req = endpoint + "/datasets" rsp = session.post(req, data=json.dumps(payload), headers=headers) if rsp.status_code != 201: - sys.exit("Failed to create datset: {}".format(rsp.status_code)) + sys.exit("Failed to create dataset: {}".format(rsp.status_code)) rsp_json = json.loads(rsp.text) dset_id = rsp_json["id"] @@ -92,7 +92,7 @@ def getRequestHeaders(domain=None, username=None, password=None, **kwargs): print(dset_id) rsp = session.put(req, data=json.dumps(payload), headers=headers) if rsp.status_code != 201: - sys.exit("Failed to link datset: {}".format(rsp.status_code)) + sys.exit("Failed to link dataset: {}".format(rsp.status_code)) rsp_json = json.loads(rsp.text) session.close() diff --git a/tests/load/kubewrite/setup.py b/tests/load/kubewrite/setup.py index fdbaa04c..008b754e 100644 --- a/tests/load/kubewrite/setup.py +++ b/tests/load/kubewrite/setup.py @@ -75,7 +75,7 @@ def getRequestHeaders(domain=None, username=None, password=None, **kwargs): req = endpoint + "/datasets" rsp = session.post(req, data=json.dumps(payload), headers=headers) if rsp.status_code != 201: - sys.exit("Failed to create datset: {}".format(rsp.status_code)) + sys.exit("Failed to create dataset: {}".format(rsp.status_code)) rsp_json = json.loads(rsp.text) dset_id = rsp_json["id"] @@ -87,7 +87,7 @@ def getRequestHeaders(domain=None, username=None, password=None, **kwargs): print(dset_id) rsp = session.put(req, data=json.dumps(payload), headers=headers) if rsp.status_code != 201: - sys.exit("Failed to link datset: {}".format(rsp.status_code)) + sys.exit("Failed to link dataset: {}".format(rsp.status_code)) rsp_json = json.loads(rsp.text) session.close() diff --git a/tests/perf/append/append_1d.py b/tests/perf/append/append_1d.py index b81e948b..380641ac 100644 --- a/tests/perf/append/append_1d.py +++ b/tests/perf/append/append_1d.py @@ -23,7 +23,7 @@ def usage(): def addRow(dset): - """add a row to the datset""" + """add a row to the dataset""" rows = dset.shape[0] now = time.time() sensor = random.randrange(0, 16) diff --git a/tests/perf/append/append_1d_async.py b/tests/perf/append/append_1d_async.py index 0ba73f20..8782a870 100644 --- a/tests/perf/append/append_1d_async.py +++ b/tests/perf/append/append_1d_async.py @@ -195,7 +195,7 @@ async def work(self): logging.info(f"task: {sensor_id}: exiting") async def getDatasetId(self, session): - """Get the datset id for the dataset (hopefully) at H5_PATH""" + """Get the dataset id for the dataset (hopefully) at H5_PATH""" headers = self.getHeaders() dsetid = None req = f"{self.endpoint}/datasets/" @@ -226,7 +226,7 @@ async def addrow(self, session, dsetid, sensor_id, seq_num): headers = self.getHeaders() req = f"{self.endpoint}/datasets/{dsetid}/value" # the 'append' param enables the row to be added to - # the end of the datset without explictly extending the + # the end of the dataset without explicitly extending the # dataspace payload = {"value": row, "append": 1} diff --git a/tests/perf/async_http/async_client.py b/tests/perf/async_http/async_client.py index cf923e61..01bb17c4 100644 --- a/tests/perf/async_http/async_client.py +++ b/tests/perf/async_http/async_client.py @@ -29,9 +29,6 @@ def print_responses(results): print("num results: {}".format(len(results))) for res in results: print(res) - # delay = res.headers.get("DELAY") - # d = res.headers.get("DATE") - # print("{}:{} delay {}".format(d, res.url, delay)) loop = asyncio.get_event_loop() diff --git a/tests/perf/nrel/wtk/wtk_conus.py b/tests/perf/nrel/wtk/wtk_conus.py index 0f608a1d..a98fa485 100644 --- a/tests/perf/nrel/wtk/wtk_conus.py +++ b/tests/perf/nrel/wtk/wtk_conus.py @@ -73,7 +73,7 @@ def print_stats(filepath, index, arr): msg += "[--h5path=dataset_name] [--index=index] [--bucket=bucket_name] " msg += "[--iter=count] [--usecache] [--mp]" print(msg) - print(f" --folder: path to wtk conus files (defalt: {folderpath})") + print(f" --folder: path to wtk conus files (default: {folderpath})") print(" path can be an HSDS domain path or s3 uri to HDF5 files") print(" example: --folder=s3://nrel-pds-wtk/conus/v1.0.0/") print(f" --h5path: hdf5 path to dataset (default: {cfg['h5path']})") diff --git a/tests/perf/smallobj/small_obj_test.py b/tests/perf/smallobj/small_obj_test.py index f1c87e63..22683b77 100644 --- a/tests/perf/smallobj/small_obj_test.py +++ b/tests/perf/smallobj/small_obj_test.py @@ -102,7 +102,7 @@ async def create_group(parent_grp_id, grp_name): async def create_dataset(parent_grp_id, dataset_name, value=None): - logging.info("create_dataset: {dataset_name}") + logging.info(f"create_dataset: {dataset_name}") req = globals["endpoint"] + "/datasets" headers = get_headers() params = {"domain": globals["domain"]} @@ -241,7 +241,7 @@ async def store_items(grp_names): elif log_level == "error": logging.basicConfig(format='%(asctime)s %(message)s', level=logging.ERROR) else: - raise ValueError(f"unexepcted loglevel: {log_level}") + raise ValueError(f"unexpected loglevel: {log_level}") # set globals @@ -286,7 +286,7 @@ async def store_items(grp_names): timingsData = np.zeros(N) timingsIm = np.zeros(N) -logging.info("creating domain: {fqdn}") +logging.info(f"creating domain: {domain}") with h5py.File(domain, mode="w") as f: start = time.time() g = f.require_group("/test") diff --git a/tests/perf/socket/client.py b/tests/perf/socket/client.py index c16ef2fa..287c6e76 100644 --- a/tests/perf/socket/client.py +++ b/tests/perf/socket/client.py @@ -38,7 +38,6 @@ data = s.recv(BATCH_SIZE) if not data: break - # print(f"got {len(data)} bytes") if use_shared_mem: # read the name of the shm block from socket shm_name = data.decode("ascii") @@ -50,7 +49,7 @@ buffer[n:m] = data total_bytes += len(data) except KeyboardInterrupt: - print("quiting") + print("quitting") break tEnd = time.time() diff --git a/tests/perf/socket/server.py b/tests/perf/socket/server.py index a2b4f0d6..828682fc 100644 --- a/tests/perf/socket/server.py +++ b/tests/perf/socket/server.py @@ -25,17 +25,12 @@ else: shm_block = None -# print("creating rand arr") -# print(time.time()) arr = np.random.rand(nextent) -# print(time.time()) -# print('to buffer') if shm_block: shm_block.buf[:NUM_BYTES] = arr.tobytes()[:] buffer = shm_block.name.encode("ascii") else: buffer = arr.tobytes() -# print(time.time()) tmp_dir = tempfile.TemporaryDirectory() with socket.socket(socket_type, socket.SOCK_STREAM) as s: diff --git a/tests/perf/stream/helper.py b/tests/perf/stream/helper.py index 94ad1429..7effcd2a 100644 --- a/tests/perf/stream/helper.py +++ b/tests/perf/stream/helper.py @@ -98,8 +98,6 @@ def getRequestHeaders(domain=None, username=None, bucket=None, password=None, ** password = config.get("user2_password") headers = dict() if domain is not None: - # if config.get("bucket_name"): - # domain = config.get("bucket_name") + domain headers['X-Hdf-domain'] = domain.encode('utf-8') if username and password: auth_string = username + ':' + password diff --git a/tests/perf/stream/stream_test.py b/tests/perf/stream/stream_test.py index 932dbf3a..87c0f5d4 100644 --- a/tests/perf/stream/stream_test.py +++ b/tests/perf/stream/stream_test.py @@ -118,7 +118,7 @@ def testStream2D(self): if create_dataset: # create dataset - print(f"create datset with shape: [{num_row}, {num_col}]") + print(f"create dataset with shape: [{num_row}, {num_col}]") data = {"type": "H5T_STD_U64LE", "shape": [num_row, num_col]} req = self.endpoint + "/datasets" diff --git a/tests/unit/array_util_test.py b/tests/unit/array_util_test.py deleted file mode 100644 index 1a4f40e5..00000000 --- a/tests/unit/array_util_test.py +++ /dev/null @@ -1,1025 +0,0 @@ -############################################################################## -# Copyright by The HDF Group. # -# All rights reserved. # -# # -# This file is part of HSDS (HDF5 Scalable Data Service), Libraries and # -# Utilities. The full HSDS copyright notice, including # -# terms governing use, modification, and redistribution, is contained in # -# the file COPYING, which can be found at the root of the source code # -# distribution tree. If you do not have access to this file, you may # -# request a copy from help@hdfgroup.org. # -############################################################################## -import unittest -import json -import numpy as np - -import sys -import base64 - -sys.path.append("../..") -from hsds.util.arrayUtil import ( - bytesArrayToList, - toTuple, - getNumElements, - jsonToArray, - arrayToBytes, - bytesToArray, - getByteArraySize, - IndexIterator, - ndarray_compare, - getNumpyValue, - getBroadcastShape -) -from hsds.util.hdf5dtype import special_dtype -from hsds.util.hdf5dtype import check_dtype -from hsds.util.hdf5dtype import createDataType - - -class ArrayUtilTest(unittest.TestCase): - def __init__(self, *args, **kwargs): - super(ArrayUtilTest, self).__init__(*args, **kwargs) - # main - - def testByteArrayToList(self): - data_items = ( - 42, - "foo", - b"foo", - [1, 2, 3], - (1, 2, 3), - ["A", "B", "C"], - [b"A", b"B", b"C"], - [["A", "B"], [b"a", b"b", b"c"]], - ) - for data in data_items: - json_data = bytesArrayToList(data) - # will throw TypeError if not able to convert - json.dumps(json_data) - - def testToTuple(self): - data0d = 42 # scalar - data1d1 = [1] # one dimensional, one element list - data1d = [1, 2, 3, 4, 5] # list - data2d1 = [ - [1, 2], - ] # two dimensional, one element - data2d = [[1, 0.1], [2, 0.2], [3, 0.3], [4, 0.4]] # list of two-element lists - data3d = [[[0, 0.0], [1, 0.1]], [[2, 0.2], [3, 0.3]]] # list of list of lists - out = toTuple(0, data0d) - self.assertEqual(data0d, out) - out = toTuple(1, data1d1) - self.assertEqual(data1d1, out) - out = toTuple(1, data1d) - self.assertEqual(data1d, out) - out = toTuple(2, data2d) - self.assertEqual(data2d, out) - out = toTuple(1, data2d1) - self.assertEqual([(1, 2)], out) - out = toTuple(3, data3d) - self.assertEqual(data3d, out) - out = toTuple(1, data2d) # treat input as 1d array of two-field compound types - self.assertEqual([(1, 0.1), (2, 0.2), (3, 0.3), (4, 0.4)], out) - out = toTuple(2, data3d) # treat input as 2d array of two-field compound types - self.assertEqual([[(0, 0.0), (1, 0.1)], [(2, 0.2), (3, 0.3)]], out) - out = toTuple(1, data3d) # treat input a 1d array of compound type of compound types - self.assertEqual([((0, 0.0), (1, 0.1)), ((2, 0.2), (3, 0.3))], out) - - def testGetNumElements(self): - shape = (4,) - nelements = getNumElements(shape) - self.assertEqual(nelements, 4) - - shape = [10,] - nelements = getNumElements(shape) - self.assertEqual(nelements, 10) - - shape = (10, 8) - nelements = getNumElements(shape) - self.assertEqual(nelements, 80) - - def testJsonToArray(self): - dt = np.dtype("i4") - shape = [4, ] - data = [0, 2, 4, 6] - out = jsonToArray(shape, dt, data) - - self.assertTrue(isinstance(out, np.ndarray)) - self.assertEqual(out.shape, (4,)) - for i in range(4): - self.assertEqual(out[i], i * 2) - - # compound type - dt = np.dtype([("a", "i4"), ("b", "S5")]) - shape = [2, ] - data = [[4, "four"], [5, "five"]] - out = jsonToArray(shape, dt, data) - self.assertTrue(isinstance(out, np.ndarray)) - - self.assertEqual(out.shape, (2,)) - self.assertTrue(isinstance(out[0], np.void)) - e0 = out[0].tolist() - self.assertEqual(e0, (4, b"four")) - self.assertTrue(isinstance(out[1], np.void)) - e1 = out[1].tolist() - self.assertEqual(e1, (5, b"five")) - - shape = [1, ] - data = [ - [6, "six"], - ] - out = jsonToArray(shape, dt, data) - e0 = out[0].tolist() - self.assertEqual(e0, (6, b"six")) - - data = [6, "six"] - out = jsonToArray(shape, dt, data) - e0 = out[0].tolist() - self.assertEqual(e0, (6, b"six")) - - # test ascii chars >127 - dt = np.dtype("S26") - data = "extended ascii char 241: " + chr(241) - out = jsonToArray(shape, dt, data) - self.assertEqual(out[0], b'extended ascii char 241: \xc3') - - dt = np.dtype("S12") - data = "eight: \u516b" - out = jsonToArray(shape, dt, data) - self.assertEqual(out[0], b'eight: \xe5\x85\xab') - - # VLEN ascii - dt = special_dtype(vlen=bytes) - data = [b"one", b"two", b"three", b"four", b"five"] - shape = [5, ] - out = jsonToArray(shape, dt, data) - self.assertTrue("vlen" in out.dtype.metadata) - self.assertEqual(out.dtype.metadata["vlen"], bytes) - self.assertEqual(out.dtype.kind, "O") - self.assertEqual(out.shape, (5,)) - # TBD: code does not actually enforce use of bytes vs. str, - # probably not worth the effort to fix - self.assertEqual(out[2], b"three") - self.assertEqual(out[3], b"four") - - # VLEN str - dt = special_dtype(vlen=str) - data = [ - [b"part 1 - section A", b"part 1 - section B"], - [b"part 2 - section A", b"part 2 - section B"], - ] - shape = [2,] - out = jsonToArray(shape, dt, data) - self.assertTrue("vlen" in out.dtype.metadata) - self.assertEqual(out.dtype.metadata["vlen"], str) - self.assertEqual(out.dtype.kind, "O") - self.assertEqual(out.shape, (2,)) - self.assertEqual(out[0], tuple(data[0])) - self.assertEqual(out[1], tuple(data[1])) - - # VLEN Scalar str - dt = special_dtype(vlen=str) - data = "I'm a string!" - shape = [1, ] - out = jsonToArray(shape, dt, data) - - # VLEN unicode - dt = special_dtype(vlen=bytes) - data = ["one", "two", "three", "four", "five"] - shape = [5, ] - out = jsonToArray(shape, dt, data) - self.assertTrue("vlen" in out.dtype.metadata) - self.assertEqual(out.dtype.metadata["vlen"], bytes) - self.assertEqual(out.dtype.kind, "O") - self.assertEqual(out[2], b"three") - - # VLEN data - dt = special_dtype(vlen=np.dtype("int32")) - shape = [4, ] - data = [ - [1,], - [1, 2], - [1, 2, 3], - [1, 2, 3, 4], - ] - out = jsonToArray(shape, dt, data) - self.assertTrue(isinstance(out, np.ndarray)) - self.assertEqual(check_dtype(vlen=out.dtype), np.dtype("int32")) - - self.assertEqual(out.shape, (4,)) - self.assertEqual(out.dtype.kind, "O") - self.assertEqual(check_dtype(vlen=out.dtype), np.dtype("int32")) - for i in range(4): - e = out[i] # .tolist() - self.assertTrue(isinstance(e, tuple)) - self.assertEqual(e, tuple(range(1, i + 2))) - - # VLEN 2D data - dt = special_dtype(vlen=np.dtype("int32")) - shape = [2, 2] - data = [ - [ - [0,], - [1, 2], - ], - [ - [1,], - [2, 3], - ], - ] - out = jsonToArray(shape, dt, data) - self.assertTrue(isinstance(out, np.ndarray)) - self.assertEqual(check_dtype(vlen=out.dtype), np.dtype("int32")) - - self.assertEqual(out.shape, (2, 2)) - self.assertEqual(out.dtype.kind, "O") - self.assertEqual(check_dtype(vlen=out.dtype), np.dtype("int32")) - for i in range(2): - for j in range(2): - e = out[i, j] # .tolist() - self.assertTrue(isinstance(e, tuple)) - - # create VLEN of obj ref's - ref_type = {"class": "H5T_REFERENCE", "base": "H5T_STD_REF_OBJ"} - vlen_type = {"class": "H5T_VLEN", "base": ref_type} - dt = createDataType(vlen_type) # np datatype - - id0 = b"g-a4f455b2-c8cf-11e7-8b73-0242ac110009" - id1 = b"g-a50af844-c8cf-11e7-8b73-0242ac110009" - id2 = b"g-a5236276-c8cf-11e7-8b73-0242ac110009" - - data = [ - [id0, ], - [id0, id1], - [id0, id1, id2], - ] - shape = [3, ] - out = jsonToArray(shape, dt, data) - self.assertTrue(isinstance(out, np.ndarray)) - base_type = check_dtype(vlen=out.dtype) - self.assertEqual(base_type.kind, "S") - self.assertEqual(base_type.itemsize, 48) - - self.assertEqual(out.shape, (3,)) - self.assertEqual(out.dtype.kind, "O") - self.assertEqual(check_dtype(vlen=out.dtype), np.dtype("S48")) - - e = out[0] - self.assertTrue(isinstance(e, tuple)) - self.assertEqual(e, (id0,)) - e = out[1] - self.assertTrue(isinstance(e, tuple)) - self.assertEqual(e, (id0, id1)) - e = out[2] - self.assertTrue(isinstance(e, tuple)) - self.assertEqual(e, (id0, id1, id2)) - - # compound type with array field - dt = np.dtype([("a", ("i4", 3)), ("b", "S5")]) - shape = [2, ] - data = [[[4, 8, 12], "four"], [[5, 10, 15], "five"]] - out = jsonToArray(shape, dt, data) - self.assertTrue(isinstance(out, np.ndarray)) - - self.assertEqual(out.shape, (2,)) - self.assertTrue(isinstance(out[0], np.void)) - e0 = out[0] - self.assertEqual(len(e0), 2) - e0a = e0[0] - self.assertTrue(isinstance(e0a, np.ndarray)) - self.assertEqual(e0a[0], 4) - self.assertEqual(e0a[1], 8) - self.assertEqual(e0a[2], 12) - e0b = e0[1] - self.assertEqual(e0b, b"four") - self.assertTrue(isinstance(out[1], np.void)) - e1 = out[1] - self.assertEqual(len(e1), 2) - e1a = e1[0] - self.assertTrue(isinstance(e1a, np.ndarray)) - self.assertEqual(e1a[0], 5) - self.assertEqual(e1a[1], 10) - self.assertEqual(e1a[2], 15) - e1b = e1[1] - self.assertEqual(e1b, b"five") - - def testToBytes(self): - # Simple array - dt = np.dtype(" expected_num_bytes) - - # convert buffer back to arr - arr_copy = bytesToArray(buffer, dt, (4,), encoding="base64") - self.assertTrue(np.array_equal(arr, arr_copy)) - - # fixed length string - dt = np.dtype("S8") - arr = np.asarray(("abcdefgh", "ABCDEFGH", "12345678"), dtype=dt) - buffer = arrayToBytes(arr, encoding="base64") - - # convert back to array - arr_copy = bytesToArray(buffer, dt, (3,), encoding="base64") - self.assertTrue(ndarray_compare(arr, arr_copy)) - - # Compound non-vlen - dt = np.dtype([("x", "f8"), ("y", "i4")]) - arr = np.zeros((4,), dtype=dt) - arr[0] = (3.12, 42) - arr[3] = (1.28, 69) - buffer = arrayToBytes(arr, encoding="base64") - - # convert back to array - arr_copy = bytesToArray(buffer, dt, (4,), encoding="base64") - self.assertTrue(ndarray_compare(arr, arr_copy)) - - # VLEN of int32's - dt = np.dtype("O", metadata={"vlen": np.dtype("int32")}) - arr = np.zeros((4,), dtype=dt) - arr[0] = np.int32([1, ]) - arr[1] = np.int32([1, 2]) - arr[2] = 0 # test un-intialized value - arr[3] = np.int32([1, 2, 3]) - buffer = arrayToBytes(arr, encoding="base64") - - # convert back to array - arr_copy = bytesToArray(buffer, dt, (4,), encoding="base64") - self.assertTrue(ndarray_compare(arr, arr_copy)) - - # VLEN of strings - dt = np.dtype("O", metadata={"vlen": str}) - arr = np.zeros((5,), dtype=dt) - arr[0] = "one: \u4e00" - arr[1] = "two: \u4e8c" - arr[2] = "three: \u4e09" - arr[3] = "four: \u56db" - arr[4] = 0 - buffer = arrayToBytes(arr, encoding="base64") - - # convert back to array - arr_copy = bytesToArray(buffer, dt, (5,), encoding="base64") - self.assertTrue(ndarray_compare(arr, arr_copy)) - # VLEN of bytes - dt = np.dtype("O", metadata={"vlen": bytes}) - arr = np.zeros((5,), dtype=dt) - arr[0] = b"Parting" - arr[1] = b"is such" - arr[2] = b"sweet" - arr[3] = b"sorrow" - arr[4] = 0 - - buffer = arrayToBytes(arr, encoding="base64") - - # convert back to array - arr_copy = bytesToArray(buffer, dt, (5,), encoding="base64") - self.assertTrue(ndarray_compare(arr, arr_copy)) - - # - # Compound str vlen - # - dt_vstr = np.dtype("O", metadata={"vlen": str}) - dt = np.dtype([("x", "i4"), ("tag", dt_vstr), ("code", "S4")]) - arr = np.zeros((4,), dtype=dt) - arr[0] = (42, "Hello", "X1") - arr[3] = (84, "Bye", "XYZ") - count = getByteArraySize(arr) - buffer = arrayToBytes(arr, encoding="base64") - - # convert back to array - arr_copy = bytesToArray(buffer, dt, (4,), encoding="base64") - self.assertTrue(ndarray_compare(arr, arr_copy)) - - # - # Compound int vlen - # - dt_vint = np.dtype("O", metadata={"vlen": "int32"}) - dt = np.dtype([("x", "int32"), ("tag", dt_vint)]) - arr = np.zeros((4,), dtype=dt) - arr[0] = (42, np.array((), dtype="int32")) - arr[3] = (84, np.array((1, 2, 3), dtype="int32")) - count = getByteArraySize(arr) - self.assertEqual(count, 44) - buffer = arrayToBytes(arr, encoding="base64") - - # convert back to array - arr_copy = bytesToArray(buffer, dt, (4,), encoding="base64") - self.assertTrue(ndarray_compare(arr, arr_copy)) - - # - # VLEN utf string with array type - # - dt_arr_str = np.dtype("(2,)O", metadata={"vlen": str}) - dt = np.dtype([("x", "i4"), ("tag", dt_arr_str)]) - arr = np.zeros((4,), dtype=dt) - dt_str = np.dtype("O", metadata={"vlen": str}) - arr[0] = (42, np.asarray(["hi", "bye"], dtype=dt_str)) - arr[3] = (84, np.asarray(["hi-hi", "bye-bye"], dtype=dt_str)) - buffer = arrayToBytes(arr, encoding="base64") - - # convert back to array - arr_copy = bytesToArray(buffer, dt, (4,), encoding="base64") - - self.assertEqual(arr.dtype, arr_copy.dtype) - self.assertEqual(arr.shape, arr_copy.shape) - for i in range(4): - e = arr[i] - e_copy = arr_copy[i] - self.assertTrue(np.array_equal(e, e_copy)) - # - # VLEN ascii with array type - # - dt_arr_str = np.dtype("(2,)O", metadata={"vlen": bytes}) - dt = np.dtype([("x", "i4"), ("tag", dt_arr_str)]) - arr = np.zeros((4,), dtype=dt) - dt_str = np.dtype("O", metadata={"vlen": bytes}) - arr[0] = (42, np.asarray([b"hi", b"bye"], dtype=dt_str)) - arr[3] = (84, np.asarray([b"hi-hi", b"bye-bye"], dtype=dt_str)) - buffer = arrayToBytes(arr, encoding="base64") - - # convert back to array - arr_copy = bytesToArray(buffer, dt, (4,), encoding="base64") - self.assertTrue(ndarray_compare(arr, arr_copy)) - - def testArrayCompareInt(self): - # Simple array - dt = np.dtype(" 42') - variables = p.getVariables() - self.assertEqual(len(variables), 2) - self.assertTrue("x1" in variables) - self.assertTrue("y2" in variables) - self.assertTrue(p.evaluate({"x1": "hi", "y2": 43})) - eval_str = p.getEvalStr() - self.assertEqual(eval_str, "(x1 == 'hi') & (y2 > 42.0)") - - p = BooleanParser('x1 == "hi" & y2 > 42') - variables = p.getVariables() - self.assertEqual(len(variables), 2) - self.assertTrue("x1" in variables) - self.assertTrue("y2" in variables) - self.assertTrue(p.evaluate({"x1": "hi", "y2": 43})) - eval_str = p.getEvalStr() - self.assertEqual(eval_str, "(x1 == 'hi') & (y2 > 42.0)") - - p = BooleanParser('x1 == "hi" & y2 > 42') - variables = p.getVariables() - self.assertEqual(len(variables), 2) - self.assertTrue("x1" in variables) - self.assertTrue("y2" in variables) - self.assertTrue(p.evaluate({"x1": "hi", "y2": 43})) - eval_str = p.getEvalStr() - self.assertEqual(eval_str, "(x1 == 'hi') & (y2 > 42.0)") - - # use single instead of double quotes - p = BooleanParser("x1 == 'hi' AND y2 > 42") - variables = p.getVariables() - - self.assertEqual(len(variables), 2) - self.assertTrue("x1" in variables) - self.assertTrue("y2" in variables) - self.assertTrue(p.evaluate({"x1": "hi", "y2": 43})) - eval_str = p.getEvalStr() - self.assertEqual(eval_str, "(x1 == 'hi') & (y2 > 42.0)") - - # string compare - p = BooleanParser("x == 'hi' OR x == 'bye'") - variables = p.getVariables() - self.assertEqual(len(variables), 1) - self.assertTrue("x" in variables) - self.assertTrue(p.evaluate({"x": "bye"})) - self.assertFalse(p.evaluate({"x": "aloha"})) - eval_str = p.getEvalStr() - self.assertEqual(eval_str, "(x == 'hi') | (x == 'bye')") - - # byte string compare - p = BooleanParser("x == 'hi' OR x == b'bye'") - variables = p.getVariables() - self.assertEqual(len(variables), 1) - self.assertTrue("x" in variables) - self.assertTrue(p.evaluate({"x": "bye"})) - self.assertFalse(p.evaluate({"x": "aloha"})) - eval_str = p.getEvalStr() - self.assertEqual(eval_str, "(x == 'hi') | (x == b'bye')") - - # do lexigraphical comparison - p = BooleanParser('x1 >= "cat" AND x1 <= "pig"') - variables = p.getVariables() - self.assertEqual(len(variables), 1) - self.assertTrue("x1" in variables) - self.assertTrue(p.evaluate({"x1": "cat"})) - self.assertFalse(p.evaluate({"x1": "aardvark"})) - self.assertTrue(p.evaluate({"x1": "dog"})) - self.assertTrue(p.evaluate({"x1": "pig"})) - self.assertFalse(p.evaluate({"x1": "piglet"})) - eval_str = p.getEvalStr() - self.assertEqual(eval_str, "(x1 >= 'cat') & (x1 <= 'pig')") - - p = BooleanParser("x > 2 AND y < 3") - self.assertTrue(p.evaluate({"x": 3, "y": 1})) - self.assertFalse(p.evaluate({"x": 1, "y": 1})) - eval_str = p.getEvalStr() - self.assertEqual(eval_str, "(x > 2.0) & (y < 3.0)") - - try: - p.evaluate({"x": "3", "y": 1}) - self.assertTrue(False) # expected exception - except TypeError: - pass # expected - type of x is not int - - try: - p.evaluate({"x": {"a": 1, "b": 2}, "y": 1}) - self.assertTrue(False) # expected exception - dict pased for x value - except TypeError: - pass # expected - type of x is not int - - try: - p.evaluate({"y": 1}) - self.assertTrue(False) # expected exception - except TypeError: - pass # expected - missing 'x' in dict - - try: - BooleanParser("x > 2 AND") - self.assertTrue(False) # expected exception - except IndexError: - pass # expected - malformed exception - - try: - BooleanParser("1 + 1 = 2") - self.assertTrue(False) # expected exception - except Exception: - pass # expected - malformed exception - - -if __name__ == "__main__": - # setup test files - - unittest.main() diff --git a/tests/unit/chunk_util_test.py b/tests/unit/chunk_util_test.py old mode 100755 new mode 100644 index 37d1e512..995674f5 --- a/tests/unit/chunk_util_test.py +++ b/tests/unit/chunk_util_test.py @@ -14,16 +14,15 @@ import sys import numpy as np +from h5json import selections + sys.path.append("../..") -from hsds.util.dsetUtil import getHyperslabSelection from hsds.util.chunkUtil import ( ChunkIterator, chunkReadSelection, chunkWriteSelection, chunkReadPoints, chunkWritePoints, - chunkQuery, - guessChunk, getNumChunks, getChunkIds, getChunkId, @@ -33,14 +32,7 @@ getChunkSelection, getChunkCoverage, getDataCoverage, - getChunkSize, - shrinkChunk, - expandChunk, getDatasetId, - getContiguousLayout, - _getEvalStr, - _getWhereFieldName, - _getWhereElements, ) @@ -50,364 +42,92 @@ def __init__(self, *args, **kwargs): # main logging.getLogger().setLevel(logging.ERROR) - def testGuessChunk(self): - - typesize = "H5T_VARIABLE" - logging.debug("hello") - - shape = {"class": "H5S_NULL"} - layout = guessChunk(shape, typesize) - self.assertTrue(layout is None) - - shape = {"class": "H5S_SCALAR"} - layout = guessChunk(shape, typesize) - self.assertEqual(layout, (1,)) - - shape = {"class": "H5S_SIMPLE", "dims": [100, 100]} - layout = guessChunk(shape, typesize) - self.assertTrue(len(layout), 2) - for i in range(2): - self.assertTrue(layout[i] >= 1) - self.assertTrue(layout[i] <= 100) - - typesize = 8 - layout = guessChunk(shape, typesize) - self.assertTrue(len(layout), 2) - for i in range(2): - self.assertTrue(layout[i] >= 1) - self.assertTrue(layout[i] <= 100) - - shape = {"class": "H5S_SIMPLE", "dims": [5]} - layout = guessChunk(shape, typesize) - self.assertEqual(layout, (5,)) - - shape = {"class": "H5S_SIMPLE", "dims": [100, 100, 100]} - layout = guessChunk(shape, typesize) - print("layout:", layout) - self.assertTrue(len(layout), 3) - for i in range(3): - self.assertTrue(layout[i] >= 1) - self.assertTrue(layout[i] <= 100) - - shape = {"class": "H5S_SIMPLE", "dims": [100, 0], "maxdims": [100, 0]} - layout = guessChunk(shape, typesize) - self.assertTrue(len(layout), 2) - for i in range(2): - self.assertTrue(layout[i] >= 1) - self.assertTrue(layout[i] <= 1024) - - shape = {"class": "H5S_SCALAR"} - layout = guessChunk(shape, typesize) - self.assertEqual(layout, (1,)) - - shape = {"class": "H5S_NULL"} - layout = guessChunk(shape, typesize) - self.assertEqual(layout, None) - - def testShrinkChunk(self): - CHUNK_MIN = 500 - CHUNK_MAX = 5000 - typesize = 1 - layout = (1, 2, 3) - shrunk = shrinkChunk(layout, typesize, chunk_max=CHUNK_MAX) - self.assertEqual(shrunk, layout) - - layout = (100, 200, 300) - num_bytes = getChunkSize(layout, typesize) - self.assertTrue(num_bytes > CHUNK_MAX) - shrunk = shrinkChunk(layout, typesize, chunk_max=CHUNK_MAX) - rank = len(layout) - for i in range(rank): - self.assertTrue(shrunk[i] >= 1) - self.assertTrue(shrunk[i] <= 1000 * (i + 1)) - num_bytes = getChunkSize(shrunk, typesize) - self.assertTrue(num_bytes > CHUNK_MIN) - self.assertTrue(num_bytes < CHUNK_MAX) - - layout = (300, 200, 100) - num_bytes = getChunkSize(layout, typesize) - self.assertTrue(num_bytes > CHUNK_MAX) - shrunk = shrinkChunk(layout, typesize, chunk_max=CHUNK_MAX) - rank = len(layout) - for i in range(rank): - self.assertTrue(shrunk[i] >= 1) - self.assertTrue(shrunk[i] <= 1000 * (3 - i)) - num_bytes = getChunkSize(shrunk, typesize) - self.assertTrue(num_bytes > CHUNK_MIN) - self.assertTrue(num_bytes < CHUNK_MAX) - - CHUNK_MIN = 1 * 1024 * 1024 - CHUNK_MAX = 4 * 1024 * 1024 - typesize = 4 - layout = (117, 201, 189, 1) - num_bytes = getChunkSize(layout, typesize) - self.assertTrue(num_bytes > CHUNK_MAX) - shrunk = shrinkChunk(layout, typesize, chunk_max=CHUNK_MAX) - self.assertEqual(shrunk, (59, 101, 95, 1)) - num_bytes = getChunkSize(shrunk, typesize) - self.assertTrue(num_bytes > CHUNK_MIN) - self.assertTrue(num_bytes < CHUNK_MAX) - - def testExpandChunk(self): - CHUNK_MIN = 5000 - CHUNK_MAX = 50000 - - typesize = 20 - shape = {"class": "H5S_SIMPLE", "dims": [12, ], "maxdims": [20, ]} - layout = (20,) - num_bytes = getChunkSize(layout, typesize) - self.assertTrue(num_bytes < CHUNK_MIN) - expanded = expandChunk(layout, typesize, shape, chunk_min=CHUNK_MIN) - num_bytes = getChunkSize(expanded, typesize) - # chunk layout can't be larger than dataspace - self.assertTrue(num_bytes < CHUNK_MIN) - self.assertEqual(expanded, (20,)) - - typesize = 1 - shape = {"class": "H5S_SIMPLE", "dims": [10, 10, 10]} - layout = (10, 10, 10) - num_bytes = getChunkSize(layout, typesize) - self.assertTrue(num_bytes < CHUNK_MIN) - expanded = expandChunk(layout, typesize, shape, chunk_min=CHUNK_MIN) - num_bytes = getChunkSize(expanded, typesize) - # chunk layout can't be larger than dataspace - self.assertTrue(num_bytes < CHUNK_MIN) - self.assertEqual(expanded, (10, 10, 10)) - - shape = {"class": "H5S_SIMPLE", "dims": [1000, 2000, 3000]} - layout = (10, 10, 10) - num_bytes = getChunkSize(layout, typesize) - self.assertTrue(num_bytes < CHUNK_MIN) - expanded = expandChunk(layout, typesize, shape, chunk_min=CHUNK_MIN) - num_bytes = getChunkSize(expanded, typesize) - self.assertTrue(num_bytes > CHUNK_MIN) - self.assertTrue(num_bytes < CHUNK_MAX) - - shape = {"class": "H5S_SIMPLE", "dims": [1000,]} - layout = (10,) - num_bytes = getChunkSize(layout, "H5T_VARIABLE") - self.assertTrue(num_bytes < CHUNK_MIN) - expanded = expandChunk(layout, "H5T_VARIABLE", shape, chunk_min=CHUNK_MIN) - num_bytes = getChunkSize(expanded, "H5T_VARIABLE") - self.assertTrue(num_bytes > CHUNK_MIN) - self.assertTrue(num_bytes < CHUNK_MAX) - - shape = { - "class": "H5S_SIMPLE", - "dims": [1000, 10, 1000], - "maxdims": [1000, 100, 1000], - } - layout = (10, 10, 10) - num_bytes = getChunkSize(layout, typesize) - self.assertTrue(num_bytes < CHUNK_MIN) - expanded = expandChunk(layout, typesize, shape, chunk_min=CHUNK_MIN) - num_bytes = getChunkSize(expanded, typesize) - self.assertTrue(num_bytes > CHUNK_MIN) - self.assertTrue(num_bytes < CHUNK_MAX) - - shape = { - "class": "H5S_SIMPLE", - "dims": [1000, 0, 1000], - "maxdims": [1000, 100, 1000], - } - layout = (10, 10, 10) - num_bytes = getChunkSize(layout, typesize) - self.assertTrue(num_bytes < CHUNK_MIN) - expanded = expandChunk(layout, typesize, shape, chunk_min=CHUNK_MIN) - num_bytes = getChunkSize(expanded, typesize) - self.assertTrue(num_bytes > CHUNK_MIN) - self.assertTrue(num_bytes < CHUNK_MAX) - - shape = { - "class": "H5S_SIMPLE", - "dims": [1000, 10, 1000], - "maxdims": [1000, 0, 1000], - } - layout = (10, 10, 10) - num_bytes = getChunkSize(layout, typesize) - self.assertTrue(num_bytes < CHUNK_MIN) - expanded = expandChunk(layout, typesize, shape, chunk_min=CHUNK_MIN) - num_bytes = getChunkSize(expanded, typesize) - self.assertTrue(num_bytes > CHUNK_MIN) - self.assertTrue(num_bytes < CHUNK_MAX) - - def testGetContiguiousLayout(self): - - typesize = 4 - chunk_min = 400 - chunk_max = 800 - - def get_num_bytes(dims): - num_bytes = typesize - for n in dims: - num_bytes *= n - return num_bytes - - try: - shape = {"class": "H5S_SIMPLE", "dims": [100, 100]} - layout = getContiguousLayout(shape, "H5T_VARIABLE") - self.assertTrue(False) - except ValueError: - pass # expected - - shape = {"class": "H5S_NULL"} - layout = getContiguousLayout(shape, typesize) - self.assertTrue(layout is None) - - shape = {"class": "H5S_SCALAR"} - layout = getContiguousLayout(shape, typesize) - self.assertEqual(layout, (1,)) - - for extent in (1, 100, 10000): - dims = [ - extent, - ] - shape = {"class": "H5S_SIMPLE", "dims": dims} - layout = getContiguousLayout( - shape, typesize, chunk_min=chunk_min, chunk_max=chunk_max - ) - self.assertTrue(len(layout), 1) - chunk_bytes = get_num_bytes(layout) - space_bytes = get_num_bytes(dims) - if space_bytes > chunk_min: - self.assertTrue(chunk_bytes >= chunk_min) - - self.assertTrue(chunk_bytes <= chunk_max) - - for extent in (1, 9, 90): - dims = [extent, extent] - shape = {"class": "H5S_SIMPLE", "dims": dims} - layout = getContiguousLayout( - shape, typesize, chunk_min=chunk_min, chunk_max=chunk_max - ) - self.assertTrue(len(layout), 2) - for i in range(2): - self.assertTrue(layout[i] >= 1) - self.assertTrue(layout[i] <= extent) - self.assertEqual(layout[1], extent) - chunk_bytes = get_num_bytes(layout) - space_bytes = get_num_bytes(dims) - - if space_bytes > chunk_min: - self.assertTrue(chunk_bytes >= chunk_min) - self.assertTrue(chunk_bytes <= chunk_max) - - for extent in (1, 10, 100): - dims = [extent, extent, 50] - shape = {"class": "H5S_SIMPLE", "dims": dims} - layout = getContiguousLayout( - shape, typesize, chunk_min=chunk_min, chunk_max=chunk_max - ) - self.assertTrue(len(layout), 3) - for i in range(3): - self.assertTrue(layout[i] >= 1) - self.assertTrue(layout[i] <= dims[i]) - - chunk_bytes = get_num_bytes(layout) - space_bytes = get_num_bytes(dims) - - if space_bytes > chunk_min: - self.assertTrue(chunk_bytes >= chunk_min) - self.assertTrue(chunk_bytes <= chunk_max) - - for extent in (1, 100, 1000): - dims = [extent, 4] - shape = {"class": "H5S_SIMPLE", "dims": dims} - layout = getContiguousLayout( - shape, typesize, chunk_min=chunk_min, chunk_max=chunk_max - ) - self.assertTrue(len(layout), 2) - for i in range(2): - self.assertTrue(layout[i] >= 1) - self.assertTrue(layout[i] <= dims[i]) - - chunk_bytes = get_num_bytes(layout) - space_bytes = get_num_bytes(dims) - - if space_bytes > chunk_min: - self.assertTrue(chunk_bytes >= chunk_min) - self.assertTrue(chunk_bytes <= chunk_max) - def testGetNumChunks(self): - datashape = [ - 100, - ] + datashape = (100,) layout = (10,) - selection = getHyperslabSelection(datashape) + selection = selections.select(datashape, ...) count = getNumChunks(selection, layout) self.assertEqual(count, 10) - selection = getHyperslabSelection(datashape, 12, 83) + selection = selections.select(datashape, (slice(12, 83),)) count = getNumChunks(selection, layout) self.assertEqual(count, 8) - selection = getHyperslabSelection(datashape, 12, 80) + selection = selections.select(datashape, (slice(12, 80),)) count = getNumChunks(selection, layout) self.assertEqual(count, 7) - selection = getHyperslabSelection(datashape, 10, 83) + selection = selections.select(datashape, (slice(10, 83),)) count = getNumChunks(selection, layout) self.assertEqual(count, 8) - selection = getHyperslabSelection(datashape, 12, 17) + selection = selections.select(datashape, (slice(12, 17),)) count = getNumChunks(selection, layout) self.assertEqual(count, 1) - selection = [ - (2, 5, 9, 88), - ] # coord list + selection = selections.select(datashape, ([2, 5, 9, 88,],)) # coord list count = getNumChunks(selection, layout) self.assertEqual(count, 2) - selection = [ - (1, 12, 23, 34, 45, 56, 67, 78, 89, 90), - ] # coord list + # coord list + coords = [1, 12, 23, 34, 45, 56, 67, 78, 89, 90] + selection = selections.select(datashape, (coords,)) count = getNumChunks(selection, layout) self.assertEqual(count, 10) # try with different increment - selection = getHyperslabSelection(datashape, 0, 10, 5) + selection = selections.select(datashape, (slice(0, 10, 5),)) count = getNumChunks(selection, layout) self.assertEqual(count, 1) - selection = getHyperslabSelection(datashape, 0, 11, 5) + selection = selections.select(datashape, (slice(0, 11, 5),)) count = getNumChunks(selection, layout) self.assertEqual(count, 2) - selection = getHyperslabSelection(datashape, 6, 11, 5) + selection = selections.select(datashape, (slice(6, 11, 5),),) count = getNumChunks(selection, layout) self.assertEqual(count, 1) - selection = getHyperslabSelection(datashape, 12, 83, 2) + selection = selections.select(datashape, (slice(12, 83, 2),)) count = getNumChunks(selection, layout) self.assertEqual(count, 8) - selection = getHyperslabSelection(datashape, 12, 83, 20) + selection = selections.select(datashape, (slice(12, 83, 20),)) count = getNumChunks(selection, layout) self.assertEqual(count, 4) - selection = getHyperslabSelection(datashape, 10, 83, 20) + selection = selections.select(datashape, (slice(10, 83, 20),)) count = getNumChunks(selection, layout) self.assertEqual(count, 4) - datashape = [100, 100] + datashape = (100, 100) layout = (10, 5) - selection = getHyperslabSelection(datashape) + selection = selections.select(datashape, ...) count = getNumChunks(selection, layout) self.assertEqual(count, 200) - selection = getHyperslabSelection(datashape, (41, 6), (49, 9)) + selection = selections.select(datashape, (slice(41, 49), slice(6, 9),)) count = getNumChunks(selection, layout) self.assertEqual(count, 1) - selection = getHyperslabSelection(datashape, (39, 4), (47, 7)) + selection = selections.select(datashape, (slice(39, 47), slice(4, 7),)) count = getNumChunks(selection, layout) self.assertEqual(count, 4) - selection = ((3, 6, 12, 35), selection[1]) + selection = selections.select(datashape, ((3, 6, 12, 35), slice(4, 7))) count = getNumChunks(selection, layout) self.assertEqual(count, 6) # try with different increment - selection = getHyperslabSelection(datashape, (39, 4), (47, 7), (3, 2)) + selection = selections.select(datashape, (slice(39, 47, 3), slice(4, 7, 2),)) count = getNumChunks(selection, layout) self.assertEqual(count, 4) - selection = getHyperslabSelection(datashape, (0, 0), (100, 100), (20, 40)) + selection = selections.select(datashape, (slice(0, 100, 20), slice(0, 100, 40),)) count = getNumChunks(selection, layout) self.assertEqual(count, 15) + # test with scalar + datashape = () + layout = (1, ) + selection = selections.select(datashape, ...) + count = getNumChunks(selection, layout) + self.assertEqual(count, 1) def testGetChunkIds(self): # getChunkIds(dset_id, selection, layout, dim=0, prefix=None, chunk_ids=None): dset_id = "d-12345678-1234-1234-1234-1234567890ab" - datashape = [1,] + datashape = () layout = (1,) - selection = getHyperslabSelection(datashape) + + selection = selections.select(datashape, ...) + num_chunks = getNumChunks(selection, layout) + + self.assertEqual(num_chunks, 1) chunk_ids = getChunkIds(dset_id, selection, layout) self.assertEqual(len(chunk_ids), 1) chunk_id = chunk_ids[0] @@ -417,9 +137,31 @@ def testGetChunkIds(self): self.assertEqual(len(chunk_id), 2 + 36 + 2) self.assertEqual(getDatasetId(chunk_id), dset_id) - datashape = [100,] + selection = selections.select(datashape, ...) + chunk_ids = getChunkIds(dset_id, selection, layout) + self.assertEqual(len(chunk_ids), 1) + chunk_id = chunk_ids[0] + self.assertTrue(chunk_id.startswith("c-")) + self.assertTrue(chunk_id.endswith("_0")) + self.assertEqual(chunk_id[2:-2], dset_id[2:]) + self.assertEqual(len(chunk_id), 2 + 36 + 2) + self.assertEqual(getDatasetId(chunk_id), dset_id) + + datashape = (1,) + layout = (1,) + selection = selections.select(datashape, ...) + chunk_ids = getChunkIds(dset_id, selection, layout) + self.assertEqual(len(chunk_ids), 1) + chunk_id = chunk_ids[0] + self.assertTrue(chunk_id.startswith("c-")) + self.assertTrue(chunk_id.endswith("_0")) + self.assertEqual(chunk_id[2:-2], dset_id[2:]) + self.assertEqual(len(chunk_id), 2 + 36 + 2) + self.assertEqual(getDatasetId(chunk_id), dset_id) + + datashape = (100,) layout = (10,) - selection = getHyperslabSelection(datashape) + selection = selections.select(datashape, ...) chunk_ids = getChunkIds(dset_id, selection, layout) partition_count = 10 self.assertEqual(len(chunk_ids), 10) @@ -439,7 +181,7 @@ def testGetChunkIds(self): self.assertTrue(partition >= 0) self.assertTrue(partition < partition_count) - selection = getHyperslabSelection(datashape, 20) + selection = selections.select(datashape, (slice(20, 100),)) chunk_ids = getChunkIds(dset_id, selection, layout) self.assertEqual(len(chunk_ids), 8) for i in range(8): @@ -449,7 +191,7 @@ def testGetChunkIds(self): self.assertEqual(chunk_id[2:-2], dset_id[2:]) self.assertEqual(len(chunk_id), 2 + 36 + 2) - selection = getHyperslabSelection(datashape, 20, 81) + selection = selections.select(datashape, (slice(20, 81),)) chunk_ids = getChunkIds(dset_id, selection, layout) self.assertEqual(len(chunk_ids), 7) for i in range(7): @@ -459,7 +201,7 @@ def testGetChunkIds(self): self.assertEqual(chunk_id[2:-2], dset_id[2:]) self.assertEqual(len(chunk_id), 2 + 36 + 2) - selection = getHyperslabSelection(datashape, 29, 81) + selection = selections.select(datashape, (slice(29, 81),)) chunk_ids = getChunkIds(dset_id, selection, layout) self.assertEqual(len(chunk_ids), 7) for i in range(7): @@ -469,7 +211,7 @@ def testGetChunkIds(self): self.assertEqual(chunk_id[2:-2], dset_id[2:]) self.assertEqual(len(chunk_id), 2 + 36 + 2) - selection = getHyperslabSelection(datashape, 29, 81, 2) + selection = selections.select(datashape, (slice(29, 81, 2),)) chunk_ids = getChunkIds(dset_id, selection, layout) self.assertEqual(len(chunk_ids), 6) for i in range(6): @@ -479,7 +221,7 @@ def testGetChunkIds(self): self.assertEqual(chunk_id[2:-2], dset_id[2:]) self.assertEqual(len(chunk_id), 2 + 36 + 2) - selection = getHyperslabSelection(datashape, 29, 81, 20) + selection = selections.select(datashape, (slice(29, 81, 20),)) chunk_ids = getChunkIds(dset_id, selection, layout) self.assertEqual(len(chunk_ids), 3) for i in range(3): @@ -489,16 +231,16 @@ def testGetChunkIds(self): self.assertEqual(chunk_id[2:-2], dset_id[2:]) self.assertEqual(len(chunk_id), 2 + 36 + 2) - datashape = [3207353,] + datashape = (3207353,) layout = (60000,) - selection = getHyperslabSelection(datashape, 1234567, 1234568) + selection = selections.select(datashape, (slice(1234567, 1234568),)) chunk_ids = getChunkIds(dset_id, selection, layout) self.assertEqual(len(chunk_ids), 1) self.assertTrue(chunk_ids[0].endswith("_20")) - datashape = [100, 100] + datashape = (100, 100) layout = (10, 20) - selection = getHyperslabSelection(datashape) + selection = selections.select(datashape, ...) chunk_ids = getChunkIds(dset_id, selection, layout) self.assertEqual(len(chunk_ids), 50) chunk_ids.reverse() # so we can pop off the front @@ -511,7 +253,7 @@ def testGetChunkIds(self): self.assertEqual(index1, i) self.assertEqual(index2, j) - selection = getHyperslabSelection(datashape, (12, 23), (88, 80)) + selection = selections.select(datashape, (slice(12, 88), slice(23, 80))) chunk_ids = getChunkIds(dset_id, selection, layout) self.assertEqual(len(chunk_ids), 24) chunk_ids.reverse() # so we can pop off the front @@ -524,7 +266,7 @@ def testGetChunkIds(self): self.assertEqual(index1, i + 1) self.assertEqual(index2, j + 1) - selection = getHyperslabSelection(datashape, (12, 23), (88, 80), (6, 16)) + selection = selections.select(datashape, (slice(12, 88, 6), slice(23, 80, 16))) chunk_ids = getChunkIds(dset_id, selection, layout) self.assertEqual(len(chunk_ids), 24) chunk_ids.reverse() # so we can pop off the front @@ -537,7 +279,7 @@ def testGetChunkIds(self): self.assertEqual(index1, i + 1) self.assertEqual(index2, j + 1) - selection = getHyperslabSelection(datashape, (12, 23), (88, 80), (16, 44)) + selection = selections.select(datashape, (slice(12, 88, 16), slice(23, 80, 44))) chunk_ids = getChunkIds(dset_id, selection, layout) self.assertEqual(len(chunk_ids), 10) chunk_ids.reverse() # so we can pop off the front @@ -553,9 +295,9 @@ def testGetChunkIds(self): self.assertEqual(index2, yindex[j]) # 3d test - datashape = [365, 720, 1440] + datashape = (365, 720, 1440) layout = (2, 180, 720) - selection = getHyperslabSelection(datashape, (0, 0, 0), (1, 720, 1440)) + selection = selections.select(datashape, (slice(0, 1), slice(0, 720), slice(0, 1440))) chunk_ids = getChunkIds(dset_id, selection, layout) self.assertEqual(len(chunk_ids), 8) chunk_ids.reverse() # so we can pop off the front @@ -570,8 +312,8 @@ def testGetChunkIds(self): self.assertEqual(index1, i) self.assertEqual(index2, j) - selection = getHyperslabSelection( - datashape, (0, 0, 0), (1, 720, 1440), (1, 25, 25) + selection = selections.select( + datashape, (slice(0, 1, 1), slice(0, 720, 25), slice(0, 1440, 25)) ) chunk_ids = getChunkIds(dset_id, selection, layout) self.assertEqual(len(chunk_ids), 8) @@ -588,9 +330,9 @@ def testGetChunkIds(self): self.assertEqual(index2, j) # 2d test - laarge number of chunks - datashape = [7639, 6307] + datashape = (7639, 6307) layout = (1, 6308) - selection = getHyperslabSelection(datashape, (0, 0), (7639, 6307)) + selection = selections.select(datashape, (slice(0, 7639), slice(0, 6307))) chunk_ids = getChunkIds(dset_id, selection, layout) self.assertEqual(len(chunk_ids), 7639) index_set = set() @@ -616,120 +358,118 @@ def testGetChunkIndex(self): def testGetChunkSelection(self): # 1-d test dset_id = "d-12345678-1234-1234-1234-1234567890ab" - datashape = [ - 100, - ] + datashape = (100,) layout = (10,) - selection = getHyperslabSelection(datashape, 42, 62) + selection = selections.select(datashape, (slice(42, 62),)) chunk_ids = getChunkIds(dset_id, selection, layout) self.assertEqual(len(chunk_ids), 3) chunk_id = chunk_ids[0] - sel = getChunkSelection(chunk_id, selection, layout) + sel = getChunkSelection(chunk_id, selection, layout).slices self.assertEqual(sel[0].start, 42) self.assertEqual(sel[0].stop, 50) self.assertEqual(sel[0].step, 1) chunk_id = chunk_ids[1] - sel = getChunkSelection(chunk_id, selection, layout) + sel = getChunkSelection(chunk_id, selection, layout).slices self.assertEqual(sel[0].start, 50) self.assertEqual(sel[0].stop, 60) self.assertEqual(sel[0].step, 1) chunk_id = chunk_ids[2] - sel = getChunkSelection(chunk_id, selection, layout) + sel = getChunkSelection(chunk_id, selection, layout).slices self.assertEqual(sel[0].start, 60) self.assertEqual(sel[0].stop, 62) self.assertEqual(sel[0].step, 1) # 1-d with step - selection = getHyperslabSelection(datashape, 42, 62, 4) + selection = selections.select(datashape, (slice(42, 62, 4),)) chunk_ids = getChunkIds(dset_id, selection, layout) self.assertEqual(len(chunk_ids), 2) chunk_id = chunk_ids[0] - sel = getChunkSelection(chunk_id, selection, layout) + sel = getChunkSelection(chunk_id, selection, layout).slices self.assertEqual(sel[0].start, 42) - self.assertEqual(sel[0].stop, 47) + self.assertEqual(sel[0].stop, 50) self.assertEqual(sel[0].step, 4) chunk_id = chunk_ids[1] - sel = getChunkSelection(chunk_id, selection, layout) + sel = getChunkSelection(chunk_id, selection, layout).slices self.assertEqual(sel[0].start, 50) - self.assertEqual(sel[0].stop, 59) + self.assertEqual(sel[0].stop, 62) self.assertEqual(sel[0].step, 4) # another 1-d with step - selection = getHyperslabSelection(datashape, 40, 63, 2) + selection = selections.select(datashape, (slice(40, 63, 2),)) chunk_ids = getChunkIds(dset_id, selection, layout) self.assertEqual(len(chunk_ids), 3) chunk_id = chunk_ids[0] - sel = getChunkSelection(chunk_id, selection, layout) + sel = getChunkSelection(chunk_id, selection, layout).slices self.assertEqual(sel[0].start, 40) - self.assertEqual(sel[0].stop, 49) + self.assertEqual(sel[0].stop, 50) self.assertEqual(sel[0].step, 2) chunk_id = chunk_ids[1] - sel = getChunkSelection(chunk_id, selection, layout) + sel = getChunkSelection(chunk_id, selection, layout).slices self.assertEqual(sel[0].start, 50) - self.assertEqual(sel[0].stop, 59) + self.assertEqual(sel[0].stop, 60) self.assertEqual(sel[0].step, 2) chunk_id = chunk_ids[2] - sel = getChunkSelection(chunk_id, selection, layout) + sel = getChunkSelection(chunk_id, selection, layout).slices self.assertEqual(sel[0].start, 60) - self.assertEqual(sel[0].stop, 63) + self.assertEqual(sel[0].stop, 64) self.assertEqual(sel[0].step, 2) # test with step > chunk size - selection = getHyperslabSelection(datashape, 0, 100, 15) + selection = selections.select(datashape, (slice(0, 100, 15),)) chunk_ids = getChunkIds(dset_id, selection, layout) self.assertEqual(len(chunk_ids), 7) chunk_id = chunk_ids[0] - sel = getChunkSelection(chunk_id, selection, layout) + sel = getChunkSelection(chunk_id, selection, layout).slices self.assertEqual(sel[0].start, 0) - self.assertEqual(sel[0].stop, 1) + self.assertEqual(sel[0].stop, 15) self.assertEqual(sel[0].step, 15) chunk_id = chunk_ids[1] - sel = getChunkSelection(chunk_id, selection, layout) + sel = getChunkSelection(chunk_id, selection, layout).slices self.assertEqual(sel[0].start, 15) - self.assertEqual(sel[0].stop, 16) + self.assertEqual(sel[0].stop, 30) self.assertEqual(sel[0].step, 15) chunk_id = chunk_ids[2] - sel = getChunkSelection(chunk_id, selection, layout) + sel = getChunkSelection(chunk_id, selection, layout).slices self.assertEqual(sel[0].start, 30) - self.assertEqual(sel[0].stop, 31) + self.assertEqual(sel[0].stop, 45) self.assertEqual(sel[0].step, 15) # test with coordinate - selection = ((12, 13, 33),) + selection = selections.select(datashape, ([12, 13, 33],)) chunk_ids = getChunkIds(dset_id, selection, layout) self.assertEqual(len(chunk_ids), 2) chunk_id = f"c-{dset_id[2:]}_1" self.assertTrue(chunk_id in chunk_ids) - sel = getChunkSelection(chunk_id, selection, layout) + sel = getChunkSelection(chunk_id, selection, layout).slices self.assertEqual(sel[0], [12, 13]) chunk_id = f"c-{dset_id[2:]}_3" self.assertTrue(chunk_id in chunk_ids) - sel = getChunkSelection(chunk_id, selection, layout) + sel = getChunkSelection(chunk_id, selection, layout).slices self.assertEqual(sel[0], [33,],) # 2-d test - datashape = [100, 100] + datashape = (100, 100) layout = (10, 10) - selection = getHyperslabSelection(datashape, (42, 46), (52, 58)) + selection = selections.select(datashape, (slice(42, 52), slice(46, 58))) chunk_ids = getChunkIds(dset_id, selection, layout) self.assertEqual(len(chunk_ids), 4) chunk_id = f"c-{dset_id[2:]}_4_4" self.assertTrue(chunk_id in chunk_ids) - sel = getChunkSelection(chunk_id, selection, layout) + sel = getChunkSelection(chunk_id, selection, layout).slices self.assertEqual(sel[0].start, 42) self.assertEqual(sel[0].stop, 50) self.assertEqual(sel[0].step, 1) @@ -739,7 +479,7 @@ def testGetChunkSelection(self): chunk_id = f"c-{dset_id[2:]}_4_5" self.assertTrue(chunk_id in chunk_ids) - sel = getChunkSelection(chunk_id, selection, layout) + sel = getChunkSelection(chunk_id, selection, layout).slices self.assertEqual(sel[0].start, 42) self.assertEqual(sel[0].stop, 50) self.assertEqual(sel[0].step, 1) @@ -749,7 +489,7 @@ def testGetChunkSelection(self): chunk_id = f"c-{dset_id[2:]}_5_4" self.assertTrue(chunk_id in chunk_ids) - sel = getChunkSelection(chunk_id, selection, layout) + sel = getChunkSelection(chunk_id, selection, layout).slices self.assertEqual(sel[0].start, 50) self.assertEqual(sel[0].stop, 52) self.assertEqual(sel[0].step, 1) @@ -759,7 +499,7 @@ def testGetChunkSelection(self): chunk_id = f"c-{dset_id[2:]}_5_5" self.assertTrue(chunk_id in chunk_ids) - sel = getChunkSelection(chunk_id, selection, layout) + sel = getChunkSelection(chunk_id, selection, layout).slices self.assertEqual(sel[0].start, 50) self.assertEqual(sel[0].stop, 52) self.assertEqual(sel[0].step, 1) @@ -768,65 +508,63 @@ def testGetChunkSelection(self): self.assertEqual(sel[1].step, 1) # test with coordinate - selection = ( - slice(35, 45, 1), - (12, 13, 33), - ) + selection = selections.select(datashape, (slice(35, 45), [12, 13, 33])) + chunk_ids = getChunkIds(dset_id, selection, layout) self.assertEqual(len(chunk_ids), 4) chunk_id = f"c-{dset_id[2:]}_3_1" self.assertTrue(chunk_id in chunk_ids) - sel = getChunkSelection(chunk_id, selection, layout) + sel = getChunkSelection(chunk_id, selection, layout).slices self.assertEqual(sel[0], slice(35, 40, 1)) self.assertEqual(sel[1], [12, 13]) chunk_id = f"c-{dset_id[2:]}_3_3" self.assertTrue(chunk_id in chunk_ids) - sel = getChunkSelection(chunk_id, selection, layout) + sel = getChunkSelection(chunk_id, selection, layout).slices self.assertEqual(sel[0], slice(35, 40, 1)) self.assertEqual(sel[1], [33,]) chunk_id = f"c-{dset_id[2:]}_4_1" self.assertTrue(chunk_id in chunk_ids) - sel = getChunkSelection(chunk_id, selection, layout) + sel = getChunkSelection(chunk_id, selection, layout).slices self.assertEqual(sel[0], slice(40, 45, 1)) self.assertEqual(sel[1], [12, 13]) chunk_id = f"c-{dset_id[2:]}_4_3" self.assertTrue(chunk_id in chunk_ids) - sel = getChunkSelection(chunk_id, selection, layout) + sel = getChunkSelection(chunk_id, selection, layout).slices self.assertEqual(sel[0], slice(40, 45, 1)) self.assertEqual(sel[1], [33,],) # 1-d test with fractional chunks - datashape = [104,] + datashape = (104,) layout = (10,) - selection = getHyperslabSelection(datashape, 92, 102) + selection = selections.select(datashape, (slice(92, 102),)) chunk_ids = getChunkIds(dset_id, selection, layout) chunk_ids.sort() self.assertEqual(len(chunk_ids), 2) chunk_id = chunk_ids[0] - sel = getChunkSelection(chunk_id, selection, layout) + sel = getChunkSelection(chunk_id, selection, layout).slices sel = sel[0] self.assertEqual(sel.start, 100) self.assertEqual(sel.stop, 102) self.assertEqual(sel.step, 1) chunk_id = chunk_ids[1] - sel = getChunkSelection(chunk_id, selection, layout) + sel = getChunkSelection(chunk_id, selection, layout).slices sel = sel[0] self.assertEqual(sel.start, 92) self.assertEqual(sel.stop, 100) self.assertEqual(sel.step, 1) # 3d test - datashape = [365, 720, 1440] + datashape = (365, 720, 1440) layout = (2, 180, 720) - selection = getHyperslabSelection(datashape, (0, 0, 0), (1, 200, 300)) + selection = selections.select(datashape, (slice(0, 1), slice(0, 200), slice(0, 300))) chunk_ids = getChunkIds(dset_id, selection, layout) self.assertEqual(len(chunk_ids), 2) chunk_id = chunk_ids[0] - sel = getChunkSelection(chunk_id, selection, layout) + sel = getChunkSelection(chunk_id, selection, layout).slices self.assertEqual(sel[0].start, 0) self.assertEqual(sel[0].stop, 1) self.assertEqual(sel[0].step, 1) @@ -837,7 +575,7 @@ def testGetChunkSelection(self): self.assertEqual(sel[2].stop, 300) self.assertEqual(sel[2].step, 1) chunk_id = chunk_ids[1] - sel = getChunkSelection(chunk_id, selection, layout) + sel = getChunkSelection(chunk_id, selection, layout).slices self.assertEqual(sel[0].start, 0) self.assertEqual(sel[0].stop, 1) self.assertEqual(sel[0].step, 1) @@ -851,68 +589,68 @@ def testGetChunkSelection(self): def testGetChunkCoverage(self): # 1-d test dset_id = "d-12345678-1234-1234-1234-1234567890ab" - datashape = [100,] + datashape = (100,) layout = (10,) - selection = getHyperslabSelection(datashape, 42, 62) + selection = selections.select(datashape, (slice(42, 62),)) chunk_ids = getChunkIds(dset_id, selection, layout) self.assertEqual(len(chunk_ids), 3) chunk_id = f"c-{dset_id[2:]}_4" self.assertTrue(chunk_id in chunk_ids) - sel = getChunkCoverage(chunk_id, selection, layout) + sel = getChunkCoverage(chunk_id, selection, layout).slices self.assertEqual(sel[0].start, 2) self.assertEqual(sel[0].stop, 10) self.assertEqual(sel[0].step, 1) chunk_id = f"c-{dset_id[2:]}_5" self.assertTrue(chunk_id in chunk_ids) - sel = getChunkCoverage(chunk_id, selection, layout) + sel = getChunkCoverage(chunk_id, selection, layout).slices self.assertEqual(sel[0].start, 0) self.assertEqual(sel[0].stop, 10) self.assertEqual(sel[0].step, 1) chunk_id = f"c-{dset_id[2:]}_6" self.assertTrue(chunk_id in chunk_ids) - sel = getChunkCoverage(chunk_id, selection, layout) + sel = getChunkCoverage(chunk_id, selection, layout).slices self.assertEqual(sel[0].start, 0) self.assertEqual(sel[0].stop, 2) self.assertEqual(sel[0].step, 1) # 1 D with coordinate selection - selection = ((32, 39, 61),) + selection = selections.select(datashape, ([32, 39, 61],)) chunk_ids = getChunkIds(dset_id, selection, layout) self.assertEqual(len(chunk_ids), 2) chunk_id = f"c-{dset_id[2:]}_3" self.assertTrue(chunk_id in chunk_ids) - sel = getChunkCoverage(chunk_id, selection, layout) - self.assertEqual(sel[0], (2, 9)) + sel = getChunkCoverage(chunk_id, selection, layout).slices + self.assertEqual(sel[0], [2, 9]) # 1-d with step - selection = getHyperslabSelection(datashape, 42, 62, 4) + selection = selections.select(datashape, (slice(42, 62, 4),)) chunk_ids = getChunkIds(dset_id, selection, layout) self.assertEqual(len(chunk_ids), 2) chunk_id = chunk_ids[0] - sel = getChunkCoverage(chunk_id, selection, layout) + sel = getChunkCoverage(chunk_id, selection, layout).slices self.assertEqual(sel[0].start, 2) - self.assertEqual(sel[0].stop, 7) + self.assertEqual(sel[0].stop, 10) self.assertEqual(sel[0].step, 4) chunk_id = chunk_ids[1] - sel = getChunkCoverage(chunk_id, selection, layout) + sel = getChunkCoverage(chunk_id, selection, layout).slices self.assertEqual(sel[0].start, 0) - self.assertEqual(sel[0].stop, 9) + self.assertEqual(sel[0].stop, 12) self.assertEqual(sel[0].step, 4) # 2-d test dset_id = "d-12345678-1234-1234-1234-1234567890ab" - datashape = [100, 100] + datashape = (100, 100) layout = (10, 10) - selection = getHyperslabSelection(datashape, (42, 46), (52, 58)) + selection = selections.select(datashape, (slice(42, 52), slice(46, 58))) chunk_ids = getChunkIds(dset_id, selection, layout) self.assertEqual(len(chunk_ids), 4) chunk_id = chunk_ids[0] - sel = getChunkCoverage(chunk_id, selection, layout) + sel = getChunkCoverage(chunk_id, selection, layout).slices self.assertEqual(sel[0].start, 2) self.assertEqual(sel[0].stop, 10) self.assertEqual(sel[0].step, 1) @@ -921,7 +659,7 @@ def testGetChunkCoverage(self): self.assertEqual(sel[1].step, 1) chunk_id = chunk_ids[1] - sel = getChunkCoverage(chunk_id, selection, layout) + sel = getChunkCoverage(chunk_id, selection, layout).slices self.assertEqual(sel[0].start, 2) self.assertEqual(sel[0].stop, 10) self.assertEqual(sel[0].step, 1) @@ -930,7 +668,7 @@ def testGetChunkCoverage(self): self.assertEqual(sel[1].step, 1) chunk_id = chunk_ids[2] - sel = getChunkCoverage(chunk_id, selection, layout) + sel = getChunkCoverage(chunk_id, selection, layout).slices self.assertEqual(sel[0].start, 0) self.assertEqual(sel[0].stop, 2) self.assertEqual(sel[0].step, 1) @@ -939,7 +677,7 @@ def testGetChunkCoverage(self): self.assertEqual(sel[1].step, 1) chunk_id = chunk_ids[3] - sel = getChunkCoverage(chunk_id, selection, layout) + sel = getChunkCoverage(chunk_id, selection, layout).slices self.assertEqual(sel[0].start, 0) self.assertEqual(sel[0].stop, 2) self.assertEqual(sel[0].step, 1) @@ -947,16 +685,16 @@ def testGetChunkCoverage(self): self.assertEqual(sel[1].stop, 8) self.assertEqual(sel[1].step, 1) - # 2-d test - non-even chunks at boundry + # 2-d test - non-even chunks at boundary dset_id = "d-12345678-1234-1234-1234-1234567890ab" - datashape = [45, 54] + datashape = (45, 54) layout = (10, 10) - selection = getHyperslabSelection(datashape, (22, 2), (23, 52)) + selection = selections.select(datashape, (slice(22, 23), slice(2, 52))) chunk_ids = getChunkIds(dset_id, selection, layout) self.assertEqual(len(chunk_ids), 6) chunk_id = chunk_ids[0] - sel = getChunkCoverage(chunk_id, selection, layout) + sel = getChunkCoverage(chunk_id, selection, layout).slices self.assertEqual(sel[0].start, 2) self.assertEqual(sel[0].stop, 3) self.assertEqual(sel[0].step, 1) @@ -967,7 +705,7 @@ def testGetChunkCoverage(self): # the next 4 chunks will have same selection for i in range(1, 4): chunk_id = chunk_ids[i] - sel = getChunkCoverage(chunk_id, selection, layout) + sel = getChunkCoverage(chunk_id, selection, layout).slices self.assertEqual(sel[0].start, 2) self.assertEqual(sel[0].stop, 3) self.assertEqual(sel[0].step, 1) @@ -976,7 +714,7 @@ def testGetChunkCoverage(self): self.assertEqual(sel[1].step, 1) chunk_id = chunk_ids[5] - sel = getChunkCoverage(chunk_id, selection, layout) + sel = getChunkCoverage(chunk_id, selection, layout).slices self.assertEqual(sel[0].start, 2) self.assertEqual(sel[0].stop, 3) self.assertEqual(sel[0].step, 1) @@ -985,61 +723,61 @@ def testGetChunkCoverage(self): self.assertEqual(sel[1].step, 1) # 2-d test wiith coordinates - selection = (slice(15, 25, 1), (62, 69)) + selection = selections.select((45, 70), (slice(15, 25, 1), [62, 69])) chunk_ids = getChunkIds(dset_id, selection, layout) self.assertEqual(len(chunk_ids), 2) chunk_id = chunk_ids[0] - sel = getChunkCoverage(chunk_id, selection, layout) + sel = getChunkCoverage(chunk_id, selection, layout).slices self.assertEqual(sel[0].start, 5) self.assertEqual(sel[0].stop, 10) self.assertEqual(sel[0].step, 1) - self.assertEqual(sel[1], (2, 9)) + self.assertEqual(sel[1], [2, 9]) chunk_id = chunk_ids[1] - sel = getChunkCoverage(chunk_id, selection, layout) + sel = getChunkCoverage(chunk_id, selection, layout).slices self.assertEqual(sel[0].start, 0) self.assertEqual(sel[0].stop, 5) self.assertEqual(sel[0].step, 1) - self.assertEqual(sel[1], (2, 9)) + self.assertEqual(sel[1], [2, 9]) - # 3-d test with coodinates + # 3-d test with coordinates datashape = (5, 1000, 1000) layout = (3, 500, 500) - selection = (slice(0, 5, 1), [1, 10, 100], [10, 100, 500]) + selection = selections.select(datashape, (slice(0, 5, 1), [1, 10, 100], [10, 100, 500])) chunk_ids = getChunkIds(dset_id, selection, layout) chunk_ids.sort() self.assertEqual(len(chunk_ids), 4) chunk_id = chunk_ids[0] - sel = getChunkCoverage(chunk_id, selection, layout) + sel = getChunkCoverage(chunk_id, selection, layout).slices self.assertEqual(sel[0].start, 0) self.assertEqual(sel[0].stop, 3) self.assertEqual(sel[0].step, 1) - self.assertEqual(sel[1], (1, 10)) - self.assertEqual(sel[2], (10, 100)) + self.assertEqual(sel[1], [1, 10]) + self.assertEqual(sel[2], [10, 100]) chunk_id = chunk_ids[1] - sel = getChunkCoverage(chunk_id, selection, layout) + sel = getChunkCoverage(chunk_id, selection, layout).slices self.assertEqual(sel[0].start, 0) self.assertEqual(sel[0].stop, 3) self.assertEqual(sel[0].step, 1) - self.assertEqual(sel[1], (100,)) - self.assertEqual(sel[2], (0,)) + self.assertEqual(sel[1], [100]) + self.assertEqual(sel[2], [0]) # 1-d test with fractional chunks - datashape = [104,] + datashape = (104,) layout = (10,) - selection = getHyperslabSelection(datashape, 92, 102) + selection = selections.select(datashape, (slice(92, 102),)) chunk_ids = getChunkIds(dset_id, selection, layout) chunk_ids.sort() self.assertEqual(len(chunk_ids), 2) chunk_id = chunk_ids[0] - sel = getChunkCoverage(chunk_id, selection, layout) + sel = getChunkCoverage(chunk_id, selection, layout).slices sel = sel[0] self.assertEqual(sel.start, 0) self.assertEqual(sel.stop, 2) self.assertEqual(sel.step, 1) chunk_id = chunk_ids[1] - sel = getChunkCoverage(chunk_id, selection, layout) + sel = getChunkCoverage(chunk_id, selection, layout).slices sel = sel[0] self.assertEqual(sel.start, 2) self.assertEqual(sel.stop, 10) @@ -1048,15 +786,15 @@ def testGetChunkCoverage(self): def testGetDataCoverage(self): # 1-d test dset_id = "d-12345678-1234-1234-1234-1234567890ab" - datashape = [100,] + datashape = (100,) layout = (10,) - selection = getHyperslabSelection(datashape, 42, 62) + selection = selections.select(datashape, (slice(42, 62),)) chunk_ids = getChunkIds(dset_id, selection, layout) self.assertEqual(len(chunk_ids), 3) chunk_id = f"c-{dset_id[2:]}_4" self.assertTrue(chunk_id in chunk_ids) - sel = getDataCoverage(chunk_id, selection, layout) + sel = getDataCoverage(chunk_id, selection, layout).slices self.assertEqual(len(sel), 1) self.assertEqual(sel[0].start, 0) self.assertEqual(sel[0].stop, 8) @@ -1064,7 +802,7 @@ def testGetDataCoverage(self): chunk_id = f"c-{dset_id[2:]}_5" self.assertTrue(chunk_id in chunk_ids) - sel = getDataCoverage(chunk_id, selection, layout) + sel = getDataCoverage(chunk_id, selection, layout).slices self.assertEqual(len(sel), 1) self.assertEqual(sel[0].start, 8) self.assertEqual(sel[0].stop, 18) @@ -1072,21 +810,21 @@ def testGetDataCoverage(self): chunk_id = f"c-{dset_id[2:]}_6" self.assertTrue(chunk_id in chunk_ids) - sel = getDataCoverage(chunk_id, selection, layout) + sel = getDataCoverage(chunk_id, selection, layout).slices self.assertEqual(len(sel), 1) self.assertEqual(sel[0].start, 18) self.assertEqual(sel[0].stop, 20) self.assertEqual(sel[0].step, 1) # test with step - selection = getHyperslabSelection(datashape, 42, 68, 4) + selection = selections.select(datashape, (slice(42, 68, 4),)) self.assertEqual(len(sel), 1) chunk_ids = getChunkIds(dset_id, selection, layout) self.assertEqual(len(chunk_ids), 3) chunk_id = f"c-{dset_id[2:]}_4" self.assertTrue(chunk_id in chunk_ids) - sel = getDataCoverage(chunk_id, selection, layout) + sel = getDataCoverage(chunk_id, selection, layout).slices self.assertEqual(len(sel), 1) self.assertEqual(sel[0].start, 0) self.assertEqual(sel[0].stop, 2) @@ -1094,7 +832,7 @@ def testGetDataCoverage(self): chunk_id = f"c-{dset_id[2:]}_5" self.assertTrue(chunk_id in chunk_ids) - sel = getDataCoverage(chunk_id, selection, layout) + sel = getDataCoverage(chunk_id, selection, layout).slices self.assertEqual(len(sel), 1) self.assertEqual(sel[0].start, 2) self.assertEqual(sel[0].stop, 5) @@ -1102,34 +840,34 @@ def testGetDataCoverage(self): chunk_id = f"c-{dset_id[2:]}_6" self.assertTrue(chunk_id in chunk_ids) - sel = getDataCoverage(chunk_id, selection, layout) + sel = getDataCoverage(chunk_id, selection, layout).slices self.assertEqual(len(sel), 1) self.assertEqual(sel[0].start, 5) self.assertEqual(sel[0].stop, 7) self.assertEqual(sel[0].step, 1) # test with coordinates - selection = ((23, 28),) + selection = selections.select(datashape, ([23, 28],)) chunk_ids = getChunkIds(dset_id, selection, layout) self.assertEqual(len(chunk_ids), 1) chunk_id = f"c-{dset_id[2:]}_2" self.assertTrue(chunk_id in chunk_ids) - sel = getDataCoverage(chunk_id, selection, layout) + sel = getDataCoverage(chunk_id, selection, layout).slices self.assertEqual(len(sel), 1) self.assertEqual(sel[0], [0, 1]) # 2-d test dset_id = "d-12345678-1234-1234-1234-1234567890ab" - datashape = [100, 100] + datashape = (100, 100) layout = (10, 10) - selection = getHyperslabSelection(datashape, (42, 46), (52, 58)) + selection = selections.select(datashape, (slice(42, 52), slice(46, 58))) chunk_ids = getChunkIds(dset_id, selection, layout) self.assertEqual(len(chunk_ids), 4) chunk_id = f"c-{dset_id[2:]}_4_4" self.assertTrue(chunk_id in chunk_ids) - sel = getDataCoverage(chunk_id, selection, layout) + sel = getDataCoverage(chunk_id, selection, layout).slices self.assertEqual(sel[0].start, 0) self.assertEqual(sel[0].stop, 8) self.assertEqual(sel[0].step, 1) @@ -1139,7 +877,7 @@ def testGetDataCoverage(self): chunk_id = f"c-{dset_id[2:]}_4_5" self.assertTrue(chunk_id in chunk_ids) - sel = getDataCoverage(chunk_id, selection, layout) + sel = getDataCoverage(chunk_id, selection, layout).slices self.assertEqual(sel[0].start, 0) self.assertEqual(sel[0].stop, 8) self.assertEqual(sel[0].step, 1) @@ -1149,7 +887,7 @@ def testGetDataCoverage(self): chunk_id = f"c-{dset_id[2:]}_5_4" self.assertTrue(chunk_id in chunk_ids) - sel = getDataCoverage(chunk_id, selection, layout) + sel = getDataCoverage(chunk_id, selection, layout).slices self.assertEqual(sel[0].start, 8) self.assertEqual(sel[0].stop, 10) self.assertEqual(sel[0].step, 1) @@ -1159,7 +897,7 @@ def testGetDataCoverage(self): chunk_id = f"c-{dset_id[2:]}_5_5" self.assertTrue(chunk_id in chunk_ids) - sel = getDataCoverage(chunk_id, selection, layout) + sel = getDataCoverage(chunk_id, selection, layout).slices self.assertEqual(sel[0].start, 8) self.assertEqual(sel[0].stop, 10) self.assertEqual(sel[0].step, 1) @@ -1168,16 +906,13 @@ def testGetDataCoverage(self): self.assertEqual(sel[1].step, 1) # test with coordinates - selection = ( - slice(45, 55, 1), - (23, 28), - ) + selection = selections.select(datashape, (slice(45, 55, 1), [23, 28])) chunk_ids = getChunkIds(dset_id, selection, layout) self.assertEqual(len(chunk_ids), 2) chunk_id = f"c-{dset_id[2:]}_4_2" self.assertTrue(chunk_id in chunk_ids) - sel = getDataCoverage(chunk_id, selection, layout) + sel = getDataCoverage(chunk_id, selection, layout).slices self.assertEqual(sel[0].start, 0) self.assertEqual(sel[0].stop, 5) self.assertEqual(sel[0].step, 1) @@ -1185,44 +920,41 @@ def testGetDataCoverage(self): chunk_id = f"c-{dset_id[2:]}_5_2" self.assertTrue(chunk_id in chunk_ids) - sel = getDataCoverage(chunk_id, selection, layout) + sel = getDataCoverage(chunk_id, selection, layout).slices self.assertEqual(sel[0].start, 5) self.assertEqual(sel[0].stop, 10) self.assertEqual(sel[0].step, 1) self.assertEqual(sel[1], [0, 1]) # test with two coordinates - selection = ( - (1, 5, 55), - (23, 28, 57), - ) + selection = selections.select(datashape, ([1, 5, 55], [23, 28, 57])) chunk_ids = getChunkIds(dset_id, selection, layout) self.assertEqual(len(chunk_ids), 2) chunk_id = f"c-{dset_id[2:]}_5_5" self.assertTrue(chunk_id in chunk_ids) self.assertTrue(chunk_id in chunk_ids) - sel = getDataCoverage(chunk_id, selection, layout) + sel = getDataCoverage(chunk_id, selection, layout).slices self.assertEqual(sel[0], [2,]) chunk_id = f"c-{dset_id[2:]}_0_2" self.assertTrue(chunk_id in chunk_ids) - sel = getDataCoverage(chunk_id, selection, layout) + sel = getDataCoverage(chunk_id, selection, layout).slices self.assertEqual(len(sel), 1) self.assertEqual(sel[0], [0, 1]) # 2-d test, non-regular chunks dset_id = "d-12345678-1234-1234-1234-1234567890ab" - datashape = [45, 54] + datashape = (45, 54) layout = (10, 10) - selection = getHyperslabSelection(datashape, (22, 2), (23, 52)) + selection = selections.select(datashape, (slice(22, 23), slice(2, 52))) chunk_ids = getChunkIds(dset_id, selection, layout) self.assertEqual(len(chunk_ids), 6) chunk_id = f"c-{dset_id[2:]}_2_0" self.assertTrue(chunk_id in chunk_ids) self.assertTrue(chunk_id in chunk_ids) - sel = getDataCoverage(chunk_id, selection, layout) + sel = getDataCoverage(chunk_id, selection, layout).slices self.assertEqual(sel[0].start, 0) self.assertEqual(sel[0].stop, 1) self.assertEqual(sel[0].step, 1) @@ -1233,7 +965,7 @@ def testGetDataCoverage(self): chunk_id = f"c-{dset_id[2:]}_2_1" self.assertTrue(chunk_id in chunk_ids) self.assertTrue(chunk_id in chunk_ids) - sel = getDataCoverage(chunk_id, selection, layout) + sel = getDataCoverage(chunk_id, selection, layout).slices self.assertEqual(sel[0].start, 0) self.assertEqual(sel[0].stop, 1) self.assertEqual(sel[0].step, 1) @@ -1244,7 +976,7 @@ def testGetDataCoverage(self): chunk_id = f"c-{dset_id[2:]}_2_5" self.assertTrue(chunk_id in chunk_ids) self.assertTrue(chunk_id in chunk_ids) - sel = getDataCoverage(chunk_id, selection, layout) + sel = getDataCoverage(chunk_id, selection, layout).slices self.assertEqual(sel[0].start, 0) self.assertEqual(sel[0].stop, 1) self.assertEqual(sel[0].step, 1) @@ -1253,9 +985,9 @@ def testGetDataCoverage(self): self.assertEqual(sel[1].step, 1) # 1-d test with fractional chunks - datashape = [104,] + datashape = (104,) layout = (10,) - selection = getHyperslabSelection(datashape, 92, 102) + selection = selections.select(datashape, (slice(92, 102),)) chunk_ids = getChunkIds(dset_id, selection, layout) self.assertEqual(len(chunk_ids), 2) @@ -1263,7 +995,7 @@ def testGetDataCoverage(self): chunk_id = f"c-{dset_id[2:]}_9" self.assertTrue(chunk_id in chunk_ids) - sel = getDataCoverage(chunk_id, selection, layout) + sel = getDataCoverage(chunk_id, selection, layout).slices sel = sel[0] self.assertEqual(sel.start, 0) self.assertEqual(sel.stop, 8) @@ -1271,31 +1003,35 @@ def testGetDataCoverage(self): chunk_id = f"c-{dset_id[2:]}_10" self.assertTrue(chunk_id in chunk_ids) - sel = getDataCoverage(chunk_id, selection, layout) + sel = getDataCoverage(chunk_id, selection, layout).slices sel = sel[0] self.assertEqual(sel.start, 8) self.assertEqual(sel.stop, 10) self.assertEqual(sel.step, 1) # 3-d test with coord - datashape = [792, 1602, 2976] + datashape = (792, 1602, 2976) layout = (66, 89, 93) - selection = (slice(0, 792, 1), slice(520, 521, 1), slice(1401, 1540, 1)) + selection = selections.select( + datashape, (slice(0, 792, 1), slice(520, 521, 1), slice(1401, 1540, 1)) + ) chunk_ids = getChunkIds(dset_id, selection, layout) chunk_id = f"c-{dset_id[2:]}_0_5_16" self.assertTrue(chunk_id in chunk_ids) self.assertTrue(chunk_id in chunk_ids) - sel = getDataCoverage(chunk_id, selection, layout) + sel = getDataCoverage(chunk_id, selection, layout).slices self.assertEqual(sel[0], slice(0, 66, 1)) self.assertEqual(sel[1], slice(0, 1, 1)) self.assertEqual(sel[2], slice(87, 139, 1)) - selection = (slice(0, 792, 1), slice(520, 521, 1), [1401, 1501, 1540]) + selection = selections.select( + datashape, (slice(0, 792, 1), slice(520, 521, 1), [1401, 1501, 1540]) + ) chunk_ids = getChunkIds(dset_id, selection, layout) self.assertEqual(len(chunk_ids), 24) chunk_id = f"c-{dset_id[2:]}_0_5_16" self.assertTrue(chunk_id in chunk_ids) - sel = getDataCoverage(chunk_id, selection, layout) + sel = getDataCoverage(chunk_id, selection, layout).slices self.assertEqual(sel[0], slice(0, 66, 1)) self.assertEqual(sel[1], slice(0, 1, 1)) self.assertEqual(sel[2], [1, 2]) @@ -1347,10 +1083,10 @@ def testDimQuery(self): def testChunkIterator1d(self): dset_id = "d-12345678-1234-1234-1234-1234567890ab" - dims = [100] + dims = (100,) layout = [10,] - selection = getHyperslabSelection(dims) + selection = selections.select(dims, ...) it = ChunkIterator(dset_id, selection, layout) chunk_ids = set(getChunkIds(dset_id, selection, layout)) count = 0 @@ -1367,9 +1103,9 @@ def testChunkIterator1d(self): def testChunkIterator2d(self): dset_id = "d-12345678-1234-1234-1234-1234567890ab" - dims = [100, 100,] + dims = (100, 100) layout = [50, 50] - selection = getHyperslabSelection(dims) + selection = selections.select(dims, ...) it = ChunkIterator(dset_id, selection, layout) chunk_ids = set(getChunkIds(dset_id, selection, layout)) @@ -1387,9 +1123,9 @@ def testChunkIterator2d(self): def testChunkIterator3d(self): dset_id = "d-12345678-1234-1234-1234-1234567890ab" - dims = [100, 100, 20] + dims = (100, 100, 20) layout = [50, 50, 5] - selection = getHyperslabSelection(dims) + selection = selections.select(dims, ...) it = ChunkIterator(dset_id, selection, layout) chunk_ids = set(getChunkIds(dset_id, selection, layout)) @@ -1405,94 +1141,193 @@ def testChunkIterator3d(self): self.assertEqual(count, 16) - def testGetEvalStr(self): - queries = {} - queries["date == 23"] = "rows['date'] == 23" - queries["tgt123 == 456"] = "rows['tgt123'] == 456" - queries["wind == b'W 5'"] = "rows['wind'] == b'W 5'" - queries["temp > 61"] = "rows['temp'] > 61" - queries["(date >= 22) & (date <= 24)"] = "(rows['date'] >= 22) & (rows['date'] <= 24)" - queries["(date == 21) & (temp > 70)"] = "(rows['date'] == 21) & (rows['temp'] > 70)" - expected = "(rows['wind'] == b'E 7') | (rows['wind'] == b'S 7')" - queries["(wind == b'E 7') | (wind == b'S 7')"] = expected - queries["where 'temp' in (61, 68, 72)"] = None - queries["where 'name' in (b'Bob', b'Rob', b'Alice')"] = None - queries["date >= 22 where 'temp' in (61, 68, 72)"] = "rows['date'] >= 22" - queries["date >= 22 where 'temp F' in (61, 68, 72)"] = "rows['date'] >= 22" - - fields = ["date", "wind", "temp", "tgt123"] - - for query in queries.keys(): - eval_str = _getEvalStr(query, "rows", fields) - self.assertEqual(eval_str, queries[query]) - - def testGetWhereFieldName(self): - queries = {} - queries["date == 23"] = None - queries["where 'temp' in (61, 68, 72)"] = "temp" - queries["date >= 22 where 'temp' in (61, 68, 72)"] = "temp" - queries["date >= 22 where 'temp F' in (61, 68, 72)"] = "temp F" - queries["date >= 22 where 'temp F123' in (61, 68, 72)"] = "temp F123" - - for query in queries.keys(): - field = _getWhereFieldName(query) - self.assertEqual(field, queries[query]) - - def testGetWhereElements(self): - queries = {} - queries["where 'temp' in (61, 68, 72)"] = ["61", "68", "72"] - queries["where 'temp' in (abc, xyz, abacab)"] = ["abc", "xyz", "abacab"] - queries["where 'temp' in ('ab cd', 'xyz ', 'abacab')"] = ["ab cd", "xyz ", "abacab"] - queries["where 'temp' in (123, -456, 3.12)"] = ["123", "-456", "3.12"] - queries["where 'temp' in (b'abc', b'xyz')"] = [b'abc', b'xyz'] - - for query in queries.keys(): - elements = _getWhereElements(query) - self.assertEqual(elements, queries[query]) - - def testBadQuery(self): - queries = ( - "foobar", # no variable used - "wind = b'abc", # non-closed literal - "(wind = b'N') & (temp = 32", # missing paren - "foobar > 42", # invalid field name - "import subprocess; subprocess.call(['ls', '/'])", - ) # injection attack - - fields = ("date", "wind", "temp") - - for query in queries: - try: - _getEvalStr(query, "x", fields) - self.assertTrue(False) # shouldn't get here - except Exception: - pass # ok - def testChunkReadSelection(self): chunk_arr = np.array([2, 3, 5, 7, 11, 13, 17, 19]) - arr = chunkReadSelection(chunk_arr, slices=((slice(3, 5, 1),))) + selection = selections.select(chunk_arr.shape, (slice(3, 5, 1),)) + arr = chunkReadSelection(chunk_arr, selection=selection) self.assertEqual(arr.tolist(), [7, 11]) - arr = chunkReadSelection(chunk_arr, slices=((slice(3, 9, 2),))) + selection = selections.select(chunk_arr.shape, (slice(3, 9, 2),)) + arr = chunkReadSelection(chunk_arr, selection=selection) self.assertEqual(arr.tolist(), [7, 13, 19]) chunk_arr = np.zeros((3, 4)) for i in range(3): chunk_arr[i] = list(range(i + 1, i + 1 + 4)) - arr = chunkReadSelection(chunk_arr, slices=((slice(1, 2, 1), slice(0, 4, 1)))) + selection = selections.select(chunk_arr.shape, (slice(1, 2, 1), slice(0, 4, 1))) + arr = chunkReadSelection(chunk_arr, selection=selection) self.assertEqual(arr.tolist(), [[2.0, 3.0, 4.0, 5.0]]) - arr = chunkReadSelection(chunk_arr, slices=((slice(0, 3, 1), slice(2, 3, 1)))) + selection = selections.select(chunk_arr.shape, (slice(0, 3, 1), slice(2, 3, 1))) + arr = chunkReadSelection(chunk_arr, selection=selection) self.assertEqual(arr.tolist(), [[3.0], [4.0], [5.0]]) - arr = chunkReadSelection(chunk_arr, slices=((slice(0, 1, 1), [0, 3]))) + selection = selections.select(chunk_arr.shape, (slice(0, 1, 1), [0, 3])) + arr = chunkReadSelection(chunk_arr, selection=selection) self.assertEqual(arr.tolist(), [[1.0, 4.0]]) def testChunkWriteSelection(self): chunk_arr = np.zeros((8,)) data = np.array([2, 3, 5, 7, 11, 13, 17, 19]) - chunkWriteSelection(chunk_arr=chunk_arr, slices=(slice(0, 8, 1),), data=data) + selection = selections.select(chunk_arr.shape, (slice(0, 8, 1),)) + chunkWriteSelection(chunk_arr=chunk_arr, selection=selection, data=data) self.assertEqual(chunk_arr.tolist(), data.tolist()) data = np.array([101, 121, 131]) - chunkWriteSelection(chunk_arr=chunk_arr, slices=(slice(3, 6, 1),), data=data) + selection = selections.select(chunk_arr.shape, (slice(3, 6, 1),)) + chunkWriteSelection(chunk_arr=chunk_arr, selection=selection, data=data) self.assertEqual(chunk_arr.tolist(), [2, 3, 5, 101, 121, 131, 17, 19]) + def testChunkWriteSelectionArrayDtype(self): + # for an array/subarray dtype (H5T_ARRAY, e.g. numpy's "3i1") the + # chunk array's own shape absorbs the dtype's subarray dims + # (chunk_arr.shape == dataset_shape + dt.shape) while the + # selection is built against the dataset's logical shape only - + # chunkWriteSelection()'s rank check allows chunk_arr's/data's + # rank to exceed selection's by exactly those absorbed dims, + # rather than requiring an exact rank match. + dt = np.dtype(("i1", (3,))) + dataset_shape = (2,) + chunk_arr = np.zeros(dataset_shape, dtype=dt) + self.assertEqual(chunk_arr.shape, (2, 3)) # subarray dims absorbed + + data = np.frombuffer(bytes([1, 2, 3, 4, 5, 6]), dtype=dt) + self.assertEqual(data.shape, (2, 3)) + + # selection is built against the dataset's own (1-D) logical shape, + # not chunk_arr's subarray-absorbed shape + selection = selections.select(dataset_shape, (slice(0, 2, 1),)) + chunkWriteSelection(chunk_arr=chunk_arr, selection=selection, data=data) + self.assertEqual(chunk_arr.tolist(), data.tolist()) + + # full (non-field-restricted) read back of the same bare + # array-dtype chunk - chunkReadSelection() has the same rank + # relaxation on its own selection-rank check + arr = chunkReadSelection(chunk_arr, selection=selection) + self.assertEqual(arr.tolist(), data.tolist()) + + def testChunkWriteSelectionCompoundArrayField(self): + # compound dtype with an array-typed field - unlike a bare + # array/subarray dtype (see testChunkWriteSelectionArrayDtype + # above), a compound dtype's own shape is NOT absorbed by its + # array-typed field's dims, so chunk_arr.shape stays equal to the + # plain dataset shape and chunkWriteSelection()/chunkReadSelection() + # work correctly here - including for a partial (hyperslab) + # selection, which nothing else exercises for this dtype shape. + dt = np.dtype([("temp", (" true coordinate stop of start + count * step self.assertEqual(s.stop, 200) self.assertEqual(s.step, 8) - select = [ - (slice(0, 195, 4)), - ] # 156 byte selection - # should create 4 pages + select = selections.select(datashape, (slice(0, 195, 4),)) # 196 byte selection + # should create 3 pages pages = getSelectionPagination(select, datashape, itemsize, max_request_size) - self.assertEqual(len(pages), 2) - start = 0 + self.assertEqual(len(pages), 3) + total_points = 0 for page in pages: - self.assertEqual(len(page), 1) - s = page[0] + self.assertEqual(len(page.shape), 1) + s = page.slices[0] self.assertTrue(isinstance(s, slice)) - self.assertEqual(s.start, start) self.assertEqual( s.start % 4, 0 ) # start value always falls in step intervals self.assertEqual(s.step, 4) - self.assertTrue(s.stop > s.start + 4) - count = (s.stop - s.start) // 4 - self.assertTrue(count * itemsize < max_request_size) - start = s.stop + count = (s.stop - s.start) // s.step # s.stop is a true coordinate stop + self.assertTrue(count * itemsize <= max_request_size) + total_points += count + self.assertEqual(total_points, 49) # covers all 49 selected points coords = [] for i in range(50): coords.append(i * 4) - select = [ - coords, - ] # 160 byte coordinate selection + select = selections.select(datashape, (coords,)) # 200 byte coordinate selection pages = getSelectionPagination(select, datashape, itemsize, max_request_size) - print("pages:", pages) self.assertEqual(len(pages), 2) + total_coords = 0 for page in pages: - self.assertEqual(len(page), 1) - s = page[0] - self.assertTrue(isinstance(s, tuple)) + self.assertEqual(len(page.shape), 1) + s = page.slices[0] + self.assertTrue(isinstance(s, list)) count = len(s) - print("count:", count) - self.assertTrue(len(s) > 20) + self.assertTrue(count > 20) self.assertTrue(count * itemsize <= max_request_size) + total_coords += count + self.assertEqual(total_coords, 50) # 2D case - datashape = [200, 300] + datashape = (200, 300) max_request_size = 1000 - select = [(slice(0, 10)), (slice(0, 20))] # 800 byte selection + select = selections.select(datashape, (slice(0, 10), slice(0, 20))) # 800 byte selection # should return one page equivalent to original selection pages = getSelectionPagination(select, datashape, itemsize, max_request_size) self.assertEqual(len(pages), 1) page = pages[0] - self.assertEqual(len(page), 2) + self.assertEqual(len(page.shape), 2) for i in range(2): - self.assertEqual(page[i].start, select[i].start) - self.assertEqual(page[i].stop, select[i].stop) + self.assertEqual(page.slices[i].start, select.slices[i].start) + self.assertEqual(page.slices[i].stop, select.slices[i].stop) - select = [(slice(20, 60)), (slice(0, 20))] # 3200 byte selection - # should return one page equivalent to original selection + select = selections.select(datashape, (slice(20, 60), slice(0, 20))) # 3200 byte selection pages = getSelectionPagination(select, datashape, itemsize, max_request_size) self.assertEqual(len(pages), 4) start = 20 for page in pages: - self.assertEqual(len(page), 2) - self.assertEqual(page[0].start, start) + self.assertEqual(len(page.shape), 2) + self.assertEqual(page.slices[0].start, start) # second dimension shouldn't change - self.assertEqual(page[1].start, select[1].start) - self.assertEqual(page[1].stop, select[1].stop) - start = page[0].stop - self.assertEqual(start, select[0].stop) + self.assertEqual(page.slices[1].start, select.slices[1].start) + self.assertEqual(page.slices[1].stop, select.slices[1].stop) + start = page.slices[0].stop + self.assertEqual(start, select.slices[0].stop) - select = [(40,), (slice(0, 300))] # 1200 byte selection + select = selections.select(datashape, ([40], slice(0, 300))) # 1200 byte selection pages = getSelectionPagination(select, datashape, itemsize, max_request_size) self.assertEqual(len(pages), 2) start = 0 @@ -239,99 +168,54 @@ def testGetSelectionPagination(self): # pagination should happen along the second dimension, # since there's only one coordinate in the first for page in pages: - self.assertEqual(len(page), 2) - self.assertEqual(page[1].start, start) - # second dimension shouldn't change - self.assertEqual(page[0], (40,)) - start = page[1].stop - self.assertEqual(start, select[1].stop) + self.assertEqual(len(page.shape), 2) + self.assertEqual(page.slices[1].start, start) + # first dimension shouldn't change + self.assertEqual(page.slices[0], [40]) + start = page.slices[1].stop + self.assertEqual(start, select.slices[1].stop) itemsize = 2 datashape = (1300, 1300, 1300) max_request_size = 100 * 1024 * 1024 - select = [ - (slice(200, 400)), - (slice(0, 1300)), - (slice(0, 1300)), - ] # 644 MB selection + select = selections.select( + datashape, (slice(200, 400), slice(0, 1300), slice(0, 1300)) + ) # 644 MB selection pages = getSelectionPagination(select, datashape, itemsize, max_request_size) self.assertEqual(len(pages), 8) start = 200 for page in pages: - self.assertEqual(len(page), 3) - self.assertEqual(page[0].start, start) - self.assertEqual(page[1], slice(0, 1300)) - self.assertEqual(page[2], slice(0, 1300)) - page_size = (page[0].stop - page[0].start) * 1300 * 1300 * 2 + self.assertEqual(len(page.shape), 3) + self.assertEqual(page.slices[0].start, start) + self.assertEqual(page.slices[1], slice(0, 1300, 1)) + self.assertEqual(page.slices[2], slice(0, 1300, 1)) + page_size = (page.slices[0].stop - page.slices[0].start) * 1300 * 1300 * 2 self.assertTrue(page_size < max_request_size) - start = page[0].stop + start = page.slices[0].stop - select = [ - (slice(0, 1300)), - (slice(0, 1300)), - (slice(0, 1300)), - ] # 4.1GB selection + select = selections.select( + datashape, (slice(0, 1300), slice(0, 1300), slice(0, 1300)) + ) # 4.1GB selection pages = getSelectionPagination(select, datashape, itemsize, max_request_size) self.assertEqual(len(pages), 44) start = 0 for page in pages: - # print(page) - self.assertEqual(len(page), 3) - self.assertEqual(page[0].start, start) - self.assertEqual(page[1], slice(0, 1300)) - self.assertEqual(page[2], slice(0, 1300)) - page_size = (page[0].stop - page[0].start) * 1300 * 1300 * 2 + self.assertEqual(len(page.shape), 3) + self.assertEqual(page.slices[0].start, start) + self.assertEqual(page.slices[1], slice(0, 1300, 1)) + self.assertEqual(page.slices[2], slice(0, 1300, 1)) + page_size = (page.slices[0].stop - page.slices[0].start) * 1300 * 1300 * 2 self.assertTrue(page_size < max_request_size) - start = page[0].stop - - def testItemIterator(self): - # 1-D case - datashape = [10,] - slices = getHyperslabSelection(datashape) - it = ItemIterator(slices) - - indices = [] - count = 0 - - while True: - try: - index = it.next() - count += 1 - indices.append(index) - except StopIteration: - break - self.assertEqual(count, 10) - self.assertEqual(indices, list(range(10))) - - # 2-D case - datashape = [4, 5] - slices = getHyperslabSelection(datashape) - it = ItemIterator(slices) - - indices = [] - count = 0 - while True: - try: - index = it.next() - self.assertTrue(len(index), 2) - self.assertTrue(index[0] >= 0) - self.assertTrue(index[0] < 4) - self.assertTrue(index[1] >= 0) - self.assertTrue(index[1] < 5) - count += 1 - indices.append(index) - except StopIteration: - break - self.assertEqual(count, 20) + start = page.slices[0].stop def testSelectionList1D(self): dims = [100,] for select in ("", []): selection = getSelectionList(select, dims) - self.assertEqual(len(selection), 1) - s1 = selection[0] + self.assertEqual(len(selection.shape), 1) + s1 = selection.slices[0] self.assertTrue(isinstance(s1, slice)) self.assertEqual(s1, slice(0, 100, 1)) @@ -340,8 +224,8 @@ def testSelectionList1D(self): [5,], ): selection = getSelectionList(select, dims) - self.assertEqual(len(selection), 1) - s1 = selection[0] + self.assertEqual(len(selection.shape), 1) + s1 = selection.slices[0] self.assertTrue(isinstance(s1, slice)) self.assertEqual(s1, slice(5, 6, 1)) @@ -350,8 +234,8 @@ def testSelectionList1D(self): [":",], ): selection = getSelectionList(select, dims) - self.assertEqual(len(selection), 1) - s1 = selection[0] + self.assertEqual(len(selection.shape), 1) + s1 = selection.slices[0] self.assertTrue(isinstance(s1, slice)) self.assertEqual(s1, slice(0, 100, 1)) @@ -360,8 +244,8 @@ def testSelectionList1D(self): ["3:7",], ): selection = getSelectionList(select, dims) - self.assertEqual(len(selection), 1) - s1 = selection[0] + self.assertEqual(len(selection.shape), 1) + s1 = selection.slices[0] self.assertTrue(isinstance(s1, slice)) self.assertEqual(s1, slice(3, 7, 1)) @@ -370,8 +254,8 @@ def testSelectionList1D(self): [":4",], ): selection = getSelectionList(select, dims) - self.assertEqual(len(selection), 1) - s1 = selection[0] + self.assertEqual(len(selection.shape), 1) + s1 = selection.slices[0] self.assertTrue(isinstance(s1, slice)) self.assertEqual(s1, slice(0, 4, 1)) @@ -380,15 +264,15 @@ def testSelectionList1D(self): ["0:100",], ): selection = getSelectionList(select, dims) - self.assertEqual(len(selection), 1) - s1 = selection[0] + self.assertEqual(len(selection.shape), 1) + s1 = selection.slices[0] self.assertTrue(isinstance(s1, slice)) self.assertEqual(s1, slice(0, 100, 1)) for select in ("[[3,4,7]]", ["[3,4,7]"], [[3, 4, 7]]): selection = getSelectionList(select, dims) - self.assertEqual(len(selection), 1) - s1 = selection[0] + self.assertEqual(len(selection.shape), 1) + s1 = selection.slices[0] self.assertTrue(isinstance(s1, list)) self.assertEqual(s1, [3, 4, 7]) @@ -397,22 +281,22 @@ def testSelectionList1D(self): ["30:70:5",], ): selection = getSelectionList(select, dims) - self.assertEqual(len(selection), 1) - s1 = selection[0] + self.assertEqual(len(selection.shape), 1) + s1 = selection.slices[0] self.assertTrue(isinstance(s1, slice)) self.assertEqual(s1, slice(30, 70, 5)) body = {"start": 3, "stop": 7} selection = getSelectionList(body, dims) - self.assertEqual(len(selection), 1) - s1 = selection[0] + self.assertEqual(len(selection.shape), 1) + s1 = selection.slices[0] self.assertTrue(isinstance(s1, slice)) self.assertEqual(s1, slice(3, 7, 1)) body = {"start": 30, "stop": 70, "step": 5} selection = getSelectionList(body, dims) - self.assertEqual(len(selection), 1) - s1 = selection[0] + self.assertEqual(len(selection.shape), 1) + s1 = selection.slices[0] self.assertTrue(isinstance(s1, slice)) self.assertEqual(s1, slice(30, 70, 5)) @@ -421,101 +305,101 @@ def testSelectionList2D(self): for select in ("", []): selection = getSelectionList(select, dims) - self.assertEqual(len(selection), 2) - s1 = selection[0] + self.assertEqual(len(selection.shape), 2) + s1 = selection.slices[0] self.assertTrue(isinstance(s1, slice)) self.assertEqual(s1, slice(0, 50, 1)) - s2 = selection[1] + s2 = selection.slices[1] self.assertTrue(isinstance(s2, slice)) self.assertEqual(s2, slice(0, 100, 1)) for select in ("[5,40]", ["5", "40"], [5, 40]): selection = getSelectionList(select, dims) - self.assertEqual(len(selection), 2) - s1 = selection[0] + self.assertEqual(len(selection.shape), 2) + s1 = selection.slices[0] self.assertTrue(isinstance(s1, slice)) self.assertEqual(s1, slice(5, 6, 1)) - s2 = selection[1] + s2 = selection.slices[1] self.assertTrue(isinstance(s2, slice)) self.assertEqual(s2, slice(40, 41, 1)) for select in ("[3:7,12]", ["3:7", "12"], ["3:7", 12]): selection = getSelectionList(select, dims) - self.assertEqual(len(selection), 2) - s1 = selection[0] + self.assertEqual(len(selection.shape), 2) + s1 = selection.slices[0] self.assertTrue(isinstance(s1, slice)) self.assertEqual(s1, slice(3, 7, 1)) - s2 = selection[1] + s2 = selection.slices[1] self.assertTrue(isinstance(s2, slice)) self.assertEqual(s2, slice(12, 13, 1)) for select in ("[:,[3,4,7]]", [":", "[3,4,7]"], [":", [3, 4, 7]]): selection = getSelectionList(select, dims) - self.assertEqual(len(selection), 2) - s1 = selection[0] + self.assertEqual(len(selection.shape), 2) + s1 = selection.slices[0] self.assertTrue(isinstance(s1, slice)) self.assertEqual(s1, slice(0, dims[0], 1)) - s2 = selection[1] + s2 = selection.slices[1] self.assertTrue(isinstance(s2, list)) self.assertEqual(s2, [3, 4, 7]) for select in ("[[2, 5, 8],[3,4,7]]", ["[2, 5, 8]", "[3,4,7]"], [[2, 5, 8], [3, 4, 7]]): selection = getSelectionList(select, dims) - self.assertEqual(len(selection), 2) - s1 = selection[0] + self.assertEqual(len(selection.shape), 2) + s1 = selection.slices[0] self.assertTrue(isinstance(s1, list)) self.assertEqual(s1, [2, 5, 8]) - s2 = selection[1] + s2 = selection.slices[1] self.assertTrue(isinstance(s2, list)) self.assertEqual(s2, [3, 4, 7]) for select in ("[[2,5,8],[7,4,3]]", ["[2, 5, 8]", "[7,4,3]"], [[2, 5, 8], [7, 4, 3]]): selection = getSelectionList(select, dims) - self.assertEqual(len(selection), 2) - s1 = selection[0] + self.assertEqual(len(selection.shape), 2) + s1 = selection.slices[0] self.assertTrue(isinstance(s1, list)) self.assertEqual(s1, [2, 5, 8]) - s2 = selection[1] + s2 = selection.slices[1] self.assertTrue(isinstance(s2, list)) self.assertEqual(s2, [7, 4, 3]) for select in ("[1:20, 30:70:5]", ["1:20", "30:70:5"]): selection = getSelectionList(select, dims) - self.assertEqual(len(selection), 2) - s1 = selection[0] + self.assertEqual(len(selection.shape), 2) + s1 = selection.slices[0] self.assertTrue(isinstance(s1, slice)) self.assertEqual(s1, slice(1, 20, 1)) - s2 = selection[1] + s2 = selection.slices[1] self.assertTrue(isinstance(s2, slice)) self.assertEqual(s2, slice(30, 70, 5)) for select in ("[0:50, 0:100]", ["0:50", "0:100"]): selection = getSelectionList(select, dims) - self.assertEqual(len(selection), 2) - s1 = selection[0] + self.assertEqual(len(selection.shape), 2) + s1 = selection.slices[0] self.assertTrue(isinstance(s1, slice)) self.assertEqual(s1, slice(0, 50, 1)) - s2 = selection[1] + s2 = selection.slices[1] self.assertTrue(isinstance(s2, slice)) self.assertEqual(s2, slice(0, 100, 1)) body = {"start": [3, 5], "stop": [7, 9]} selection = getSelectionList(body, dims) - self.assertEqual(len(selection), 2) - s1 = selection[0] + self.assertEqual(len(selection.shape), 2) + s1 = selection.slices[0] self.assertTrue(isinstance(s1, slice)) self.assertEqual(s1, slice(3, 7, 1)) - s2 = selection[1] + s2 = selection.slices[1] self.assertTrue(isinstance(s2, slice)) self.assertEqual(s2, slice(5, 9, 1)) body = {"start": [0, 30], "stop": [10, 70], "step": [1, 5]} selection = getSelectionList(body, dims) - self.assertEqual(len(selection), 2) - s1 = selection[0] + self.assertEqual(len(selection.shape), 2) + s1 = selection.slices[0] self.assertTrue(isinstance(s1, slice)) self.assertEqual(s1, slice(0, 10, 1)) - s2 = selection[1] + s2 = selection.slices[1] self.assertTrue(isinstance(s2, slice)) self.assertEqual(s2, slice(30, 70, 5)) @@ -605,6 +489,124 @@ def testInvalidSelectionList(self): except ValueError: pass # expected + def testParseRegionRefParam(self): + group_id = createObjId("groups") + dset_id = createObjId("datasets", root_id=group_id) + + collection, obj_id, attr_name = parseRegionRefParam( + f"/groups/{group_id}/attributes/foo" + ) + self.assertEqual(collection, "groups") + self.assertEqual(obj_id, group_id) + self.assertEqual(attr_name, "foo") + + collection, obj_id, attr_name = parseRegionRefParam( + f"/datasets/{dset_id}/attributes/bar" + ) + self.assertEqual(collection, "datasets") + self.assertEqual(obj_id, dset_id) + self.assertEqual(attr_name, "bar") + + collection, obj_id, attr_name = parseRegionRefParam(f"/datasets/{dset_id}") + self.assertEqual(collection, "datasets") + self.assertEqual(obj_id, dset_id) + self.assertIsNone(attr_name) + + invalid_paths = ( + f"groups/{group_id}/attributes/foo", # no leading slash + f"/datasets/{dset_id}/attributes/", # empty attr name + f"/groups/{dset_id}", # bare group form not supported + f"/datasets/{group_id}", # wrong id type for collection + "/datasets/not-a-valid-id", + f"/foo/{dset_id}", + "", + ) + for path in invalid_paths: + try: + parseRegionRefParam(path) + self.fail(f"expected HTTPBadRequest for regionref path: {path}") + except HTTPBadRequest: + pass # expected + + def testExtractJsonArrayElement(self): + dt = special_dtype(ref=RegionReference) + root_id = createObjId("groups") + dset_id = createObjId("datasets", root_id=root_id) + + pts_sel = selections.select((3, 16), ([0, 2], [1, 11])) + ref_pts = RegionReference(dset_id, pts_sel) + hs_sel = selections.select((3, 16), (slice(0, 2), slice(0, 4))) + ref_hs = RegionReference(dset_id, hs_sel) + value = [ref_pts.to_json(), ref_hs.to_json(), None] + + sel0 = selections.select((3,), (0,)) + elem0 = extractJsonArrayElement((3,), dt, value, sel0) + self.assertEqual(elem0["select_type"], "H5S_SEL_POINTS") + + sel1 = selections.select((3,), (1,)) + elem1 = extractJsonArrayElement((3,), dt, value, sel1) + self.assertEqual(elem1["select_type"], "H5S_SEL_HYPERSLABS") + + sel2 = selections.select((3,), (2,)) + elem2 = extractJsonArrayElement((3,), dt, value, sel2) + self.assertIsNone(elem2) + + # 2-D source, coordinate-list select picking a single element + value2d = [[ref_pts.to_json(), ref_hs.to_json()], [None, ref_hs.to_json()]] + sel2d = selections.select((2, 2), ([1], [0])) + elem2d = extractJsonArrayElement((2, 2), dt, value2d, sel2d) + self.assertIsNone(elem2d) + + def testUnwrapSingleElement(self): + self.assertEqual(unwrapSingleElement(42), 42) + self.assertIsNone(unwrapSingleElement(None)) + self.assertEqual(unwrapSingleElement([{"id": "x"}]), {"id": "x"}) + self.assertEqual(unwrapSingleElement([[None]]), None) + try: + unwrapSingleElement([1, 2]) + self.fail("expected ValueError for multi-element list") + except ValueError: + pass # expected + + def testRegionRefSelectionToTargetSelection(self): + root_id = createObjId("groups") + dset_id = createObjId("datasets", root_id=root_id) + + hs_sel = selections.select((3, 16), (slice(0, 2), slice(0, 4))) + ref_hs = RegionReference(dset_id, hs_sel) + ref_json = ref_hs.to_json() + + # matching rank, in bounds + target = regionRefSelectionToTargetSelection(ref_json, (3, 16)) + self.assertEqual(target.shape, (3, 16)) + self.assertEqual(target.start, (0, 0)) + self.assertEqual(target.count, (2, 4)) + + # rank mismatch + try: + regionRefSelectionToTargetSelection(ref_json, (3, 16, 2)) + self.fail("expected HTTPBadRequest for rank mismatch") + except HTTPBadRequest: + pass # expected + + # out of bounds (target smaller than the ref's selection extent) + try: + regionRefSelectionToTargetSelection(ref_json, (3, 3)) + self.fail("expected HTTPBadRequest for out-of-bounds selection") + except HTTPBadRequest: + pass # expected + + # bare {"id": ...} with no selection info -> whole target selected + bare_ref_json = {"id": getUuidFromId(dset_id)} + whole = regionRefSelectionToTargetSelection(bare_ref_json, (5, 7)) + self.assertEqual(whole.nselect, 35) + + # points selection also round-trips correctly + pts_sel = selections.select((3, 16), ([0, 2], [1, 11])) + ref_pts = RegionReference(dset_id, pts_sel) + pts_target = regionRefSelectionToTargetSelection(ref_pts.to_json(), (3, 16)) + self.assertEqual(pts_target.nselect, 2) + if __name__ == "__main__": # setup test files diff --git a/tests/unit/glob_parser_test.py b/tests/unit/glob_parser_test.py old mode 100755 new mode 100644 diff --git a/tests/unit/hdf5_dtype_test.py b/tests/unit/hdf5_dtype_test.py deleted file mode 100755 index e51913a6..00000000 --- a/tests/unit/hdf5_dtype_test.py +++ /dev/null @@ -1,717 +0,0 @@ -############################################################################## -# Copyright by The HDF Group. # -# All rights reserved. # -# # -# This file is part of HSDS (HDF5 Scalable Data Service), Libraries and # -# Utilities. The full HSDS copyright notice, including # -# terms governing use, modification, and redistribution, is contained in # -# the file COPYING, which can be found at the root of the source code # -# distribution tree. If you do not have access to this file, you may # -# request a copy from help@hdfgroup.org. # -############################################################################## -import unittest -import logging -import numpy as np -import sys - -sys.path.append("../..") -from hsds.util import hdf5dtype -from hsds.util.hdf5dtype import special_dtype -from hsds.util.hdf5dtype import check_dtype -from hsds.util.hdf5dtype import Reference -from hsds.util.hdf5dtype import RegionReference - - -class Hdf5dtypeTest(unittest.TestCase): - def __init__(self, *args, **kwargs): - super(Hdf5dtypeTest, self).__init__(*args, **kwargs) - # main - self.logger = logging.getLogger() - self.logger.setLevel(logging.INFO) - - def testGetBaseTypeJson(self): - type_json = hdf5dtype.getBaseTypeJson("H5T_IEEE_F64LE") - self.assertTrue("class" in type_json) - self.assertEqual(type_json["class"], "H5T_FLOAT") - self.assertTrue("base" in type_json) - self.assertEqual(type_json["base"], "H5T_IEEE_F64LE") - - type_json = hdf5dtype.getBaseTypeJson("H5T_IEEE_F16LE") - self.assertTrue("class" in type_json) - self.assertEqual(type_json["class"], "H5T_FLOAT") - self.assertTrue("base" in type_json) - self.assertEqual(type_json["base"], "H5T_IEEE_F16LE") - - type_json = hdf5dtype.getBaseTypeJson("H5T_STD_I32LE") - self.assertTrue("class" in type_json) - self.assertEqual(type_json["class"], "H5T_INTEGER") - self.assertTrue("base" in type_json) - self.assertEqual(type_json["base"], "H5T_STD_I32LE") - - try: - hdf5dtype.getBaseTypeJson("foobar") - self.assertTrue(False) - except TypeError: - pass # expected - - def testBaseIntegerTypeItem(self): - dt = np.dtype("") - self.assertEqual(dt.kind, "u") - - dt = hdf5dtype.createDataType("H5T_STD_I16LE") - self.assertEqual(dt.name, "int16") - self.assertEqual(dt.kind, "i") - - dt = hdf5dtype.createDataType("H5T_IEEE_F64LE") - self.assertEqual(dt.name, "float64") - self.assertEqual(dt.kind, "f") - - dt = hdf5dtype.createDataType("H5T_IEEE_F32LE") - self.assertEqual(dt.name, "float32") - self.assertEqual(dt.kind, "f") - - typeItem = {"class": "H5T_INTEGER", "base": "H5T_STD_I32BE"} - typeSize = hdf5dtype.getItemSize(typeItem) - dt = hdf5dtype.createDataType(typeItem) - self.assertEqual(dt.name, "int32") - self.assertEqual(dt.kind, "i") - self.assertEqual(typeSize, 4) - - def testCreateBaseStringType(self): - typeItem = {"class": "H5T_STRING", "charSet": "H5T_CSET_ASCII", "length": 6} - typeSize = hdf5dtype.getItemSize(typeItem) - dt = hdf5dtype.createDataType(typeItem) - self.assertEqual(dt.name, "bytes48") - self.assertEqual(dt.kind, "S") - self.assertEqual(typeSize, 6) - - def testCreateBaseUnicodeType(self): - typeItem = {"class": "H5T_STRING", "charSet": "H5T_CSET_UTF8", "length": 6} - - dt = hdf5dtype.createDataType(typeItem) - typeSize = hdf5dtype.getItemSize(typeItem) - self.assertTrue(dt is not None) - self.assertEqual(dt.name, "bytes48") - self.assertEqual(dt.kind, "S") # uses byte - self.assertEqual(typeSize, 6) - - def testCreateNullTermStringType(self): - typeItem = { - "class": "H5T_STRING", - "charSet": "H5T_CSET_ASCII", - "length": 6, - "strPad": "H5T_STR_NULLTERM", - } - typeSize = hdf5dtype.getItemSize(typeItem) - dt = hdf5dtype.createDataType(typeItem) - - self.assertEqual(dt.name, "bytes48") - self.assertEqual(dt.kind, "S") - self.assertEqual(typeSize, 6) - - def testCreateVLenStringType(self): - typeItem = { - "class": "H5T_STRING", - "charSet": "H5T_CSET_ASCII", - "length": "H5T_VARIABLE", - } - typeSize = hdf5dtype.getItemSize(typeItem) - dt = hdf5dtype.createDataType(typeItem) - self.assertEqual(dt.name, "object") - self.assertEqual(dt.kind, "O") - self.assertEqual(check_dtype(vlen=dt), bytes) - self.assertEqual(typeSize, "H5T_VARIABLE") - - def testCreateVLenUTF8Type(self): - typeItem = { - "class": "H5T_STRING", - "charSet": "H5T_CSET_UTF8", - "length": "H5T_VARIABLE", - } - typeSize = hdf5dtype.getItemSize(typeItem) - dt = hdf5dtype.createDataType(typeItem) - self.assertEqual(dt.name, "object") - self.assertEqual(dt.kind, "O") - self.assertEqual(check_dtype(vlen=dt), str) - self.assertEqual(typeSize, "H5T_VARIABLE") - - def testCreateVLenDataType(self): - typeItem = {"class": "H5T_VLEN", "base": "H5T_STD_I32BE"} - typeSize = hdf5dtype.getItemSize(typeItem) - self.assertEqual(typeSize, "H5T_VARIABLE") - dt = hdf5dtype.createDataType(typeItem) - self.assertEqual(dt.name, "object") - self.assertEqual(dt.kind, "O") - - def testCreateOpaqueType(self): - typeItem = {"class": "H5T_OPAQUE", "size": 200} - typeSize = hdf5dtype.getItemSize(typeItem) - dt = hdf5dtype.createDataType(typeItem) - self.assertEqual(dt.name, "void1600") - self.assertEqual(dt.kind, "V") - self.assertEqual(typeSize, 200) - - def testCreateEnumType(self): - typeItem = { - "class": "H5T_ENUM", - "base": {"base": "H5T_STD_I16LE", "class": "H5T_INTEGER"}, - "mapping": {"GAS": 2, "LIQUID": 1, "PLASMA": 3, "SOLID": 0}, - } - - typeSize = hdf5dtype.getItemSize(typeItem) - self.assertEqual(typeSize, 2) - dt = hdf5dtype.createDataType(typeItem) - self.assertEqual(dt.name, "int16") - self.assertEqual(dt.kind, "i") - mapping = check_dtype(enum=dt) - self.assertTrue(isinstance(mapping, dict)) - self.assertEqual(mapping["SOLID"], 0) - self.assertEqual(mapping["LIQUID"], 1) - self.assertEqual(mapping["GAS"], 2) - self.assertEqual(mapping["PLASMA"], 3) - - def testCreateBoolType(self): - typeItem = { - "class": "H5T_ENUM", - "base": {"base": "H5T_STD_I8LE", "class": "H5T_INTEGER"}, - "mapping": {"TRUE": 1, "FALSE": 0}, - } - - typeSize = hdf5dtype.getItemSize(typeItem) - self.assertEqual(typeSize, 1) - dt = hdf5dtype.createDataType(typeItem) - self.assertEqual(dt.name, "bool") - self.assertEqual(dt.kind, "b") - self.assertEqual(typeSize, hdf5dtype.getDtypeItemSize(dt)) - - def testCreateCompoundType(self): - typeItem = { - "class": "H5T_COMPOUND", - "fields": [ - {"name": "temp", "type": "H5T_IEEE_F32LE"}, - {"name": "pressure", "type": "H5T_IEEE_F32LE"}, - { - "name": "location", - "type": { - "length": "H5T_VARIABLE", - "charSet": "H5T_CSET_ASCII", - "class": "H5T_STRING", - "strPad": "H5T_STR_NULLTERM", - }, - }, - {"name": "wind", "type": "H5T_STD_I16LE"}, - ], - } - typeSize = hdf5dtype.getItemSize(typeItem) - dt = hdf5dtype.createDataType(typeItem) - self.assertEqual(dt.name, "void144") - self.assertEqual(dt.kind, "V") - self.assertEqual(len(dt.fields), 4) - self.assertEqual(typeSize, hdf5dtype.getDtypeItemSize(dt)) - - dtLocation = dt[2] - self.assertEqual(dtLocation.name, "object") - self.assertEqual(dtLocation.kind, "O") - self.assertEqual(check_dtype(vlen=dtLocation), bytes) - self.assertEqual(typeSize, "H5T_VARIABLE") - self.assertEqual(typeSize, hdf5dtype.getDtypeItemSize(dtLocation)) - - def testCreateCompoundInvalidFieldName(self): - typeItem = { - "class": "H5T_COMPOUND", - "fields": [ - { - "name": "\u03b1", - "type": {"base": "H5T_STD_I32LE", "class": "H5T_INTEGER"}, - }, - { - "name": "\u03c9", - "type": {"base": "H5T_STD_I32LE", "class": "H5T_INTEGER"}, - }, - ], - } - try: - hdf5dtype.createDataType(typeItem) - self.assertTrue(False) - except TypeError: - pass # expected - - def testCreateCompoundOfCompoundType(self): - typeItem = { - "class": "H5T_COMPOUND", - "fields": [ - { - "name": "field1", - "type": { - "class": "H5T_COMPOUND", - "fields": [ - { - "name": "x", - "type": { - "class": "H5T_FLOAT", - "base": "H5T_IEEE_F32LE", - }, - }, - { - "name": "y", - "type": { - "class": "H5T_FLOAT", - "base": "H5T_IEEE_F32LE", - }, - }, - ], - }, - }, - { - "name": "field2", - "type": { - "class": "H5T_COMPOUND", - "fields": [ - { - "name": "a", - "type": { - "class": "H5T_FLOAT", - "base": "H5T_IEEE_F32LE", - }, - }, - { - "name": "b", - "type": { - "class": "H5T_FLOAT", - "base": "H5T_IEEE_F32LE", - }, - }, - { - "name": "c", - "type": { - "class": "H5T_FLOAT", - "base": "H5T_IEEE_F32LE", - }, - }, - ], - }, - }, - ], - } - dt = hdf5dtype.createDataType(typeItem) - self.assertEqual(dt.name, "void160") - self.assertEqual(dt.kind, "V") - self.assertEqual(len(dt.fields), 2) - dt_field1 = dt[0] - self.assertEqual(dt_field1.name, "void64") - self.assertEqual(dt_field1.kind, "V") - self.assertEqual(len(dt_field1.fields), 2) - dt_field2 = dt[1] - self.assertEqual(dt_field2.name, "void96") - self.assertEqual(dt_field2.kind, "V") - self.assertEqual(len(dt_field2.fields), 3) - - def testCreateCompoundTypeUnicodeFields(self): - typeItem = { - "class": "H5T_COMPOUND", - "fields": [ - {"name": u"temp", "type": "H5T_IEEE_F32LE"}, - {"name": u"pressure", "type": "H5T_IEEE_F32LE"}, - {"name": u"wind", "type": "H5T_STD_I16LE"}, - ], - } - typeSize = hdf5dtype.getItemSize(typeItem) - dt = hdf5dtype.createDataType(typeItem) - self.assertEqual(dt.name, "void80") - self.assertEqual(dt.kind, "V") - self.assertEqual(len(dt.fields), 3) - self.assertEqual(typeSize, 10) - self.assertEqual(typeSize, hdf5dtype.getDtypeItemSize(dt)) - - def testCreateArrayType(self): - typeItem = {"class": "H5T_ARRAY", "base": "H5T_STD_I64LE", "dims": (3, 5)} - typeSize = hdf5dtype.getItemSize(typeItem) - dt = hdf5dtype.createDataType(typeItem) - self.assertEqual(dt.name, "void960") - self.assertEqual(dt.kind, "V") - self.assertEqual(typeSize, 120) - self.assertEqual(typeSize, hdf5dtype.getDtypeItemSize(dt)) - - def testCreateArrayIntegerType(self): - typeItem = {"class": "H5T_INTEGER", "base": "H5T_STD_I64LE", "dims": (3, 5)} - - try: - hdf5dtype.createDataType(typeItem) - self.assertTrue(False) # expected exception - dims used with non-array type - except TypeError: - pass # should get exception - - def testCreateCompoundArrayType(self): - typeItem = { - "class": "H5T_COMPOUND", - "fields": [ - {"type": {"base": "H5T_STD_I8LE", "class": "H5T_INTEGER"}, "name": "a"}, - { - "type": { - "dims": [10], - "base": { - "length": 1, - "charSet": "H5T_CSET_ASCII", - "class": "H5T_STRING", - "strPad": "H5T_STR_NULLPAD", - }, - "class": "H5T_ARRAY", - }, - "name": "b", - }, - ], - } - typeSize = hdf5dtype.getItemSize(typeItem) - dt = hdf5dtype.createDataType(typeItem) - self.assertEqual(len(dt.fields), 2) - self.assertTrue("a" in dt.fields.keys()) - self.assertTrue("b" in dt.fields.keys()) - self.assertEqual(typeSize, 11) - self.assertEqual(typeSize, hdf5dtype.getDtypeItemSize(dt)) - - def testCompoundArrayType(self): - typeItem = { - "class": "H5T_COMPOUND", - "fields": [ - { - "type": {"class": "H5T_INTEGER", "base": "H5T_STD_U64BE"}, - "name": "VALUE1", - }, - { - "type": {"class": "H5T_FLOAT", "base": "H5T_IEEE_F64BE"}, - "name": "VALUE2", - }, - { - "type": { - "class": "H5T_ARRAY", - "dims": [2], - "base": { - "class": "H5T_STRING", - "charSet": "H5T_CSET_ASCII", - "strPad": "H5T_STR_NULLTERM", - "length": "H5T_VARIABLE", - }, - }, - "name": "VALUE3", - }, - ], - } - dt = hdf5dtype.createDataType(typeItem) - typeSize = hdf5dtype.getItemSize(typeItem) - self.assertEqual(typeSize, "H5T_VARIABLE") - self.assertEqual(len(dt), 3) - self.assertTrue("VALUE1" in dt.fields.keys()) - self.assertTrue("VALUE2" in dt.fields.keys()) - self.assertTrue("VALUE3" in dt.fields.keys()) - self.assertEqual(typeSize, hdf5dtype.getDtypeItemSize(dt)) - - dt3 = dt["VALUE3"] - self.assertEqual(check_dtype(vlen=dt3), bytes) - - -if __name__ == "__main__": - # setup test files - - unittest.main() diff --git a/tests/unit/id_util_test.py b/tests/unit/id_util_test.py deleted file mode 100755 index 06f974c4..00000000 --- a/tests/unit/id_util_test.py +++ /dev/null @@ -1,212 +0,0 @@ -############################################################################## -# Copyright by The HDF Group. # -# All rights reserved. # -# # -# This file is part of HSDS (HDF5 Scalable Data Service), Libraries and # -# Utilities. The full HSDS copyright notice, including # -# terms governing use, modification, and redistribution, is contained in # -# the file COPYING, which can be found at the root of the source code # -# distribution tree. If you do not have access to this file, you may # -# request a copy from help@hdfgroup.org. # -############################################################################## -import unittest -import sys - -sys.path.append("../..") -from hsds.util.idUtil import getObjPartition, isValidUuid, validateUuid -from hsds.util.idUtil import createObjId, getCollectionForId -from hsds.util.idUtil import isObjId, isS3ObjKey, getS3Key, getObjId, isSchema2Id -from hsds.util.idUtil import isRootObjId, getRootObjId - - -class IdUtilTest(unittest.TestCase): - def __init__(self, *args, **kwargs): - super(IdUtilTest, self).__init__(*args, **kwargs) - # main - - def testCreateObjId(self): - id_len = 38 # 36 for uuid plus two for prefix ("g-", "d-") - ids = set() - for obj_class in ("groups", "datasets", "datatypes", "chunks"): - for i in range(100): - id = createObjId(obj_class) - self.assertEqual(len(id), id_len) - self.assertTrue(id[0] in ("g", "d", "t", "c")) - self.assertEqual(id[1], "-") - ids.add(id) - - self.assertEqual(len(ids), 400) - try: - createObjId("bad_class") - self.assertTrue(False) # should throw exception - except ValueError: - pass # expected - - def testIsValidUuid(self): - group1_id = "g-314d61b8-9954-11e6-a733-3c15c2da029e" # orig schema - group2_id = "g-314d61b8-995411e6-a733-3c15c2-da029e" - root_id = "g-f9aaa28e-d42e10e5-7122-2a065c-a6986d" - dataset1_id = "d-4c48f3ae-9954-11e6-a3cd-3c15c2da029e" # orig schema - dataset2_id = "d-4c48f3ae-995411e6-a3cd-3c15c2-da029e" - ctype1_id = "t-8c785f1c-9953-11e6-9bc2-0242ac110005" # orig schema - ctype2_id = "t-8c785f1c-995311e6-9bc2-0242ac-110005" - chunk1_id = "c-8c785f1c-9953-11e6-9bc2-0242ac110005_7_2" # orig schema - chunk2_id = "c-8c785f1c-995311e6-9bc2-0242ac-110005_7_2" - domain_id = "mybucket/bob/mydata.h5" - s3_domain_id = "s3://mybucket/bob/mydata.h5" - file_domain_id = "file://mybucket/bob/mydata.h5" - azure_domain_id = "https://myaccount.blob.core.windows.net/mybucket/bob/mydata.h5" - valid_id_map = { - group1_id: "a49be-g-314d61b8-9954-11e6-a733-3c15c2da029e", - group2_id: "db/314d61b8-995411e6/g/a733-3c15c2-da029e/.group.json", - dataset1_id: "26928-d-4c48f3ae-9954-11e6-a3cd-3c15c2da029e", - dataset2_id: "db/4c48f3ae-995411e6/d/a3cd-3c15c2-da029e/.dataset.json", - ctype1_id: "5a9cf-t-8c785f1c-9953-11e6-9bc2-0242ac110005", - ctype2_id: "db/8c785f1c-995311e6/t/9bc2-0242ac-110005/.datatype.json", - chunk1_id: "dc4ce-c-8c785f1c-9953-11e6-9bc2-0242ac110005_7_2", - chunk2_id: "db/8c785f1c-995311e6/d/9bc2-0242ac-110005/7_2", - domain_id: "bob/mydata.h5/.domain.json", - s3_domain_id: "bob/mydata.h5/.domain.json", - file_domain_id: "bob/mydata.h5/.domain.json", - azure_domain_id: "bob/mydata.h5/.domain.json", } - - bad_ids = ("g-1e76d862", "/bob/mydata.h5") - - self.assertTrue(isValidUuid(group1_id)) - self.assertFalse(isSchema2Id(group1_id)) - self.assertTrue(isValidUuid(group1_id, obj_class="Group")) - self.assertTrue(isValidUuid(group1_id, obj_class="group")) - self.assertTrue(isValidUuid(group1_id, obj_class="groups")) - self.assertTrue(isSchema2Id(root_id)) - self.assertTrue(isValidUuid(root_id, obj_class="Group")) - self.assertTrue(isValidUuid(root_id, obj_class="group")) - self.assertTrue(isValidUuid(root_id, obj_class="groups")) - self.assertTrue(isRootObjId(root_id)) - self.assertTrue(isValidUuid(dataset1_id, obj_class="datasets")) - self.assertFalse(isSchema2Id(dataset1_id)) - self.assertTrue(isValidUuid(ctype1_id, obj_class="datatypes")) - self.assertFalse(isSchema2Id(ctype1_id)) - self.assertTrue(isValidUuid(chunk1_id, obj_class="chunks")) - self.assertFalse(isSchema2Id(chunk1_id)) - self.assertTrue(isValidUuid(group2_id)) - self.assertTrue(isSchema2Id(group2_id)) - self.assertTrue(isValidUuid(group2_id, obj_class="Group")) - self.assertTrue(isValidUuid(group2_id, obj_class="group")) - self.assertTrue(isValidUuid(group2_id, obj_class="groups")) - self.assertFalse(isRootObjId(group2_id)) - self.assertTrue(isValidUuid(dataset2_id, obj_class="datasets")) - self.assertTrue(isSchema2Id(dataset2_id)) - self.assertTrue(isValidUuid(ctype2_id, obj_class="datatypes")) - self.assertTrue(isSchema2Id(ctype2_id)) - self.assertTrue(isValidUuid(chunk2_id, obj_class="chunks")) - self.assertTrue(isSchema2Id(chunk2_id)) - validateUuid(group1_id) - try: - isRootObjId(group1_id) - self.assertTrue(False) - except ValueError: - # only works for v2 schema - pass # expected - - for item in valid_id_map: - self.assertTrue(isObjId(item)) - s3key = getS3Key(item) - self.assertTrue(s3key[0] != "/") - self.assertTrue(isS3ObjKey(s3key)) - expected = valid_id_map[item] - self.assertEqual(s3key, expected) - if item.find("/") > 0: - continue # bucket name gets lost when domain ids get converted to s3keys - objid = getObjId(s3key) - self.assertEqual(objid, item) - for item in bad_ids: - self.assertFalse(isValidUuid(item)) - self.assertFalse(isObjId(item)) - - def testGetObjPartition(self): - node_count = 12 - for obj_class in ("groups", "datasets", "datatypes", "chunks"): - for i in range(100): - id = createObjId(obj_class) - node_number = getObjPartition(id, node_count) - self.assertTrue(node_number >= 0) - self.assertTrue(node_number < node_count) - # try a domain partition - node_number = getObjPartition("/home/test_user1", node_count) - self.assertTrue(node_number >= 0) - self.assertTrue(node_number < node_count) - - def testGetCollection(self): - group_id = "g-314d61b8-9954-11e6-a733-3c15c2da029e" - dataset_id = "d-4c48f3ae-9954-11e6-a3cd-3c15c2da029e" - ctype_id = "t-8c785f1c-9953-11e6-9bc2-0242ac110005" - bad_id = "x-59647858-9954-11e6-95d2-3c15c2da029e" - self.assertEqual(getCollectionForId(group_id), "groups") - self.assertEqual(getCollectionForId(dataset_id), "datasets") - self.assertEqual(getCollectionForId(ctype_id), "datatypes") - try: - getCollectionForId(bad_id) - self.assertTrue(False) - except ValueError: - pass # expected - try: - getCollectionForId(None) - self.assertTrue(False) - except ValueError: - pass # expected - - def testSchema2Id(self): - root_id = createObjId("roots") - group_id = createObjId("groups", rootid=root_id) - dataset_id = createObjId("datasets", rootid=root_id) - ctype_id = createObjId("datatypes", rootid=root_id) - - self.assertEqual(getCollectionForId(root_id), "groups") - self.assertEqual(getCollectionForId(group_id), "groups") - self.assertEqual(getCollectionForId(dataset_id), "datasets") - self.assertEqual(getCollectionForId(ctype_id), "datatypes") - chunk_id = "c" + dataset_id[1:] + "_1_2" - print(chunk_id) - chunk_partition_id = "c42-" + dataset_id[2:] + "_1_2" - - for id in (chunk_id, chunk_partition_id): - try: - getCollectionForId(id) - self.assertTrue(False) - except ValueError: - pass # expected - valid_ids = ( - group_id, - dataset_id, - ctype_id, - chunk_id, - chunk_partition_id, - root_id, - ) - s3prefix = getS3Key(root_id) - self.assertTrue(s3prefix.endswith("/.group.json")) - s3prefix = s3prefix[: -(len(".group.json"))] - for oid in valid_ids: - print("oid:", oid) - self.assertTrue(len(oid) >= 38) - parts = oid.split("-") - self.assertEqual(len(parts), 6) - self.assertTrue(oid[0] in ("g", "d", "t", "c")) - self.assertTrue(isSchema2Id(oid)) - if oid == root_id: - self.assertTrue(isRootObjId(oid)) - else: - self.assertFalse(isRootObjId(oid)) - self.assertEqual(getRootObjId(oid), root_id) - - s3key = getS3Key(oid) - print(s3key) - self.assertTrue(s3key.startswith(s3prefix)) - self.assertEqual(getObjId(s3key), oid) - self.assertTrue(isS3ObjKey(s3key)) - - -if __name__ == "__main__": - # setup test files - - unittest.main() diff --git a/tests/unit/lru_cache_test.py b/tests/unit/lru_cache_test.py old mode 100755 new mode 100644 index 5e747c92..002ca822 --- a/tests/unit/lru_cache_test.py +++ b/tests/unit/lru_cache_test.py @@ -14,9 +14,16 @@ import sys import numpy as np +from h5json.objid import createObjId + sys.path.append("../..") from hsds.util.lruCache import LruCache -from hsds.util.idUtil import createObjId + + +def _createId(): + objid = createObjId("groups") + objid = 'c' + objid[1:] # fake a chunk id + return objid class LruCacheTest(unittest.TestCase): @@ -34,7 +41,7 @@ def testSimple(self): self.assertFalse("xyz" in cc) - id = createObjId("chunks") + id = _createId() try: # only dict objects can be added cc[id] = list(range(20)) @@ -42,7 +49,7 @@ def testSimple(self): except TypeError: pass # expected - rand_id = createObjId("chunks") + rand_id = _createId() np_arr = np.random.random((500, 500)) # smaller than our chunk cache size cc[rand_id] = np_arr # add to cache cc.consistencyCheck() @@ -104,7 +111,7 @@ def testLRU(self): ids = [] # add chunks to the cache for i in range(10): - id = createObjId("chunks") + id = _createId() ids.append(id) arr = np.empty((16, 16), dtype="i4") # 1024 bytes arr[...] = i @@ -165,7 +172,7 @@ def testClearCache(self): ids = [] # add chunks to the cache for i in range(10): - id = createObjId("chunks") + id = _createId() ids.append(id) arr = np.empty((16, 16), dtype="i4") # 1024 bytes arr[...] = i @@ -190,7 +197,7 @@ def testMemUtil(self): self.assertEqual(len(cc), 0) ids = set() for i in range(10): - id = createObjId("chunks") + id = _createId() ids.add(id) arr = np.empty((16, 16), dtype="i4") # 1024 bytes arr[...] = i @@ -208,7 +215,7 @@ def testMemUtil(self): # add 10 more chunks, but set dirty to true each time for i in range(10): - id = createObjId("chunks") + id = _createId() ids.add(id) arr = np.empty((16, 16), dtype="i4") # 1024 bytes arr[...] = i @@ -255,7 +262,7 @@ def testMetaDataCache(self): data = {"x": 123, "y": 456} - rand_id = createObjId("groups") + rand_id = _createId() data = {"foo": "bar"} cc[rand_id] = data # add to cache cc.consistencyCheck() diff --git a/tests/unit/openapi_test.py b/tests/unit/openapi_test.py new file mode 100644 index 00000000..3d828a07 --- /dev/null +++ b/tests/unit/openapi_test.py @@ -0,0 +1,51 @@ +############################################################################## +# Copyright by The HDF Group. # +# All rights reserved. # +# # +# This file is part of HSDS (HDF5 Scalable Data Service), Libraries and # +# Utilities. The full HSDS copyright notice, including # +# terms governing use, modification, and redistribution, is contained in # +# the file COPYING, which can be found at the root of the source code # +# distribution tree. If you do not have access to this file, you may # +# request a copy from help@hdfgroup.org. # +############################################################################## +import unittest +import os +import yaml + +# resolve relative to this file (not cwd) so this test works whether it's +# run from tests/unit, from the repo root, or via testall.py +THIS_DIR = os.path.dirname(os.path.abspath(__file__)) +REPO_ROOT = os.path.normpath(os.path.join(THIS_DIR, "..", "..")) +OPENAPI_PATH = os.path.join(REPO_ROOT, "openapi.yml") + + +class OpenApiTest(unittest.TestCase): + def testFileExists(self): + self.assertTrue( + os.path.isfile(OPENAPI_PATH), f"expected to find {OPENAPI_PATH}" + ) + + def testValidYaml(self): + with open(OPENAPI_PATH) as f: + doc = yaml.safe_load(f) + self.assertTrue(isinstance(doc, dict)) + for key in ("openapi", "info", "paths", "components"): + self.assertTrue(key in doc, f"expected top-level key: {key}") + self.assertTrue(doc["openapi"].startswith("3.")) + self.assertTrue(len(doc["paths"]) > 0) + + def testValidOpenApiSchema(self): + try: + from openapi_spec_validator import validate + from openapi_spec_validator.readers import read_from_filename + except ImportError: + self.skipTest("openapi_spec_validator not installed") + return + + spec_dict, _base_uri = read_from_filename(OPENAPI_PATH) + validate(spec_dict) # raises if the spec is invalid + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/shuffle_test.py b/tests/unit/shuffle_test.py old mode 100755 new mode 100644 diff --git a/tests/unit/stor_util_test.py b/tests/unit/stor_util_test.py old mode 100755 new mode 100644 index c0074bf2..6a3205ea --- a/tests/unit/stor_util_test.py +++ b/tests/unit/stor_util_test.py @@ -224,7 +224,7 @@ async def stor_util_test(self, app): await releaseStorageClient(app) def testStorUtil(self): - # run synchronus tests + # run synchronous tests self.s3path_test() bucket = config.get("hsds_unit_test_bucket") diff --git a/tests/util/delete_obj_dn.py b/tests/util/delete_obj_dn.py index 62fb29d3..496f91e0 100644 --- a/tests/util/delete_obj_dn.py +++ b/tests/util/delete_obj_dn.py @@ -40,7 +40,7 @@ def printUsage(): print("uri:", uri) if uri[0] != "/": - sys.exit("uri must start wtih '/'") + sys.exit("uri must start with '/'") req = "http://" + endpoint + ":" + str(port) + uri diff --git a/tests/util/get_obj_dn.py b/tests/util/get_obj_dn.py index 44e0fbf9..f269a538 100644 --- a/tests/util/get_obj_dn.py +++ b/tests/util/get_obj_dn.py @@ -39,7 +39,7 @@ def printUsage(): print("uri:", uri) if uri[0] != "/": - sys.exit("uri must start wtih '/'") + sys.exit("uri must start with '/'") req = "http://" + endpoint + ":" + str(port) + uri diff --git a/tests/util/get_obj_sn.py b/tests/util/get_obj_sn.py index 4f8e0f68..b88b9848 100644 --- a/tests/util/get_obj_sn.py +++ b/tests/util/get_obj_sn.py @@ -56,7 +56,7 @@ def printUsage(): print("user:", username) if uri[0] != "/": - sys.exit("uri must start wtih '/'") + sys.exit("uri must start with '/'") headers = {"host": domain} if username and password: diff --git a/tools/lambda_log_filter.py b/tools/lambda_log_filter.py deleted file mode 100644 index 48b97917..00000000 --- a/tools/lambda_log_filter.py +++ /dev/null @@ -1,112 +0,0 @@ -import sys - -""" -This script takes the output of Cloud watch log csv file generated by HSDS Lambda, -and converts it to a tab-seperated output with columns: - timestamp, loglevel, process, message -Note: timestamps may not be strictly increasing since there can be a log between -the time the message is logged in the subprocess and time it is processed in the -parent process (and then printed to stdout). -It might be useful to open in Excel and sort by the timestamp column. -""" -DEBUG = 10 -INFO = 20 -WARNING = 30 -ERROR = 40 - - -def getLogLevel(name): - if name.startswith("DEBUG"): - level = DEBUG - elif name.startswith("INFO"): - level = INFO - elif name.startswith("WARN"): - level = WARNING - elif name.startswith("ERROR"): - level = ERROR - else: - level = INFO - return level - - -# -# Main -# -if len(sys.argv) < 2 or sys.argv[1] in ("-h", "--help"): - print("usage: python zero_adjust_timestamp.py [log_file]") - sys.exit(0) - -log_file = sys.argv[1] -ts_zero = None - -# each element will be ts, loglevel, process (lambda, sn, dn[1-9], rg), msg -print("process\tlog_level\ttimestamp\tmessage") - -# cloud watch log format consists of lines like: -# 1636040451248,"got cpu_count of: 6" -# outuput lines timestamp adjusted to be deltas from first line - -# 1636134916895,"sn 1636134915.882 INFO> running in standalone mode - -with open(log_file, "r") as f: - while True: - line = f.readline() - if not line: - break - line = line.strip() - index = line.find(",") - if index < 0: - continue - ts_field = line[:index] - if ts_field == "timestamp": - # header line - continue - n = index + 2 - msg_field = line[n:] - if len(msg_field) > 500: - msg_field = msg_field[:500] + "..." - fields = msg_field.split() - if len(fields) == 0: - continue - f_0 = fields[0] - if f_0[0:2] in ("sn", "dn", "rg"): - # process hsds container log - process = fields[0] - timestamp = int(float(fields[1]) * 1000.0) - if ts_zero: - timestamp -= ts_zero - log_level = getLogLevel(fields[2]) - msg_field = fields[3] - for i in range(4, len(fields)): - msg_field += " " - msg_field += fields[i] - else: - process = "lambda" - timestamp = int(ts_field) - if ts_zero is None: - ts_zero = timestamp - timestamp -= ts_zero - if fields[0] == "DEBUG>": - log_level = DEBUG - debug_len = len("DEBUG> ") - msg_field = msg_field[debug_len:] - elif fields[0] == "INFO>": - log_level = INFO - info_len = len("INFO> ") - msg_field = msg_field[info_len:] - elif fields[0] == "WARN>": - log_level = WARNING - warn_len = len("WARN> ") - msg_field = msg_field[warn_len:] - elif fields[0] == "WARNING>": - log_level = WARNING - warning_len = len("WARNING> ") - msg_field = msg_field[warning_len:] - elif fields[0] == "ERROR>": - log_level = ERROR - error_len = len("ERROR> ") - msg_field = msg_field[error_len:] - else: - log_level = INFO - - print(f"{process}\t{log_level}\t{timestamp:6d}\t{msg_field}") diff --git a/tools/status_check.py b/tools/status_check.py index 66229686..42a87d22 100644 --- a/tools/status_check.py +++ b/tools/status_check.py @@ -7,26 +7,61 @@ # Continually check server status and output time and state. # -if "HSDS_ENDPOINT" not in os.environ: - print("HSDS_ENDPOINT not set") - sys.exit(1) +hsds_endpoint = None +no_stream = False +usage = f"usage: python {sys.argv[0]} [--endpoint 1 and sys.argv[1] in ("-h", "--help"): + print() + sys.exit(usage) + +argn = 1 +while len(sys.argv) > argn: + arg = sys.argv[argn] + if arg == "--no-stream": + no_stream = True + argn += 1 + elif arg == "--quiet": + quiet = True + argn += 1 + elif arg == "--endpoint": + if len(sys.argv) == argn: + sys.exit(usage) + endpoint = sys.argv[argn + 1] + argn += 2 + else: + sys.exit(usage) -hsds_endpoint = os.environ["HSDS_ENDPOINT"] +if hsds_endpoint is None: + if "HSDS_ENDPOINT" in os.environ: + hsds_endpoint = os.environ["HSDS_ENDPOINT"] + else: + sys.exit("HSDS_ENDPOINT not set") while True: now = int(time.time()) state = "" - status = 503 node_count = 0 try: rsp = requests.get(f"{hsds_endpoint}/about") - status_code = rsp.status_code + status = rsp.status_code - if rsp.status_code == 200: + if status == 200: rsp_json = rsp.json() state = rsp_json["state"] node_count = rsp_json["node_count"] except Exception as e: state = str(e) - print(f"{now}: {status_code}: {state} ({node_count} nodes)") + if not quiet: + print(f"{now}: {status}: {state} ({node_count} nodes)") + if no_stream: + break time.sleep(1) + +if state == "READY": + sys.exit(0) +else: + sys.exit(1)