diff --git a/.github/workflows/publish_snapshot.yml b/.github/workflows/publish_snapshot.yml index bbdc27c7..a6cf25c8 100644 --- a/.github/workflows/publish_snapshot.yml +++ b/.github/workflows/publish_snapshot.yml @@ -36,7 +36,7 @@ jobs: with: persist-credentials: false - - uses: actions/setup-java@dd06d9cba3e5552c54d9f8ea23572deb30010f7c # v6.0.0 + - uses: actions/setup-java@de7274f081f381c8f8158605e0321c36c376e2e6 # v6.0.1 with: distribution: temurin java-version: 17 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2aa0df74..95612595 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -50,7 +50,7 @@ jobs: with: persist-credentials: false - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@dd06d9cba3e5552c54d9f8ea23572deb30010f7c # v6.0.0 + uses: actions/setup-java@de7274f081f381c8f8158605e0321c36c376e2e6 # v6.0.1 with: java-version: ${{ matrix.java }} distribution: temurin diff --git a/LogicalTypes.md b/LogicalTypes.md index 559030fe..a5cf0843 100644 --- a/LogicalTypes.md +++ b/LogicalTypes.md @@ -1097,6 +1097,48 @@ optional group my_map (MAP_KEY_VALUE) { } ``` +### Finite Vectors + +`FINITE_VECTOR` is used to annotate fixed-length ordered sequences of finite and +non-null elements. + +`FiniteVectorType` annotation has one required parameter, `num_elements`, +which is the number of elements in each non-null vector and MUST be greater +than zero. + +`FINITE_VECTOR` MUST always annotate the canonical 3-level structure: + +``` + group (FINITE_VECTOR()) { + repeated group list { + required element; + } +} +``` + +The element types supported are: + +* unannotated `BOOLEAN`, `FLOAT` or `DOUBLE` +* `INT32` or `INT64`, either unannotated or annotated with `INTEGER` or + `DECIMAL` +* `FIXED_LEN_BYTE_ARRAY` annotated with `FLOAT16` or `DECIMAL`. + +Every numeric element MUST be finite. For `FLOAT`, `DOUBLE`, and `FLOAT16`, +this excludes NaN, positive infinity, and negative infinity. + +For example, a nullable vector containing 768 required `FLOAT` elements is: + +``` +optional group embedding (FINITE_VECTOR(768)) { + repeated group list { + required float element; + } +} +``` + +Every non-null vector must contain exactly `num_elements` elements. +Because `num_elements` is greater than zero, a non-null vector cannot be empty. + ## UNKNOWN (always null) Sometimes, when discovering the schema of existing data, values are always null diff --git a/dev/install-thrift.sh b/dev/install-thrift.sh index f2ac40b0..bdc15387 100755 --- a/dev/install-thrift.sh +++ b/dev/install-thrift.sh @@ -22,7 +22,7 @@ set -euo pipefail -THRIFT_VERSION="${THRIFT_VERSION:-0.24.0}" +THRIFT_VERSION="${THRIFT_VERSION:-0.25.0}" sudo apt-get update -qq sudo apt-get install -qq protobuf-compiler diff --git a/pom.xml b/pom.xml index 3e30a789..0a37462c 100644 --- a/pom.xml +++ b/pom.xml @@ -76,7 +76,7 @@ 1.8 shaded.parquet thrift - 0.24.0 + 0.25.0 0.10.0 @@ -243,7 +243,7 @@ org.codehaus.mojo exec-maven-plugin - 3.5.1 + 3.6.4 check-thrift-version diff --git a/src/main/thrift/parquet.thrift b/src/main/thrift/parquet.thrift index fea13fbe..c58b5631 100644 --- a/src/main/thrift/parquet.thrift +++ b/src/main/thrift/parquet.thrift @@ -479,6 +479,19 @@ struct GeographyType { struct FileType { } +/** +* Finite fixed-length vector logical type annotation +* +* Annotates the outer group of a three-level structure that is the same as +* the one used for LIST-annotated groups. Each non-null vector MUST contain +* exactly num_elements elements. +* +* See LogicalTypes.md for details. +*/ +struct FiniteVectorType { + 1: required i32 num_elements +} + /** * LogicalType annotations to replace ConvertedType. * @@ -513,6 +526,7 @@ union LogicalType { 17: GeometryType GEOMETRY // no compatible ConvertedType 18: GeographyType GEOGRAPHY // no compatible ConvertedType 19: FileType FILE // no compatible ConvertedType + 20: FiniteVectorType FINITE_VECTOR // use ConvertedType LIST } /** @@ -1129,6 +1143,7 @@ union ColumnOrder { * GEOMETRY - undefined * GEOGRAPHY - undefined * FILE - undefined + * FINITE_VECTOR - undefined * * In the absence of logical types, the sort order is determined by the physical type: * BOOLEAN - false, true