diff --git a/NOTICE.txt b/NOTICE.txt index b572b1fa29..5a99405303 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -8,3 +8,9 @@ This product includes software developed at Apache Gluten (https://github.com/apache/incubator-gluten/) Specifically: - Optimizer rule to replace SortMergeJoin with ShuffleHashJoin + +This product includes software developed at +Apache Hadoop (https://hadoop.apache.org/) +Specifically: +- The libhdfs C sources under native/hdfs-sys/libhdfs/, modified as described + in native/hdfs-sys/README.md diff --git a/dev/ci/compute-changes.py b/dev/ci/compute-changes.py index 9716ced234..1eea11973d 100644 --- a/dev/ci/compute-changes.py +++ b/dev/ci/compute-changes.py @@ -108,6 +108,9 @@ ], "spark_3_4": [ "native/**/src/**", + # The vendored Hadoop C in native/hdfs-sys/ is compiled into libcomet but + # lives outside any src/ directory, so it needs its own entry. + "native/hdfs-sys/**", "native/**/Cargo.toml", "native/Cargo.lock", "common/src/main/**", @@ -134,6 +137,9 @@ ], "spark_3_5": [ "native/**/src/**", + # The vendored Hadoop C in native/hdfs-sys/ is compiled into libcomet but + # lives outside any src/ directory, so it needs its own entry. + "native/hdfs-sys/**", "native/**/Cargo.toml", "native/Cargo.lock", "common/src/main/**", @@ -160,6 +166,9 @@ ], "spark_4_0": [ "native/**/src/**", + # The vendored Hadoop C in native/hdfs-sys/ is compiled into libcomet but + # lives outside any src/ directory, so it needs its own entry. + "native/hdfs-sys/**", "native/**/Cargo.toml", "native/Cargo.lock", "common/src/main/**", @@ -186,6 +195,9 @@ ], "spark_4_1": [ "native/**/src/**", + # The vendored Hadoop C in native/hdfs-sys/ is compiled into libcomet but + # lives outside any src/ directory, so it needs its own entry. + "native/hdfs-sys/**", "native/**/Cargo.toml", "native/Cargo.lock", "common/src/main/**", @@ -212,6 +224,9 @@ ], "iceberg_1_8": [ "native/**/src/**", + # The vendored Hadoop C in native/hdfs-sys/ is compiled into libcomet but + # lives outside any src/ directory, so it needs its own entry. + "native/hdfs-sys/**", "native/**/Cargo.toml", "native/Cargo.lock", "common/src/main/**", @@ -236,6 +251,9 @@ ], "iceberg_1_9": [ "native/**/src/**", + # The vendored Hadoop C in native/hdfs-sys/ is compiled into libcomet but + # lives outside any src/ directory, so it needs its own entry. + "native/hdfs-sys/**", "native/**/Cargo.toml", "native/Cargo.lock", "common/src/main/**", @@ -260,6 +278,9 @@ ], "iceberg_1_10": [ "native/**/src/**", + # The vendored Hadoop C in native/hdfs-sys/ is compiled into libcomet but + # lives outside any src/ directory, so it needs its own entry. + "native/hdfs-sys/**", "native/**/Cargo.toml", "native/Cargo.lock", "common/src/main/**", @@ -284,6 +305,9 @@ ], "iceberg_1_11": [ "native/**/src/**", + # The vendored Hadoop C in native/hdfs-sys/ is compiled into libcomet but + # lives outside any src/ directory, so it needs its own entry. + "native/hdfs-sys/**", "native/**/Cargo.toml", "native/Cargo.lock", "common/src/main/**", diff --git a/native/Cargo.lock b/native/Cargo.lock index df5fd4ac14..545f105b1f 100644 --- a/native/Cargo.lock +++ b/native/Cargo.lock @@ -3314,8 +3314,6 @@ dependencies = [ [[package]] name = "hdfs-sys" version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33e2d5cefba2d51a26b44d2a493f963a32725a0f6593c91be4a610ad449c49cb" dependencies = [ "cc", "java-locator", diff --git a/native/Cargo.toml b/native/Cargo.toml index 1805a185e9..b3fbe60a0c 100644 --- a/native/Cargo.toml +++ b/native/Cargo.toml @@ -17,7 +17,10 @@ [workspace] default-members = ["core", "spark-expr", "common", "proto", "jni-bridge", "shuffle"] -members = ["core", "spark-expr", "common", "proto", "jni-bridge", "shuffle"] +# `hdfs-sys` is a member so that `cargo fmt --all` and `cargo clippy --workspace` cover it, but +# deliberately not a default member: it is only reached through `[patch.crates-io]` when the +# `hdfs-opendal` feature is on, and building it needs a JDK and a C compiler. +members = ["core", "spark-expr", "common", "proto", "jni-bridge", "shuffle", "hdfs-sys"] # The contrib crate at ../contrib/delta/native is intentionally NOT a workspace member # (workspace members must live hierarchically under the workspace root). It's pulled in # as a path dep by `core/Cargo.toml` when the `contrib-delta` feature is enabled. @@ -82,3 +85,17 @@ codegen-units = 16 # Parallel codegen (faster compile, slightly larger binary) debug-assertions = true panic = "unwind" # Allow panics to be caught and logged across FFI boundary # overflow-checks inherited as false from release + +# `hdfs-sys` 0.3.0, the only release, ships a libhdfs whose pthread TLS +# destructor detaches threads it did not attach, dereferencing a freed JNIEnv and +# crashing the JVM (apache/datafusion-comet#5023). `hdfs-sys` is reached through +# `opendal`'s `services-hdfs` -> `hdrs`, so there is no version of the dependency +# to move to and no upstream release to wait for. `hdfs-sys/` vendors the Hadoop +# sources with the fix applied; see its README.md for the removal condition. +# +# Note that `[patch]` applies only to builds of this workspace. Comet's native +# crates are not published, so nothing downstream is affected today, but a +# published crate would silently resolve the unpatched `hdfs-sys`. +[patch.crates-io] +hdfs-sys = { path = "hdfs-sys" } + diff --git a/native/hdfs-sys/Cargo.toml b/native/hdfs-sys/Cargo.toml new file mode 100644 index 0000000000..0f53dc7c97 --- /dev/null +++ b/native/hdfs-sys/Cargo.toml @@ -0,0 +1,56 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[package] +# Deliberately named after the crates.io crate: `native/Cargo.toml` substitutes +# this for it via `[patch.crates-io]`, which requires a matching name and a +# semver-compatible version. See README.md. +name = "hdfs-sys" +version = "0.3.0" +edition = "2021" +description = "Bindings to the Apache Hadoop libhdfs C API, carrying the HDFS-16021 fix" +authors = ["Apache DataFusion "] +license = "Apache-2.0" +publish = false +readme = "README.md" + +[features] +# The crates.io crate gates each vendored Hadoop version behind a feature, with +# each enabling the one below it. Comet's dependency graph asks for `hdfs_3_3` +# (from `core/Cargo.toml`) and the default (from `hdrs`), and only the 3.3 +# sources are vendored here, so the older names exist purely to keep those +# requests resolvable. +default = ["hdfs_2_6"] +hdfs_2_2 = [] +hdfs_2_3 = ["hdfs_2_2"] +hdfs_2_4 = ["hdfs_2_3"] +hdfs_2_5 = ["hdfs_2_4"] +hdfs_2_6 = ["hdfs_2_5"] +hdfs_2_7 = ["hdfs_2_6"] +hdfs_2_8 = ["hdfs_2_7"] +hdfs_2_9 = ["hdfs_2_8"] +hdfs_2_10 = ["hdfs_2_9"] +hdfs_3_0 = ["hdfs_2_10"] +hdfs_3_1 = ["hdfs_3_0"] +hdfs_3_2 = ["hdfs_3_1"] +hdfs_3_3 = ["hdfs_3_2"] +# Skip the system-libhdfs search and always compile the vendored sources. +vendored = [] + +[build-dependencies] +cc = "1" +java-locator = { version = "0.1.9", features = ["locate-jdk-only"] } diff --git a/native/hdfs-sys/README.md b/native/hdfs-sys/README.md new file mode 100644 index 0000000000..884f44f0e8 --- /dev/null +++ b/native/hdfs-sys/README.md @@ -0,0 +1,98 @@ + + +# hdfs-sys + +Bindings to the Apache Hadoop `libhdfs` C API, carrying the +[HDFS-16021](https://issues.apache.org/jira/browse/HDFS-16021) thread-ownership fix. + +This crate substitutes for the crates.io [`hdfs-sys`](https://github.com/Xuanwo/hdfs-sys) crate +through a `[patch.crates-io]` entry in `native/Cargo.toml`. It is not published and is not +intended for use outside Comet. + +## Why it exists + +`libhdfs` registers a pthread thread-local destructor, `hdfsThreadDestructor`, which detaches the +current thread from the JVM. It does so for every thread that has a cached `JNIEnv`, including +threads that libhdfs did not attach. Comet attaches each of its Tokio worker threads with +`AttachCurrentThreadAsDaemon` and detaches them itself on thread stop, so by the time the pthread +destructor runs the `JNIEnv` has already been freed. Dereferencing it jumps through a null function +pointer and takes the JVM down with `SIGSEGV at pc=0x0`. + +That is [apache/datafusion-comet#5023](https://github.com/apache/datafusion-comet/issues/5023), +which reproduces on both Linux and macOS in the `[scans]` CI bucket. `ParquetReadFromFakeHadoopFsSuite` +is what routes a read through `libhdfs`, but the destructor fires whenever one of those threads +later exits, so the crash lands in an unrelated suite sharing the same JVM. + +There is no released `hdfs-sys` with the fix, and no way to avoid the dependency: it arrives through +`opendal`'s `services-hdfs` feature by way of `hdrs`. The last crates.io release is 0.3.0 from July +2023, and a fix merged upstream in January 2026 is still unreleased, so waiting is not a strategy. + +## Provenance + +| Component | Source | +| --------------------- | ---------------------------------------------------------------------------------------------------------------------------------------- | +| `libhdfs/hdfs_3_3/**` | Apache Hadoop, via the `hdfs-sys` 0.3.0 crate's vendored copy of `hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs` | +| `libhdfs/config.h` | Written for Comet. Hadoop generates this with CMake; the upstream crate ships it empty | +| `src/lib.rs` | Written for Comet, transcribed from the vendored `libhdfs/hdfs_3_3/include/hdfs/hdfs.h` | +| `build.rs` | Written for Comet, following the file list in Hadoop's `hadoop-hdfs-native-client/src/CMakeLists.txt` | + +The C sources are Apache Hadoop's own, carry their original ASF license headers, and are +unmodified apart from the changes listed below. No code authored by the `hdfs-sys` maintainer is +copied here: the Rust binding layer and the build script were written for Comet against Hadoop's +public header, which is why only the API surface `hdrs` uses is declared. + +Relative to the upstream crate this copy also drops everything Comet does not build: the twelve +vendored Hadoop versions older than 3.3, the Windows platform layer, and the bundled `libdirent` +(MIT). Comet ships no Windows native artifacts. + +## Modifications + +Three files differ from Hadoop's originals. Each carries a notice at the top of the file, as +required by section 4(b) of the Apache License. + +- `libhdfs/hdfs_3_3/os/thread_local_storage.h` — adds an `attachedByLibhdfs` flag to + `struct ThreadLocalState`. +- `libhdfs/hdfs_3_3/jni_helper.c` — `getGlobalJNIEnv` reports whether it attached the current + thread, and calls `GetEnv` before `AttachCurrentThread` so that an attachment made by the JVM or + by the embedding application is reused rather than claimed. +- `libhdfs/hdfs_3_3/os/posix/thread_local_storage.c` — `hdfsThreadDestructor` detaches only when + `attachedByLibhdfs` is set, and `threadLocalStorageCreate` initialises the two fields it + previously left holding `malloc` garbage. + +The first and third come from the patch attached to HDFS-16021. The `GetEnv` check in the second +does not, and is the part that matters for Comet: `AttachCurrentThread` succeeds on an +already-attached thread and returns the same `JNIEnv`, so without it libhdfs would still record +itself as the owner of an attachment Comet made. + +The same changes are proposed upstream as +[Xuanwo/hdfs-sys#47](https://github.com/Xuanwo/hdfs-sys/pull/47). + +## Removal condition + +Delete this directory, drop the `[patch.crates-io]` entry from `native/Cargo.toml`, and remove the +NOTICE.txt stanza once a crates.io release of `hdfs-sys` contains the fix. Hadoop's own copy is not +sufficient on its own: HDFS-16021 is still open, and trunk still has the unguarded destructor. + +## A system libhdfs will not carry the fix + +`build.rs` keeps the upstream resolution order, so setting `HDFS_LIB_DIR` or `HADOOP_HOME` links a +prebuilt `libhdfs` instead of compiling these sources, and that library has whatever behaviour its +own build gave it. The `vendored` feature, which Comet enables on macOS through `hdrs`, skips the +search. On Linux neither variable is set in CI, so the vendored sources are built there too. diff --git a/native/hdfs-sys/build.rs b/native/hdfs-sys/build.rs new file mode 100644 index 0000000000..1aa1d7ca0d --- /dev/null +++ b/native/hdfs-sys/build.rs @@ -0,0 +1,142 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Links `libjvm` and either finds a system `libhdfs` or compiles the vendored +//! Apache Hadoop sources under `libhdfs/hdfs_3_3/`. +//! +//! The resolution order matches the crates.io `hdfs-sys` crate this stands in +//! for, so setting `HDFS_LIB_DIR` or `HADOOP_HOME` still selects a system +//! `libhdfs`. Note that such a library will not carry the HDFS-16021 fix; see +//! `README.md`. + +use std::env; +use std::path::Path; + +type Result = std::result::Result>; + +/// The only vendored libhdfs version. Upstream ships every Hadoop release from +/// 2.2 onwards behind a feature; Comet builds `hdfs_3_3` and nothing else. +const SRC: &str = "libhdfs/hdfs_3_3"; + +fn main() -> Result<()> { + // Nothing to link against when docs.rs builds documentation. + if env::var_os("DOCS_RS").is_some() { + return Ok(()); + } + + link_jvm()?; + + if !find_system_libhdfs()? { + build_vendored_libhdfs()?; + } + + Ok(()) +} + +/// Points the linker at the `libjvm` belonging to the JDK that `java-locator` +/// resolves, which is `JAVA_HOME` when it is set. +fn link_jvm() -> Result<()> { + let jvm_path = java_locator::locate_jvm_dyn_library()?; + println!("cargo:rustc-link-lib=jvm"); + println!("cargo:rustc-link-search=native={jvm_path}"); + Ok(()) +} + +/// Returns `true` when a prebuilt `libhdfs` was found and linked. +/// +/// Checks `HDFS_LIB_DIR` first, then `HADOOP_HOME`. `HDFS_STATIC` selects static +/// linking. The `vendored` feature skips the search outright. +fn find_system_libhdfs() -> Result { + println!("cargo:rerun-if-env-changed=HDFS_LIB_DIR"); + println!("cargo:rerun-if-env-changed=HDFS_STATIC"); + println!("cargo:rerun-if-env-changed=HADOOP_HOME"); + + if cfg!(feature = "vendored") { + return Ok(false); + } + + let lib_dir = if let Ok(lib_dir) = env::var("HDFS_LIB_DIR") { + lib_dir + } else if let Ok(hadoop_home) = env::var("HADOOP_HOME") { + format!("{hadoop_home}/lib/native") + } else { + return Ok(false); + }; + + let mode = if env::var_os("HDFS_STATIC").is_some() { + "static" + } else { + "dylib" + }; + println!("cargo:rustc-link-search=native={lib_dir}"); + println!("cargo:rustc-link-lib={mode}=hdfs"); + + Ok(true) +} + +/// Compiles the vendored Hadoop C into a static `libhdfs.a`. +/// +/// The file list follows `hadoop-hdfs-native-client/src/CMakeLists.txt` for the +/// 3.3 tree: the three top-level translation units, `jclasses.c` (added in 3.3) +/// and the POSIX platform layer. `htable.c` was removed in 3.3, and the Windows +/// platform layer is not vendored because Comet ships no Windows native builds. +fn build_vendored_libhdfs() -> Result<()> { + let java_home = java_locator::locate_java_home()?; + + println!("cargo:rustc-link-lib=static=hdfs"); + + let mut builder = cc::Build::new(); + // The vendored sources are Hadoop's, not ours, and are not warning-clean. + builder.warnings(false); + builder.flag_if_supported("-w"); + builder.flag_if_supported("-fvisibility=hidden"); + // Restore the pre-GCC-10 tentative-definition behaviour the sources assume. + builder.flag_if_supported("-fcommon"); + + builder.include(format!("{java_home}/include")); + if cfg!(target_os = "linux") { + builder.include(format!("{java_home}/include/linux")); + } + if cfg!(target_os = "macos") { + builder.include(format!("{java_home}/include/darwin")); + } + + builder + .include("libhdfs") + .include(SRC) + .include(format!("{SRC}/include")) + .include(format!("{SRC}/os")) + .include(format!("{SRC}/os/posix")); + + for file in [ + "exception.c", + "jni_helper.c", + "hdfs.c", + "jclasses.c", + "os/posix/mutexes.c", + "os/posix/thread.c", + "os/posix/thread_local_storage.c", + ] { + let path = format!("{SRC}/{file}"); + assert!(Path::new(&path).exists(), "missing vendored source {path}"); + println!("cargo:rerun-if-changed={path}"); + builder.file(path); + } + + builder.compile("hdfs"); + Ok(()) +} diff --git a/native/hdfs-sys/libhdfs/config.h b/native/hdfs-sys/libhdfs/config.h new file mode 100644 index 0000000000..5950534a16 --- /dev/null +++ b/native/hdfs-sys/libhdfs/config.h @@ -0,0 +1,26 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * In Apache Hadoop this header is generated by CMake and defines the feature + * probes libhdfs compiles against, chiefly HAVE_BETTER_TLS. jni_helper.c + * includes it unconditionally, so it has to exist. Leaving every probe undefined + * selects the portable pthread_getspecific path in os/thread_local_storage.h, + * which is what this crate wants and what the upstream hdfs-sys crate also + * shipped (as a zero-byte file). + */ diff --git a/native/hdfs-sys/libhdfs/hdfs_3_3/exception.c b/native/hdfs-sys/libhdfs/hdfs_3_3/exception.c new file mode 100644 index 0000000000..fec9a103b4 --- /dev/null +++ b/native/hdfs-sys/libhdfs/hdfs_3_3/exception.c @@ -0,0 +1,272 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "exception.h" +#include "hdfs/hdfs.h" +#include "jclasses.h" +#include "jni_helper.h" +#include "platform.h" + +#include +#include +#include + +#define EXCEPTION_INFO_LEN (sizeof(gExceptionInfo)/sizeof(gExceptionInfo[0])) + +struct ExceptionInfo { + const char * const name; + int noPrintFlag; + int excErrno; +}; + +static const struct ExceptionInfo gExceptionInfo[] = { + { + "java.io.FileNotFoundException", + NOPRINT_EXC_FILE_NOT_FOUND, + ENOENT, + }, + { + "org.apache.hadoop.security.AccessControlException", + NOPRINT_EXC_ACCESS_CONTROL, + EACCES, + }, + { + "org.apache.hadoop.fs.UnresolvedLinkException", + NOPRINT_EXC_UNRESOLVED_LINK, + ENOLINK, + }, + { + "org.apache.hadoop.fs.ParentNotDirectoryException", + NOPRINT_EXC_PARENT_NOT_DIRECTORY, + ENOTDIR, + }, + { + "java.lang.IllegalArgumentException", + NOPRINT_EXC_ILLEGAL_ARGUMENT, + EINVAL, + }, + { + "java.lang.OutOfMemoryError", + 0, + ENOMEM, + }, + { + "org.apache.hadoop.hdfs.server.namenode.SafeModeException", + 0, + EROFS, + }, + { + "org.apache.hadoop.fs.FileAlreadyExistsException", + 0, + EEXIST, + }, + { + "org.apache.hadoop.hdfs.protocol.QuotaExceededException", + 0, + EDQUOT, + }, + { + "java.lang.UnsupportedOperationException", + 0, + ENOTSUP, + }, + { + "org.apache.hadoop.hdfs.server.namenode.LeaseExpiredException", + 0, + ESTALE, + }, +}; + +void getExceptionInfo(const char *excName, int noPrintFlags, + int *excErrno, int *shouldPrint) +{ + int i; + + for (i = 0; i < EXCEPTION_INFO_LEN; i++) { + if (strstr(gExceptionInfo[i].name, excName)) { + break; + } + } + if (i < EXCEPTION_INFO_LEN) { + *shouldPrint = !(gExceptionInfo[i].noPrintFlag & noPrintFlags); + *excErrno = gExceptionInfo[i].excErrno; + } else { + *shouldPrint = 1; + *excErrno = EINTERNAL; + } +} + +/** + * getExceptionUtilString: A helper function that calls 'methodName' in + * ExceptionUtils. The function 'methodName' should have a return type of a + * java String. + * + * @param env The JNI environment. + * @param exc The exception to get information for. + * @param methodName The method of ExceptionUtils to call that has a String + * return type. + * + * @return A C-type string containing the string returned by + * ExceptionUtils.'methodName', or NULL on failure. + */ +static char* getExceptionUtilString(JNIEnv *env, jthrowable exc, char *methodName) +{ + jthrowable jthr; + jvalue jVal; + jstring jStr = NULL; + char *excString = NULL; + jthr = invokeMethod(env, &jVal, STATIC, NULL, JC_EXCEPTION_UTILS, + methodName, "(Ljava/lang/Throwable;)Ljava/lang/String;", exc); + if (jthr) { + destroyLocalReference(env, jthr); + return NULL; + } + jStr = jVal.l; + jthr = newCStr(env, jStr, &excString); + if (jthr) { + destroyLocalReference(env, jthr); + return NULL; + } + destroyLocalReference(env, jStr); + return excString; +} + +int printExceptionAndFreeV(JNIEnv *env, jthrowable exc, int noPrintFlags, + const char *fmt, va_list ap) +{ + int i, noPrint, excErrno; + char *className = NULL; + jthrowable jthr; + const char *stackTrace; + const char *rootCause; + + jthr = classNameOfObject(exc, env, &className); + if (jthr) { + fprintf(stderr, "PrintExceptionAndFree: error determining class name " + "of exception.\n"); + className = strdup("(unknown)"); + destroyLocalReference(env, jthr); + } + for (i = 0; i < EXCEPTION_INFO_LEN; i++) { + if (!strcmp(gExceptionInfo[i].name, className)) { + break; + } + } + if (i < EXCEPTION_INFO_LEN) { + noPrint = (gExceptionInfo[i].noPrintFlag & noPrintFlags); + excErrno = gExceptionInfo[i].excErrno; + } else { + noPrint = 0; + excErrno = EINTERNAL; + } + + // We don't want to use ExceptionDescribe here, because that requires a + // pending exception. Instead, use ExceptionUtils. + rootCause = getExceptionUtilString(env, exc, "getRootCauseMessage"); + stackTrace = getExceptionUtilString(env, exc, "getStackTrace"); + // Save the exception details in the thread-local state. + setTLSExceptionStrings(rootCause, stackTrace); + + if (!noPrint) { + vfprintf(stderr, fmt, ap); + fprintf(stderr, " error:\n"); + + if (!rootCause) { + fprintf(stderr, "(unable to get root cause for %s)\n", className); + } else { + fprintf(stderr, "%s", rootCause); + } + if (!stackTrace) { + fprintf(stderr, "(unable to get stack trace for %s)\n", className); + } else { + fprintf(stderr, "%s", stackTrace); + } + } + + destroyLocalReference(env, exc); + free(className); + return excErrno; +} + +int printExceptionAndFree(JNIEnv *env, jthrowable exc, int noPrintFlags, + const char *fmt, ...) +{ + va_list ap; + int ret; + + va_start(ap, fmt); + ret = printExceptionAndFreeV(env, exc, noPrintFlags, fmt, ap); + va_end(ap); + return ret; +} + +int printPendingExceptionAndFree(JNIEnv *env, int noPrintFlags, + const char *fmt, ...) +{ + va_list ap; + int ret; + jthrowable exc; + + exc = (*env)->ExceptionOccurred(env); + if (!exc) { + va_start(ap, fmt); + vfprintf(stderr, fmt, ap); + va_end(ap); + fprintf(stderr, " error: (no exception)"); + ret = 0; + } else { + (*env)->ExceptionClear(env); + va_start(ap, fmt); + ret = printExceptionAndFreeV(env, exc, noPrintFlags, fmt, ap); + va_end(ap); + } + return ret; +} + +jthrowable getPendingExceptionAndClear(JNIEnv *env) +{ + jthrowable jthr = (*env)->ExceptionOccurred(env); + if (!jthr) + return NULL; + (*env)->ExceptionClear(env); + return jthr; +} + +jthrowable newRuntimeError(JNIEnv *env, const char *fmt, ...) +{ + char buf[512]; + jobject out, exc; + jstring jstr; + va_list ap; + + va_start(ap, fmt); + vsnprintf(buf, sizeof(buf), fmt, ap); + va_end(ap); + jstr = (*env)->NewStringUTF(env, buf); + if (!jstr) { + // We got an out of memory exception rather than a RuntimeException. + // Too bad... + return getPendingExceptionAndClear(env); + } + exc = constructNewObjectOfClass(env, &out, "RuntimeException", + "(java/lang/String;)V", jstr); + (*env)->DeleteLocalRef(env, jstr); + // Again, we'll either get an out of memory exception or the + // RuntimeException we wanted. + return (exc) ? exc : out; +} diff --git a/native/hdfs-sys/libhdfs/hdfs_3_3/exception.h b/native/hdfs-sys/libhdfs/hdfs_3_3/exception.h new file mode 100644 index 0000000000..cdf93a1606 --- /dev/null +++ b/native/hdfs-sys/libhdfs/hdfs_3_3/exception.h @@ -0,0 +1,165 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFS_EXCEPTION_H +#define LIBHDFS_EXCEPTION_H + +/** + * Exception handling routines for libhdfs. + * + * The convention we follow here is to clear pending exceptions as soon as they + * are raised. Never assume that the caller of your function will clean up + * after you-- do it yourself. Unhandled exceptions can lead to memory leaks + * and other undefined behavior. + * + * If you encounter an exception, return a local reference to it. The caller is + * responsible for freeing the local reference, by calling a function like + * printExceptionAndFree. (You can also free exceptions directly by calling + * DeleteLocalRef. However, that would not produce an error message, so it's + * usually not what you want.) + * + * The root cause and stack trace exception strings retrieved from the last + * exception that happened on a thread are stored in the corresponding + * thread local state and are accessed by hdfsGetLastExceptionRootCause and + * hdfsGetLastExceptionStackTrace respectively. + */ + +#include "platform.h" + +#include +#include + +#include +#include +#include +#include + +/** + * Exception noprint flags + * + * Theses flags determine which exceptions should NOT be printed to stderr by + * the exception printing routines. For example, if you expect to see + * FileNotFound, you might use NOPRINT_EXC_FILE_NOT_FOUND, to avoid filling the + * logs with messages about routine events. + * + * On the other hand, if you don't expect any failures, you might pass + * PRINT_EXC_ALL. + * + * You can OR these flags together to avoid printing multiple classes of + * exceptions. + */ +#define PRINT_EXC_ALL 0x00 +#define NOPRINT_EXC_FILE_NOT_FOUND 0x01 +#define NOPRINT_EXC_ACCESS_CONTROL 0x02 +#define NOPRINT_EXC_UNRESOLVED_LINK 0x04 +#define NOPRINT_EXC_PARENT_NOT_DIRECTORY 0x08 +#define NOPRINT_EXC_ILLEGAL_ARGUMENT 0x10 + +/** + * Get information about an exception. + * + * @param excName The Exception name. + * This is a Java class name in JNI format. + * @param noPrintFlags Flags which determine which exceptions we should NOT + * print. + * @param excErrno (out param) The POSIX error number associated with the + * exception. + * @param shouldPrint (out param) Nonzero if we should print this exception, + * based on the noPrintFlags and its name. + */ +void getExceptionInfo(const char *excName, int noPrintFlags, + int *excErrno, int *shouldPrint); + +/** + * Store the information about an exception in the thread-local state and print + * it and free the jthrowable object. + * + * @param env The JNI environment + * @param exc The exception to print and free + * @param noPrintFlags Flags which determine which exceptions we should NOT + * print. + * @param fmt Printf-style format list + * @param ap Printf-style varargs + * + * @return The POSIX error number associated with the exception + * object. + */ +int printExceptionAndFreeV(JNIEnv *env, jthrowable exc, int noPrintFlags, + const char *fmt, va_list ap); + +/** + * Store the information about an exception in the thread-local state and print + * it and free the jthrowable object. + * + * @param env The JNI environment + * @param exc The exception to print and free + * @param noPrintFlags Flags which determine which exceptions we should NOT + * print. + * @param fmt Printf-style format list + * @param ... Printf-style varargs + * + * @return The POSIX error number associated with the exception + * object. + */ +int printExceptionAndFree(JNIEnv *env, jthrowable exc, int noPrintFlags, + const char *fmt, ...) TYPE_CHECKED_PRINTF_FORMAT(4, 5); + +/** + * Store the information about the pending exception in the thread-local state + * and print it and free the jthrowable object. + * + * @param env The JNI environment + * @param noPrintFlags Flags which determine which exceptions we should NOT + * print. + * @param fmt Printf-style format list + * @param ... Printf-style varargs + * + * @return The POSIX error number associated with the exception + * object. + */ +int printPendingExceptionAndFree(JNIEnv *env, int noPrintFlags, + const char *fmt, ...) TYPE_CHECKED_PRINTF_FORMAT(3, 4); + +/** + * Get a local reference to the pending exception and clear it. + * + * Once it is cleared, the exception will no longer be pending. The caller will + * have to decide what to do with the exception object. + * + * @param env The JNI environment + * + * @return The exception, or NULL if there was no exception + */ +jthrowable getPendingExceptionAndClear(JNIEnv *env); + +/** + * Create a new runtime error. + * + * This creates (but does not throw) a new RuntimeError. + * + * @param env The JNI environment + * @param fmt Printf-style format list + * @param ... Printf-style varargs + * + * @return A local reference to a RuntimeError + */ +jthrowable newRuntimeError(JNIEnv *env, const char *fmt, ...) + TYPE_CHECKED_PRINTF_FORMAT(2, 3); + +#undef TYPE_CHECKED_PRINTF_FORMAT +#endif diff --git a/native/hdfs-sys/libhdfs/hdfs_3_3/hdfs.c b/native/hdfs-sys/libhdfs/hdfs_3_3/hdfs.c new file mode 100644 index 0000000000..60f2826c74 --- /dev/null +++ b/native/hdfs-sys/libhdfs/hdfs_3_3/hdfs.c @@ -0,0 +1,3831 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "exception.h" +#include "hdfs/hdfs.h" +#include "jclasses.h" +#include "jni_helper.h" +#include "platform.h" + +#include +#include +#include +#include + +#define JAVA_VOID "V" + +/* Macros for constructing method signatures */ +#define JPARAM(X) "L" X ";" +#define JARRPARAM(X) "[L" X ";" +#define JMETHOD1(X, R) "(" X ")" R +#define JMETHOD2(X, Y, R) "(" X Y ")" R +#define JMETHOD3(X, Y, Z, R) "(" X Y Z")" R + +#define KERBEROS_TICKET_CACHE_PATH "hadoop.security.kerberos.ticket.cache.path" + +// Bit fields for hdfsFile_internal flags +#define HDFS_FILE_SUPPORTS_DIRECT_READ (1<<0) +#define HDFS_FILE_SUPPORTS_DIRECT_PREAD (1<<1) + +/** + * Reads bytes using the read(ByteBuffer) API. By using Java + * DirectByteBuffers we can avoid copying the bytes onto the Java heap. + * Instead the data will be directly copied from kernel space to the C heap. + */ +tSize readDirect(hdfsFS fs, hdfsFile f, void* buffer, tSize length); + +/** + * Reads bytes using the read(long, ByteBuffer) API. By using Java + * DirectByteBuffers we can avoid copying the bytes onto the Java heap. + * Instead the data will be directly copied from kernel space to the C heap. + */ +tSize preadDirect(hdfsFS fs, hdfsFile file, tOffset position, void* buffer, + tSize length); + +int preadFullyDirect(hdfsFS fs, hdfsFile file, tOffset position, void* buffer, + tSize length); + +static void hdfsFreeFileInfoEntry(hdfsFileInfo *hdfsFileInfo); + +/** + * The C equivalent of org.apache.org.hadoop.FSData(Input|Output)Stream . + */ +enum hdfsStreamType +{ + HDFS_STREAM_UNINITIALIZED = 0, + HDFS_STREAM_INPUT = 1, + HDFS_STREAM_OUTPUT = 2, +}; + +/** + * The 'file-handle' to a file in hdfs. + */ +struct hdfsFile_internal { + void* file; + enum hdfsStreamType type; + int flags; +}; + +#define HDFS_EXTENDED_FILE_INFO_ENCRYPTED 0x1 + +/** + * Extended file information. + */ +struct hdfsExtendedFileInfo { + int flags; +}; + +int hdfsFileIsOpenForRead(hdfsFile file) +{ + return (file->type == HDFS_STREAM_INPUT); +} + +int hdfsGetHedgedReadMetrics(hdfsFS fs, struct hdfsHedgedReadMetrics **metrics) +{ + jthrowable jthr; + jobject hedgedReadMetrics = NULL; + jvalue jVal; + struct hdfsHedgedReadMetrics *m = NULL; + int ret; + jobject jFS = (jobject)fs; + JNIEnv* env = getJNIEnv(); + + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, + JC_DISTRIBUTED_FILE_SYSTEM, "getHedgedReadMetrics", + "()Lorg/apache/hadoop/hdfs/DFSHedgedReadMetrics;"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetHedgedReadMetrics: getHedgedReadMetrics failed"); + goto done; + } + hedgedReadMetrics = jVal.l; + + m = malloc(sizeof(struct hdfsHedgedReadMetrics)); + if (!m) { + ret = ENOMEM; + goto done; + } + + jthr = invokeMethod(env, &jVal, INSTANCE, hedgedReadMetrics, + JC_DFS_HEDGED_READ_METRICS, "getHedgedReadOps", "()J"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetHedgedReadStatistics: getHedgedReadOps failed"); + goto done; + } + m->hedgedReadOps = jVal.j; + + jthr = invokeMethod(env, &jVal, INSTANCE, hedgedReadMetrics, + JC_DFS_HEDGED_READ_METRICS, "getHedgedReadWins", "()J"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetHedgedReadStatistics: getHedgedReadWins failed"); + goto done; + } + m->hedgedReadOpsWin = jVal.j; + + jthr = invokeMethod(env, &jVal, INSTANCE, hedgedReadMetrics, + JC_DFS_HEDGED_READ_METRICS, "getHedgedReadOpsInCurThread", "()J"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetHedgedReadStatistics: getHedgedReadOpsInCurThread failed"); + goto done; + } + m->hedgedReadOpsInCurThread = jVal.j; + + *metrics = m; + m = NULL; + ret = 0; + +done: + destroyLocalReference(env, hedgedReadMetrics); + free(m); + if (ret) { + errno = ret; + return -1; + } + return 0; +} + +void hdfsFreeHedgedReadMetrics(struct hdfsHedgedReadMetrics *metrics) +{ + free(metrics); +} + +int hdfsFileGetReadStatistics(hdfsFile file, + struct hdfsReadStatistics **stats) +{ + jthrowable jthr; + jobject readStats = NULL; + jvalue jVal; + struct hdfsReadStatistics *s = NULL; + int ret; + JNIEnv* env = getJNIEnv(); + + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + if (file->type != HDFS_STREAM_INPUT) { + ret = EINVAL; + goto done; + } + jthr = invokeMethod(env, &jVal, INSTANCE, file->file, + JC_HDFS_DATA_INPUT_STREAM, "getReadStatistics", + "()Lorg/apache/hadoop/hdfs/ReadStatistics;"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsFileGetReadStatistics: getReadStatistics failed"); + goto done; + } + readStats = jVal.l; + s = malloc(sizeof(struct hdfsReadStatistics)); + if (!s) { + ret = ENOMEM; + goto done; + } + jthr = invokeMethod(env, &jVal, INSTANCE, readStats, + JC_READ_STATISTICS, "getTotalBytesRead", "()J"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsFileGetReadStatistics: getTotalBytesRead failed"); + goto done; + } + s->totalBytesRead = jVal.j; + + jthr = invokeMethod(env, &jVal, INSTANCE, readStats, + JC_READ_STATISTICS, "getTotalLocalBytesRead", "()J"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsFileGetReadStatistics: getTotalLocalBytesRead failed"); + goto done; + } + s->totalLocalBytesRead = jVal.j; + + jthr = invokeMethod(env, &jVal, INSTANCE, readStats, + JC_READ_STATISTICS, "getTotalShortCircuitBytesRead", + "()J"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsFileGetReadStatistics: getTotalShortCircuitBytesRead failed"); + goto done; + } + s->totalShortCircuitBytesRead = jVal.j; + jthr = invokeMethod(env, &jVal, INSTANCE, readStats, + JC_READ_STATISTICS, "getTotalZeroCopyBytesRead", + "()J"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsFileGetReadStatistics: getTotalZeroCopyBytesRead failed"); + goto done; + } + s->totalZeroCopyBytesRead = jVal.j; + *stats = s; + s = NULL; + ret = 0; + +done: + destroyLocalReference(env, readStats); + free(s); + if (ret) { + errno = ret; + return -1; + } + return 0; +} + +int64_t hdfsReadStatisticsGetRemoteBytesRead( + const struct hdfsReadStatistics *stats) +{ + return stats->totalBytesRead - stats->totalLocalBytesRead; +} + +int hdfsFileClearReadStatistics(hdfsFile file) +{ + jthrowable jthr; + int ret; + JNIEnv* env = getJNIEnv(); + + if (env == NULL) { + errno = EINTERNAL; + return EINTERNAL; + } + if (file->type != HDFS_STREAM_INPUT) { + ret = EINVAL; + goto done; + } + jthr = invokeMethod(env, NULL, INSTANCE, file->file, + JC_HDFS_DATA_INPUT_STREAM, "clearReadStatistics", + "()V"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsFileClearReadStatistics: clearReadStatistics failed"); + goto done; + } + ret = 0; +done: + if (ret) { + errno = ret; + return ret; + } + return 0; +} + +void hdfsFileFreeReadStatistics(struct hdfsReadStatistics *stats) +{ + free(stats); +} + +int hdfsFileIsOpenForWrite(hdfsFile file) +{ + return (file->type == HDFS_STREAM_OUTPUT); +} + +int hdfsFileUsesDirectRead(hdfsFile file) +{ + return (file->flags & HDFS_FILE_SUPPORTS_DIRECT_READ) != 0; +} + +void hdfsFileDisableDirectRead(hdfsFile file) +{ + file->flags &= ~HDFS_FILE_SUPPORTS_DIRECT_READ; +} + +int hdfsFileUsesDirectPread(hdfsFile file) +{ + return (file->flags & HDFS_FILE_SUPPORTS_DIRECT_PREAD) != 0; +} + +void hdfsFileDisableDirectPread(hdfsFile file) +{ + file->flags &= ~HDFS_FILE_SUPPORTS_DIRECT_PREAD; +} + + +int hdfsDisableDomainSocketSecurity(void) +{ + jthrowable jthr; + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + jthr = invokeMethod(env, NULL, STATIC, NULL, JC_DOMAIN_SOCKET, + "disableBindPathValidation", "()V"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "DomainSocket#disableBindPathValidation"); + return -1; + } + return 0; +} + +/** + * hdfsJniEnv: A wrapper struct to be used as 'value' + * while saving thread -> JNIEnv* mappings + */ +typedef struct +{ + JNIEnv* env; +} hdfsJniEnv; + +/** + * Helper function to create a org.apache.hadoop.fs.Path object. + * @param env: The JNIEnv pointer. + * @param path: The file-path for which to construct org.apache.hadoop.fs.Path + * object. + * @return Returns a jobject on success and NULL on error. + */ +static jthrowable constructNewObjectOfPath(JNIEnv *env, const char *path, + jobject *out) +{ + jthrowable jthr; + jstring jPathString; + jobject jPath; + + //Construct a java.lang.String object + jthr = newJavaStr(env, path, &jPathString); + if (jthr) + return jthr; + //Construct the org.apache.hadoop.fs.Path object + jthr = constructNewObjectOfCachedClass(env, &jPath, JC_PATH, + "(Ljava/lang/String;)V", jPathString); + destroyLocalReference(env, jPathString); + if (jthr) + return jthr; + *out = jPath; + return NULL; +} + +static jthrowable hadoopConfGetStr(JNIEnv *env, jobject jConfiguration, + const char *key, char **val) +{ + jthrowable jthr; + jvalue jVal; + jstring jkey = NULL, jRet = NULL; + + jthr = newJavaStr(env, key, &jkey); + if (jthr) + goto done; + jthr = invokeMethod(env, &jVal, INSTANCE, jConfiguration, + JC_CONFIGURATION, "get", JMETHOD1(JPARAM(JAVA_STRING), + JPARAM(JAVA_STRING)), jkey); + if (jthr) + goto done; + jRet = jVal.l; + jthr = newCStr(env, jRet, val); +done: + destroyLocalReference(env, jkey); + destroyLocalReference(env, jRet); + return jthr; +} + +int hdfsConfGetStr(const char *key, char **val) +{ + JNIEnv *env; + int ret; + jthrowable jthr; + jobject jConfiguration = NULL; + + env = getJNIEnv(); + if (env == NULL) { + ret = EINTERNAL; + goto done; + } + jthr = constructNewObjectOfCachedClass(env, &jConfiguration, + JC_CONFIGURATION, "()V"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsConfGetStr(%s): new Configuration", key); + goto done; + } + jthr = hadoopConfGetStr(env, jConfiguration, key, val); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsConfGetStr(%s): hadoopConfGetStr", key); + goto done; + } + ret = 0; +done: + destroyLocalReference(env, jConfiguration); + if (ret) + errno = ret; + return ret; +} + +void hdfsConfStrFree(char *val) +{ + free(val); +} + +static jthrowable hadoopConfGetInt(JNIEnv *env, jobject jConfiguration, + const char *key, int32_t *val) +{ + jthrowable jthr = NULL; + jvalue jVal; + jstring jkey = NULL; + + jthr = newJavaStr(env, key, &jkey); + if (jthr) + return jthr; + jthr = invokeMethod(env, &jVal, INSTANCE, jConfiguration, + JC_CONFIGURATION, "getInt", + JMETHOD2(JPARAM(JAVA_STRING), "I", "I"), jkey, (jint)(*val)); + destroyLocalReference(env, jkey); + if (jthr) + return jthr; + *val = jVal.i; + return NULL; +} + +int hdfsConfGetInt(const char *key, int32_t *val) +{ + JNIEnv *env; + int ret; + jobject jConfiguration = NULL; + jthrowable jthr; + + env = getJNIEnv(); + if (env == NULL) { + ret = EINTERNAL; + goto done; + } + jthr = constructNewObjectOfCachedClass(env, &jConfiguration, + JC_CONFIGURATION, "()V"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsConfGetInt(%s): new Configuration", key); + goto done; + } + jthr = hadoopConfGetInt(env, jConfiguration, key, val); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsConfGetInt(%s): hadoopConfGetInt", key); + goto done; + } + ret = 0; +done: + destroyLocalReference(env, jConfiguration); + if (ret) + errno = ret; + return ret; +} + +struct hdfsBuilderConfOpt { + struct hdfsBuilderConfOpt *next; + const char *key; + const char *val; +}; + +struct hdfsBuilder { + int forceNewInstance; + const char *nn; + tPort port; + const char *kerbTicketCachePath; + const char *userName; + struct hdfsBuilderConfOpt *opts; +}; + +struct hdfsBuilder *hdfsNewBuilder(void) +{ + struct hdfsBuilder *bld = calloc(1, sizeof(struct hdfsBuilder)); + if (!bld) { + errno = ENOMEM; + return NULL; + } + return bld; +} + +int hdfsBuilderConfSetStr(struct hdfsBuilder *bld, const char *key, + const char *val) +{ + struct hdfsBuilderConfOpt *opt, *next; + + opt = calloc(1, sizeof(struct hdfsBuilderConfOpt)); + if (!opt) + return -ENOMEM; + next = bld->opts; + bld->opts = opt; + opt->next = next; + opt->key = key; + opt->val = val; + return 0; +} + +void hdfsFreeBuilder(struct hdfsBuilder *bld) +{ + struct hdfsBuilderConfOpt *cur, *next; + + cur = bld->opts; + for (cur = bld->opts; cur; ) { + next = cur->next; + free(cur); + cur = next; + } + free(bld); +} + +void hdfsBuilderSetForceNewInstance(struct hdfsBuilder *bld) +{ + bld->forceNewInstance = 1; +} + +void hdfsBuilderSetNameNode(struct hdfsBuilder *bld, const char *nn) +{ + bld->nn = nn; +} + +void hdfsBuilderSetNameNodePort(struct hdfsBuilder *bld, tPort port) +{ + bld->port = port; +} + +void hdfsBuilderSetUserName(struct hdfsBuilder *bld, const char *userName) +{ + bld->userName = userName; +} + +void hdfsBuilderSetKerbTicketCachePath(struct hdfsBuilder *bld, + const char *kerbTicketCachePath) +{ + bld->kerbTicketCachePath = kerbTicketCachePath; +} + +hdfsFS hdfsConnect(const char *host, tPort port) +{ + struct hdfsBuilder *bld = hdfsNewBuilder(); + if (!bld) + return NULL; + hdfsBuilderSetNameNode(bld, host); + hdfsBuilderSetNameNodePort(bld, port); + return hdfsBuilderConnect(bld); +} + +/** Always return a new FileSystem handle */ +hdfsFS hdfsConnectNewInstance(const char *host, tPort port) +{ + struct hdfsBuilder *bld = hdfsNewBuilder(); + if (!bld) + return NULL; + hdfsBuilderSetNameNode(bld, host); + hdfsBuilderSetNameNodePort(bld, port); + hdfsBuilderSetForceNewInstance(bld); + return hdfsBuilderConnect(bld); +} + +hdfsFS hdfsConnectAsUser(const char *host, tPort port, const char *user) +{ + struct hdfsBuilder *bld = hdfsNewBuilder(); + if (!bld) + return NULL; + hdfsBuilderSetNameNode(bld, host); + hdfsBuilderSetNameNodePort(bld, port); + hdfsBuilderSetUserName(bld, user); + return hdfsBuilderConnect(bld); +} + +/** Always return a new FileSystem handle */ +hdfsFS hdfsConnectAsUserNewInstance(const char *host, tPort port, + const char *user) +{ + struct hdfsBuilder *bld = hdfsNewBuilder(); + if (!bld) + return NULL; + hdfsBuilderSetNameNode(bld, host); + hdfsBuilderSetNameNodePort(bld, port); + hdfsBuilderSetForceNewInstance(bld); + hdfsBuilderSetUserName(bld, user); + return hdfsBuilderConnect(bld); +} + + +/** + * Calculate the effective URI to use, given a builder configuration. + * + * If there is not already a URI scheme, we prepend 'hdfs://'. + * + * If there is not already a port specified, and a port was given to the + * builder, we suffix that port. If there is a port specified but also one in + * the URI, that is an error. + * + * @param bld The hdfs builder object + * @param uri (out param) dynamically allocated string representing the + * effective URI + * + * @return 0 on success; error code otherwise + */ +static int calcEffectiveURI(struct hdfsBuilder *bld, char ** uri) +{ + const char *scheme; + char suffix[64]; + const char *lastColon; + char *u; + size_t uriLen; + + if (!bld->nn) + return EINVAL; + scheme = (strstr(bld->nn, "://")) ? "" : "hdfs://"; + if (bld->port == 0) { + suffix[0] = '\0'; + } else { + lastColon = strrchr(bld->nn, ':'); + if (lastColon && (strspn(lastColon + 1, "0123456789") == + strlen(lastColon + 1))) { + fprintf(stderr, "port %d was given, but URI '%s' already " + "contains a port!\n", bld->port, bld->nn); + return EINVAL; + } + snprintf(suffix, sizeof(suffix), ":%d", bld->port); + } + + uriLen = strlen(scheme) + strlen(bld->nn) + strlen(suffix); + u = malloc((uriLen + 1) * (sizeof(char))); + if (!u) { + fprintf(stderr, "calcEffectiveURI: out of memory"); + return ENOMEM; + } + snprintf(u, uriLen + 1, "%s%s%s", scheme, bld->nn, suffix); + *uri = u; + return 0; +} + +static const char *maybeNull(const char *str) +{ + return str ? str : "(NULL)"; +} + +static const char *hdfsBuilderToStr(const struct hdfsBuilder *bld, + char *buf, size_t bufLen) +{ + snprintf(buf, bufLen, "forceNewInstance=%d, nn=%s, port=%d, " + "kerbTicketCachePath=%s, userName=%s", + bld->forceNewInstance, maybeNull(bld->nn), bld->port, + maybeNull(bld->kerbTicketCachePath), maybeNull(bld->userName)); + return buf; +} + +hdfsFS hdfsBuilderConnect(struct hdfsBuilder *bld) +{ + JNIEnv *env = 0; + jobject jConfiguration = NULL, jFS = NULL, jURI = NULL, jCachePath = NULL; + jstring jURIString = NULL, jUserString = NULL; + jvalue jVal; + jthrowable jthr = NULL; + char *cURI = 0, buf[512]; + int ret; + jobject jRet = NULL; + struct hdfsBuilderConfOpt *opt; + + //Get the JNIEnv* corresponding to current thread + env = getJNIEnv(); + if (env == NULL) { + ret = EINTERNAL; + goto done; + } + + // jConfiguration = new Configuration(); + jthr = constructNewObjectOfCachedClass(env, &jConfiguration, + JC_CONFIGURATION, "()V"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + // set configuration values + for (opt = bld->opts; opt; opt = opt->next) { + jthr = hadoopConfSetStr(env, jConfiguration, opt->key, opt->val); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s): error setting conf '%s' to '%s'", + hdfsBuilderToStr(bld, buf, sizeof(buf)), opt->key, opt->val); + goto done; + } + } + + //Check what type of FileSystem the caller wants... + if (bld->nn == NULL) { + // Get a local filesystem. + if (bld->forceNewInstance) { + // fs = FileSytem#newInstanceLocal(conf); + jthr = invokeMethod(env, &jVal, STATIC, NULL, + JC_FILE_SYSTEM, "newInstanceLocal", + JMETHOD1(JPARAM(HADOOP_CONF), JPARAM(HADOOP_LOCALFS)), + jConfiguration); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + jFS = jVal.l; + } else { + // fs = FileSytem#getLocal(conf); + jthr = invokeMethod(env, &jVal, STATIC, NULL, + JC_FILE_SYSTEM, "getLocal", + JMETHOD1(JPARAM(HADOOP_CONF), JPARAM(HADOOP_LOCALFS)), + jConfiguration); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + jFS = jVal.l; + } + } else { + if (!strcmp(bld->nn, "default")) { + // jURI = FileSystem.getDefaultUri(conf) + jthr = invokeMethod(env, &jVal, STATIC, NULL, + JC_FILE_SYSTEM, "getDefaultUri", + "(Lorg/apache/hadoop/conf/Configuration;)Ljava/net/URI;", + jConfiguration); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + jURI = jVal.l; + } else { + // fs = FileSystem#get(URI, conf, ugi); + ret = calcEffectiveURI(bld, &cURI); + if (ret) + goto done; + jthr = newJavaStr(env, cURI, &jURIString); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + jthr = invokeMethod(env, &jVal, STATIC, NULL, + JC_URI, "create", + "(Ljava/lang/String;)Ljava/net/URI;", jURIString); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + jURI = jVal.l; + } + + if (bld->kerbTicketCachePath) { + jthr = hadoopConfSetStr(env, jConfiguration, + KERBEROS_TICKET_CACHE_PATH, bld->kerbTicketCachePath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + } + jthr = newJavaStr(env, bld->userName, &jUserString); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + if (bld->forceNewInstance) { + jthr = invokeMethod(env, &jVal, STATIC, NULL, + JC_FILE_SYSTEM, "newInstance", + JMETHOD3(JPARAM(JAVA_NET_URI), JPARAM(HADOOP_CONF), + JPARAM(JAVA_STRING), JPARAM(HADOOP_FS)), jURI, + jConfiguration, jUserString); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + jFS = jVal.l; + } else { + jthr = invokeMethod(env, &jVal, STATIC, NULL, + JC_FILE_SYSTEM, "get", + JMETHOD3(JPARAM(JAVA_NET_URI), JPARAM(HADOOP_CONF), + JPARAM(JAVA_STRING), JPARAM(HADOOP_FS)), jURI, + jConfiguration, jUserString); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + jFS = jVal.l; + } + } + jRet = (*env)->NewGlobalRef(env, jFS); + if (!jRet) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + ret = 0; + +done: + // Release unnecessary local references + destroyLocalReference(env, jConfiguration); + destroyLocalReference(env, jFS); + destroyLocalReference(env, jURI); + destroyLocalReference(env, jCachePath); + destroyLocalReference(env, jURIString); + destroyLocalReference(env, jUserString); + free(cURI); + hdfsFreeBuilder(bld); + + if (ret) { + errno = ret; + return NULL; + } + return (hdfsFS)jRet; +} + +int hdfsDisconnect(hdfsFS fs) +{ + // JAVA EQUIVALENT: + // fs.close() + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + int ret; + jobject jFS; + jthrowable jthr; + + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Parameters + jFS = (jobject)fs; + + //Sanity check + if (fs == NULL) { + errno = EBADF; + return -1; + } + + jthr = invokeMethod(env, NULL, INSTANCE, jFS, JC_FILE_SYSTEM, + "close", "()V"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsDisconnect: FileSystem#close"); + } else { + ret = 0; + } + (*env)->DeleteGlobalRef(env, jFS); + if (ret) { + errno = ret; + return -1; + } + return 0; +} + +/** + * Get the default block size of a FileSystem object. + * + * @param env The Java env + * @param jFS The FileSystem object + * @param jPath The path to find the default blocksize at + * @param out (out param) the default block size + * + * @return NULL on success; or the exception + */ +static jthrowable getDefaultBlockSize(JNIEnv *env, jobject jFS, + jobject jPath, jlong *out) +{ + jthrowable jthr; + jvalue jVal; + + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, JC_FILE_SYSTEM, + "getDefaultBlockSize", JMETHOD1(JPARAM(HADOOP_PATH), + "J"), jPath); + if (jthr) + return jthr; + *out = jVal.j; + return NULL; +} + +hdfsFile hdfsOpenFile(hdfsFS fs, const char *path, int flags, + int bufferSize, short replication, tSize blockSize) +{ + struct hdfsStreamBuilder *bld = hdfsStreamBuilderAlloc(fs, path, flags); + if (bufferSize != 0) { + hdfsStreamBuilderSetBufferSize(bld, bufferSize); + } + if (replication != 0) { + hdfsStreamBuilderSetReplication(bld, replication); + } + if (blockSize != 0) { + hdfsStreamBuilderSetDefaultBlockSize(bld, blockSize); + } + return hdfsStreamBuilderBuild(bld); +} + +struct hdfsStreamBuilder { + hdfsFS fs; + int flags; + int32_t bufferSize; + int16_t replication; + int64_t defaultBlockSize; + char path[1]; +}; + +struct hdfsStreamBuilder *hdfsStreamBuilderAlloc(hdfsFS fs, + const char *path, int flags) +{ + size_t path_len = strlen(path); + struct hdfsStreamBuilder *bld; + + // Check for overflow in path_len + if (path_len > SIZE_MAX - sizeof(struct hdfsStreamBuilder)) { + errno = EOVERFLOW; + return NULL; + } + // sizeof(hdfsStreamBuilder->path) includes one byte for the string + // terminator + bld = malloc(sizeof(struct hdfsStreamBuilder) + path_len); + if (!bld) { + errno = ENOMEM; + return NULL; + } + bld->fs = fs; + bld->flags = flags; + bld->bufferSize = 0; + bld->replication = 0; + bld->defaultBlockSize = 0; + memcpy(bld->path, path, path_len); + bld->path[path_len] = '\0'; + return bld; +} + +void hdfsStreamBuilderFree(struct hdfsStreamBuilder *bld) +{ + free(bld); +} + +int hdfsStreamBuilderSetBufferSize(struct hdfsStreamBuilder *bld, + int32_t bufferSize) +{ + if ((bld->flags & O_ACCMODE) != O_WRONLY) { + errno = EINVAL; + return -1; + } + bld->bufferSize = bufferSize; + return 0; +} + +int hdfsStreamBuilderSetReplication(struct hdfsStreamBuilder *bld, + int16_t replication) +{ + if ((bld->flags & O_ACCMODE) != O_WRONLY) { + errno = EINVAL; + return -1; + } + bld->replication = replication; + return 0; +} + +int hdfsStreamBuilderSetDefaultBlockSize(struct hdfsStreamBuilder *bld, + int64_t defaultBlockSize) +{ + if ((bld->flags & O_ACCMODE) != O_WRONLY) { + errno = EINVAL; + return -1; + } + bld->defaultBlockSize = defaultBlockSize; + return 0; +} + +/** + * Delegates to FsDataInputStream#hasCapability(String). Used to check if a + * given input stream supports certain methods, such as + * ByteBufferReadable#read(ByteBuffer). + * + * @param jFile the FsDataInputStream to call hasCapability on + * @param capability the name of the capability to query; for a full list of + * possible values see StreamCapabilities + * + * @return true if the given jFile has the given capability, false otherwise + * + * @see org.apache.hadoop.fs.StreamCapabilities + */ +static int hdfsHasStreamCapability(jobject jFile, + const char *capability) { + int ret = 0; + jthrowable jthr = NULL; + jvalue jVal; + jstring jCapabilityString = NULL; + + /* Get the JNIEnv* corresponding to current thread */ + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return 0; + } + + jthr = newJavaStr(env, capability, &jCapabilityString); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsHasStreamCapability(%s): newJavaStr", capability); + goto done; + } + jthr = invokeMethod(env, &jVal, INSTANCE, jFile, + JC_FS_DATA_INPUT_STREAM, "hasCapability", "(Ljava/lang/String;)Z", + jCapabilityString); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsHasStreamCapability(%s): FSDataInputStream#hasCapability", + capability); + goto done; + } + +done: + destroyLocalReference(env, jthr); + destroyLocalReference(env, jCapabilityString); + if (ret) { + errno = ret; + return 0; + } + if (jVal.z == JNI_TRUE) { + return 1; + } + return 0; +} + +static hdfsFile hdfsOpenFileImpl(hdfsFS fs, const char *path, int flags, + int32_t bufferSize, int16_t replication, int64_t blockSize) +{ + /* + JAVA EQUIVALENT: + File f = new File(path); + FSData{Input|Output}Stream f{is|os} = fs.create(f); + return f{is|os}; + */ + int accmode = flags & O_ACCMODE; + jstring jStrBufferSize = NULL, jStrReplication = NULL; + jobject jConfiguration = NULL, jPath = NULL, jFile = NULL; + jobject jFS = (jobject)fs; + jthrowable jthr; + jvalue jVal; + hdfsFile file = NULL; + int ret; + jint jBufferSize = bufferSize; + jshort jReplication = replication; + + /* The hadoop java api/signature */ + const char *method = NULL; + const char *signature = NULL; + + /* Get the JNIEnv* corresponding to current thread */ + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return NULL; + } + + + if (accmode == O_RDONLY || accmode == O_WRONLY) { + /* yay */ + } else if (accmode == O_RDWR) { + fprintf(stderr, "ERROR: cannot open an hdfs file in O_RDWR mode\n"); + errno = ENOTSUP; + return NULL; + } else { + fprintf(stderr, "ERROR: cannot open an hdfs file in mode 0x%x\n", + accmode); + errno = EINVAL; + return NULL; + } + + if ((flags & O_CREAT) && (flags & O_EXCL)) { + fprintf(stderr, + "WARN: hdfs does not truly support O_CREATE && O_EXCL\n"); + } + + if (accmode == O_RDONLY) { + method = "open"; + signature = JMETHOD2(JPARAM(HADOOP_PATH), "I", JPARAM(HADOOP_FSDISTRM)); + } else if (flags & O_APPEND) { + method = "append"; + signature = JMETHOD1(JPARAM(HADOOP_PATH), JPARAM(HADOOP_FSDOSTRM)); + } else { + method = "create"; + signature = JMETHOD2(JPARAM(HADOOP_PATH), "ZISJ", JPARAM(HADOOP_FSDOSTRM)); + } + + /* Create an object of org.apache.hadoop.fs.Path */ + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsOpenFile(%s): constructNewObjectOfPath", path); + goto done; + } + + /* Get the Configuration object from the FileSystem object */ + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, JC_FILE_SYSTEM, + "getConf", JMETHOD1("", JPARAM(HADOOP_CONF))); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsOpenFile(%s): FileSystem#getConf", path); + goto done; + } + jConfiguration = jVal.l; + + jStrBufferSize = (*env)->NewStringUTF(env, "io.file.buffer.size"); + if (!jStrBufferSize) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, "OOM"); + goto done; + } + jStrReplication = (*env)->NewStringUTF(env, "dfs.replication"); + if (!jStrReplication) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, "OOM"); + goto done; + } + + if (!bufferSize) { + jthr = invokeMethod(env, &jVal, INSTANCE, jConfiguration, + JC_CONFIGURATION, "getInt", + "(Ljava/lang/String;I)I", jStrBufferSize, 4096); + if (jthr) { + ret = printExceptionAndFree(env, jthr, NOPRINT_EXC_FILE_NOT_FOUND | + NOPRINT_EXC_ACCESS_CONTROL | NOPRINT_EXC_UNRESOLVED_LINK, + "hdfsOpenFile(%s): Configuration#getInt(io.file.buffer.size)", + path); + goto done; + } + jBufferSize = jVal.i; + } + + if ((accmode == O_WRONLY) && (flags & O_APPEND) == 0) { + if (!replication) { + jthr = invokeMethod(env, &jVal, INSTANCE, jConfiguration, + JC_CONFIGURATION, "getInt", + "(Ljava/lang/String;I)I", jStrReplication, 1); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsOpenFile(%s): Configuration#getInt(dfs.replication)", + path); + goto done; + } + jReplication = (jshort)jVal.i; + } + } + + /* Create and return either the FSDataInputStream or + FSDataOutputStream references jobject jStream */ + + // READ? + if (accmode == O_RDONLY) { + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, JC_FILE_SYSTEM, + method, signature, jPath, jBufferSize); + } else if ((accmode == O_WRONLY) && (flags & O_APPEND)) { + // WRITE/APPEND? + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, JC_FILE_SYSTEM, + method, signature, jPath); + } else { + // WRITE/CREATE + jboolean jOverWrite = 1; + jlong jBlockSize = blockSize; + + if (jBlockSize == 0) { + jthr = getDefaultBlockSize(env, jFS, jPath, &jBlockSize); + if (jthr) { + ret = EIO; + goto done; + } + } + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, JC_FILE_SYSTEM, + method, signature, jPath, jOverWrite, jBufferSize, + jReplication, jBlockSize); + } + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsOpenFile(%s): FileSystem#%s(%s)", path, method, signature); + goto done; + } + jFile = jVal.l; + + file = calloc(1, sizeof(struct hdfsFile_internal)); + if (!file) { + fprintf(stderr, "hdfsOpenFile(%s): OOM create hdfsFile\n", path); + ret = ENOMEM; + goto done; + } + file->file = (*env)->NewGlobalRef(env, jFile); + if (!file->file) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsOpenFile(%s): NewGlobalRef", path); + goto done; + } + file->type = (((flags & O_WRONLY) == 0) ? HDFS_STREAM_INPUT : + HDFS_STREAM_OUTPUT); + file->flags = 0; + + if ((flags & O_WRONLY) == 0) { + // Check the StreamCapabilities of jFile to see if we can do direct + // reads + if (hdfsHasStreamCapability(jFile, "in:readbytebuffer")) { + file->flags |= HDFS_FILE_SUPPORTS_DIRECT_READ; + } + + // Check the StreamCapabilities of jFile to see if we can do direct + // preads + if (hdfsHasStreamCapability(jFile, "in:preadbytebuffer")) { + file->flags |= HDFS_FILE_SUPPORTS_DIRECT_PREAD; + } + } + ret = 0; + +done: + destroyLocalReference(env, jStrBufferSize); + destroyLocalReference(env, jStrReplication); + destroyLocalReference(env, jConfiguration); + destroyLocalReference(env, jPath); + destroyLocalReference(env, jFile); + if (ret) { + if (file) { + if (file->file) { + (*env)->DeleteGlobalRef(env, file->file); + } + free(file); + } + errno = ret; + return NULL; + } + return file; +} + +hdfsFile hdfsStreamBuilderBuild(struct hdfsStreamBuilder *bld) +{ + hdfsFile file = hdfsOpenFileImpl(bld->fs, bld->path, bld->flags, + bld->bufferSize, bld->replication, bld->defaultBlockSize); + int prevErrno = errno; + hdfsStreamBuilderFree(bld); + errno = prevErrno; + return file; +} + +int hdfsTruncateFile(hdfsFS fs, const char* path, tOffset newlength) +{ + jobject jFS = (jobject)fs; + jthrowable jthr; + jvalue jVal; + jobject jPath = NULL; + + JNIEnv *env = getJNIEnv(); + + if (!env) { + errno = EINTERNAL; + return -1; + } + + /* Create an object of org.apache.hadoop.fs.Path */ + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsTruncateFile(%s): constructNewObjectOfPath", path); + return -1; + } + + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, JC_FILE_SYSTEM, + "truncate", JMETHOD2(JPARAM(HADOOP_PATH), "J", "Z"), + jPath, newlength); + destroyLocalReference(env, jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsTruncateFile(%s): FileSystem#truncate", path); + return -1; + } + if (jVal.z == JNI_TRUE) { + return 1; + } + return 0; +} + +int hdfsUnbufferFile(hdfsFile file) +{ + int ret; + jthrowable jthr; + JNIEnv *env = getJNIEnv(); + + if (!env) { + ret = EINTERNAL; + goto done; + } + if (file->type != HDFS_STREAM_INPUT) { + ret = ENOTSUP; + goto done; + } + jthr = invokeMethod(env, NULL, INSTANCE, file->file, + JC_FS_DATA_INPUT_STREAM, "unbuffer", "()V"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + HADOOP_FSDISTRM "#unbuffer failed:"); + goto done; + } + ret = 0; + +done: + errno = ret; + return ret; +} + +int hdfsCloseFile(hdfsFS fs, hdfsFile file) +{ + int ret; + // JAVA EQUIVALENT: + // file.close + + //The interface whose 'close' method to be called + CachedJavaClass cachedJavaClass; + const char *interfaceShortName; + + //Caught exception + jthrowable jthr; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Sanity check + if (!file || file->type == HDFS_STREAM_UNINITIALIZED) { + errno = EBADF; + return -1; + } + + if (file->type == HDFS_STREAM_INPUT) { + cachedJavaClass = JC_FS_DATA_INPUT_STREAM; + } else { + cachedJavaClass = JC_FS_DATA_OUTPUT_STREAM; + } + + jthr = invokeMethod(env, NULL, INSTANCE, file->file, + cachedJavaClass, "close", "()V"); + if (jthr) { + interfaceShortName = (file->type == HDFS_STREAM_INPUT) ? + "FSDataInputStream" : "FSDataOutputStream"; + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "%s#close", interfaceShortName); + } else { + ret = 0; + } + + //De-allocate memory + (*env)->DeleteGlobalRef(env, file->file); + free(file); + + if (ret) { + errno = ret; + return -1; + } + return 0; +} + +int hdfsExists(hdfsFS fs, const char *path) +{ + JNIEnv *env = getJNIEnv(); + jobject jPath; + jvalue jVal; + jobject jFS = (jobject)fs; + jthrowable jthr; + + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + if (path == NULL) { + errno = EINVAL; + return -1; + } + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsExists: constructNewObjectOfPath"); + return -1; + } + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, JC_FILE_SYSTEM, + "exists", JMETHOD1(JPARAM(HADOOP_PATH), "Z"), jPath); + destroyLocalReference(env, jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsExists: invokeMethod(%s)", + JMETHOD1(JPARAM(HADOOP_PATH), "Z")); + return -1; + } + if (jVal.z) { + return 0; + } else { + errno = ENOENT; + return -1; + } +} + +// Checks input file for readiness for reading. +static int readPrepare(JNIEnv* env, hdfsFS fs, hdfsFile f, + jobject* jInputStream) +{ + *jInputStream = (jobject)(f ? f->file : NULL); + + //Sanity check + if (!f || f->type == HDFS_STREAM_UNINITIALIZED) { + errno = EBADF; + return -1; + } + + //Error checking... make sure that this file is 'readable' + if (f->type != HDFS_STREAM_INPUT) { + fprintf(stderr, "Cannot read from a non-InputStream object!\n"); + errno = EINVAL; + return -1; + } + + return 0; +} + +/** + * If the underlying stream supports the ByteBufferReadable interface then + * this method will transparently use read(ByteBuffer). This can help + * improve performance as it avoids unnecessarily copying data on to the Java + * heap. Instead the data will be directly copied from kernel space to the C + * heap. + */ +tSize hdfsRead(hdfsFS fs, hdfsFile f, void* buffer, tSize length) +{ + jobject jInputStream; + jbyteArray jbRarray; + jvalue jVal; + jthrowable jthr; + JNIEnv* env; + + if (length == 0) { + return 0; + } else if (length < 0) { + errno = EINVAL; + return -1; + } + if (f->flags & HDFS_FILE_SUPPORTS_DIRECT_READ) { + return readDirect(fs, f, buffer, length); + } + + // JAVA EQUIVALENT: + // byte [] bR = new byte[length]; + // fis.read(bR); + + //Get the JNIEnv* corresponding to current thread + env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Parameters + if (readPrepare(env, fs, f, &jInputStream) == -1) { + return -1; + } + + //Read the requisite bytes + jbRarray = (*env)->NewByteArray(env, length); + if (!jbRarray) { + errno = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsRead: NewByteArray"); + return -1; + } + + jthr = invokeMethod(env, &jVal, INSTANCE, jInputStream, + JC_FS_DATA_INPUT_STREAM, "read", "([B)I", jbRarray); + if (jthr) { + destroyLocalReference(env, jbRarray); + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsRead: FSDataInputStream#read"); + return -1; + } + if (jVal.i < 0) { + // EOF + destroyLocalReference(env, jbRarray); + return 0; + } else if (jVal.i == 0) { + destroyLocalReference(env, jbRarray); + errno = EINTR; + return -1; + } + // We only copy the portion of the jbRarray that was actually filled by + // the call to FsDataInputStream#read; #read is not guaranteed to fill the + // entire buffer, instead it returns the number of bytes read into the + // buffer; we use the return value as the input in GetByteArrayRegion to + // ensure don't copy more bytes than necessary + (*env)->GetByteArrayRegion(env, jbRarray, 0, jVal.i, buffer); + destroyLocalReference(env, jbRarray); + if ((*env)->ExceptionCheck(env)) { + errno = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsRead: GetByteArrayRegion"); + return -1; + } + return jVal.i; +} + +tSize readDirect(hdfsFS fs, hdfsFile f, void* buffer, tSize length) +{ + // JAVA EQUIVALENT: + // ByteBuffer buf = ByteBuffer.allocateDirect(length) // wraps C buffer + // fis.read(buf); + + jobject jInputStream; + jvalue jVal; + jthrowable jthr; + jobject bb; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + if (readPrepare(env, fs, f, &jInputStream) == -1) { + return -1; + } + + //Read the requisite bytes + bb = (*env)->NewDirectByteBuffer(env, buffer, length); + if (bb == NULL) { + errno = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "readDirect: NewDirectByteBuffer"); + return -1; + } + + jthr = invokeMethod(env, &jVal, INSTANCE, jInputStream, + JC_FS_DATA_INPUT_STREAM, "read", + "(Ljava/nio/ByteBuffer;)I", bb); + destroyLocalReference(env, bb); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "readDirect: FSDataInputStream#read"); + return -1; + } + // Reached EOF, return 0 + if (jVal.i < 0) { + return 0; + } + // 0 bytes read, return error + if (jVal.i == 0) { + errno = EINTR; + return -1; + } + return jVal.i; +} + +/** + * If the underlying stream supports the ByteBufferPositionedReadable + * interface then this method will transparently use read(long, ByteBuffer). + * This can help improve performance as it avoids unnecessarily copying data + * on to the Java heap. Instead the data will be directly copied from kernel + * space to the C heap. + */ +tSize hdfsPread(hdfsFS fs, hdfsFile f, tOffset position, + void* buffer, tSize length) +{ + JNIEnv* env; + jbyteArray jbRarray; + jvalue jVal; + jthrowable jthr; + + if (length == 0) { + return 0; + } else if (length < 0) { + errno = EINVAL; + return -1; + } + if (!f || f->type == HDFS_STREAM_UNINITIALIZED) { + errno = EBADF; + return -1; + } + + if (f->flags & HDFS_FILE_SUPPORTS_DIRECT_PREAD) { + return preadDirect(fs, f, position, buffer, length); + } + + env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Error checking... make sure that this file is 'readable' + if (f->type != HDFS_STREAM_INPUT) { + fprintf(stderr, "Cannot read from a non-InputStream object!\n"); + errno = EINVAL; + return -1; + } + + // JAVA EQUIVALENT: + // byte [] bR = new byte[length]; + // fis.read(pos, bR, 0, length); + jbRarray = (*env)->NewByteArray(env, length); + if (!jbRarray) { + errno = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsPread: NewByteArray"); + return -1; + } + + jthr = invokeMethod(env, &jVal, INSTANCE, f->file, + JC_FS_DATA_INPUT_STREAM, "read", "(J[BII)I", position, + jbRarray, 0, length); + if (jthr) { + destroyLocalReference(env, jbRarray); + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsPread: FSDataInputStream#read"); + return -1; + } + if (jVal.i < 0) { + // EOF + destroyLocalReference(env, jbRarray); + return 0; + } else if (jVal.i == 0) { + destroyLocalReference(env, jbRarray); + errno = EINTR; + return -1; + } + (*env)->GetByteArrayRegion(env, jbRarray, 0, jVal.i, buffer); + destroyLocalReference(env, jbRarray); + if ((*env)->ExceptionCheck(env)) { + errno = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsPread: GetByteArrayRegion"); + return -1; + } + return jVal.i; +} + +tSize preadDirect(hdfsFS fs, hdfsFile f, tOffset position, void* buffer, + tSize length) +{ + // JAVA EQUIVALENT: + // ByteBuffer buf = ByteBuffer.allocateDirect(length) // wraps C buffer + // fis.read(position, buf); + + jvalue jVal; + jthrowable jthr; + jobject bb; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Error checking... make sure that this file is 'readable' + if (f->type != HDFS_STREAM_INPUT) { + fprintf(stderr, "Cannot read from a non-InputStream object!\n"); + errno = EINVAL; + return -1; + } + + //Read the requisite bytes + bb = (*env)->NewDirectByteBuffer(env, buffer, length); + if (bb == NULL) { + errno = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "readDirect: NewDirectByteBuffer"); + return -1; + } + + jthr = invokeMethod(env, &jVal, INSTANCE, f->file, + JC_FS_DATA_INPUT_STREAM, "read", "(JLjava/nio/ByteBuffer;)I", + position, bb); + destroyLocalReference(env, bb); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "preadDirect: FSDataInputStream#read"); + return -1; + } + // Reached EOF, return 0 + if (jVal.i < 0) { + return 0; + } + // 0 bytes read, return error + if (jVal.i == 0) { + errno = EINTR; + return -1; + } + return jVal.i; +} + +/** + * Like hdfsPread, if the underlying stream supports the + * ByteBufferPositionedReadable interface then this method will transparently + * use readFully(long, ByteBuffer). + */ +int hdfsPreadFully(hdfsFS fs, hdfsFile f, tOffset position, + void* buffer, tSize length) { + JNIEnv* env; + jbyteArray jbRarray; + jthrowable jthr; + + if (length == 0) { + return 0; + } else if (length < 0) { + errno = EINVAL; + return -1; + } + if (!f || f->type == HDFS_STREAM_UNINITIALIZED) { + errno = EBADF; + return -1; + } + + if (f->flags & HDFS_FILE_SUPPORTS_DIRECT_PREAD) { + return preadFullyDirect(fs, f, position, buffer, length); + } + + env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Error checking... make sure that this file is 'readable' + if (f->type != HDFS_STREAM_INPUT) { + fprintf(stderr, "Cannot read from a non-InputStream object!\n"); + errno = EINVAL; + return -1; + } + + // JAVA EQUIVALENT: + // byte [] bR = new byte[length]; + // fis.read(pos, bR, 0, length); + jbRarray = (*env)->NewByteArray(env, length); + if (!jbRarray) { + errno = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsPread: NewByteArray"); + return -1; + } + + jthr = invokeMethod(env, NULL, INSTANCE, f->file, + JC_FS_DATA_INPUT_STREAM, "readFully", "(J[BII)V", + position, jbRarray, 0, length); + if (jthr) { + destroyLocalReference(env, jbRarray); + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsPread: FSDataInputStream#read"); + return -1; + } + + (*env)->GetByteArrayRegion(env, jbRarray, 0, length, buffer); + destroyLocalReference(env, jbRarray); + if ((*env)->ExceptionCheck(env)) { + errno = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsPread: GetByteArrayRegion"); + return -1; + } + return 0; +} + +int preadFullyDirect(hdfsFS fs, hdfsFile f, tOffset position, void* buffer, + tSize length) +{ + // JAVA EQUIVALENT: + // ByteBuffer buf = ByteBuffer.allocateDirect(length) // wraps C buffer + // fis.read(position, buf); + + jthrowable jthr; + jobject bb; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Error checking... make sure that this file is 'readable' + if (f->type != HDFS_STREAM_INPUT) { + fprintf(stderr, "Cannot read from a non-InputStream object!\n"); + errno = EINVAL; + return -1; + } + + //Read the requisite bytes + bb = (*env)->NewDirectByteBuffer(env, buffer, length); + if (bb == NULL) { + errno = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "readDirect: NewDirectByteBuffer"); + return -1; + } + + jthr = invokeMethod(env, NULL, INSTANCE, f->file, + JC_FS_DATA_INPUT_STREAM, "readFully", + "(JLjava/nio/ByteBuffer;)V", position, bb); + destroyLocalReference(env, bb); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "preadDirect: FSDataInputStream#read"); + return -1; + } + return 0; +} + +tSize hdfsWrite(hdfsFS fs, hdfsFile f, const void* buffer, tSize length) +{ + // JAVA EQUIVALENT + // byte b[] = str.getBytes(); + // fso.write(b); + + jobject jOutputStream; + jbyteArray jbWarray; + jthrowable jthr; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Sanity check + if (!f || f->type == HDFS_STREAM_UNINITIALIZED) { + errno = EBADF; + return -1; + } + + jOutputStream = f->file; + + if (length < 0) { + errno = EINVAL; + return -1; + } + + //Error checking... make sure that this file is 'writable' + if (f->type != HDFS_STREAM_OUTPUT) { + fprintf(stderr, "Cannot write into a non-OutputStream object!\n"); + errno = EINVAL; + return -1; + } + + if (length < 0) { + errno = EINVAL; + return -1; + } + if (length == 0) { + return 0; + } + //Write the requisite bytes into the file + jbWarray = (*env)->NewByteArray(env, length); + if (!jbWarray) { + errno = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsWrite: NewByteArray"); + return -1; + } + (*env)->SetByteArrayRegion(env, jbWarray, 0, length, buffer); + if ((*env)->ExceptionCheck(env)) { + destroyLocalReference(env, jbWarray); + errno = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsWrite(length = %d): SetByteArrayRegion", length); + return -1; + } + jthr = invokeMethod(env, NULL, INSTANCE, jOutputStream, + JC_FS_DATA_OUTPUT_STREAM, "write", "([B)V", + jbWarray); + destroyLocalReference(env, jbWarray); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsWrite: FSDataOutputStream#write"); + return -1; + } + // Unlike most Java streams, FSDataOutputStream never does partial writes. + // If we succeeded, all the data was written. + return length; +} + +int hdfsSeek(hdfsFS fs, hdfsFile f, tOffset desiredPos) +{ + // JAVA EQUIVALENT + // fis.seek(pos); + + jobject jInputStream; + jthrowable jthr; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Sanity check + if (!f || f->type != HDFS_STREAM_INPUT) { + errno = EBADF; + return -1; + } + + jInputStream = f->file; + jthr = invokeMethod(env, NULL, INSTANCE, jInputStream, + JC_FS_DATA_INPUT_STREAM, "seek", "(J)V", desiredPos); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsSeek(desiredPos=%" PRId64 ")" + ": FSDataInputStream#seek", desiredPos); + return -1; + } + return 0; +} + +tOffset hdfsTell(hdfsFS fs, hdfsFile f) +{ + // JAVA EQUIVALENT + // pos = f.getPos(); + + jobject jStream; + CachedJavaClass cachedJavaClass; + jvalue jVal; + jthrowable jthr; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Sanity check + if (!f || f->type == HDFS_STREAM_UNINITIALIZED) { + errno = EBADF; + return -1; + } + + //Parameters + jStream = f->file; + if (f->type == HDFS_STREAM_INPUT) { + cachedJavaClass = JC_FS_DATA_INPUT_STREAM; + } else { + cachedJavaClass = JC_FS_DATA_OUTPUT_STREAM; + } + jthr = invokeMethod(env, &jVal, INSTANCE, jStream, + cachedJavaClass, "getPos", "()J"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsTell: %s#getPos", + ((f->type == HDFS_STREAM_INPUT) ? "FSDataInputStream" : + "FSDataOutputStream")); + return -1; + } + return jVal.j; +} + +int hdfsFlush(hdfsFS fs, hdfsFile f) +{ + // JAVA EQUIVALENT + // fos.flush(); + + jthrowable jthr; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Sanity check + if (!f || f->type != HDFS_STREAM_OUTPUT) { + errno = EBADF; + return -1; + } + jthr = invokeMethod(env, NULL, INSTANCE, f->file, + JC_FS_DATA_OUTPUT_STREAM, "flush", "()V"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsFlush: FSDataInputStream#flush"); + return -1; + } + return 0; +} + +int hdfsHFlush(hdfsFS fs, hdfsFile f) +{ + jobject jOutputStream; + jthrowable jthr; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Sanity check + if (!f || f->type != HDFS_STREAM_OUTPUT) { + errno = EBADF; + return -1; + } + + jOutputStream = f->file; + jthr = invokeMethod(env, NULL, INSTANCE, jOutputStream, + JC_FS_DATA_OUTPUT_STREAM, "hflush", "()V"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsHFlush: FSDataOutputStream#hflush"); + return -1; + } + return 0; +} + +int hdfsHSync(hdfsFS fs, hdfsFile f) +{ + jobject jOutputStream; + jthrowable jthr; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Sanity check + if (!f || f->type != HDFS_STREAM_OUTPUT) { + errno = EBADF; + return -1; + } + + jOutputStream = f->file; + jthr = invokeMethod(env, NULL, INSTANCE, jOutputStream, + JC_FS_DATA_OUTPUT_STREAM, "hsync", "()V"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsHSync: FSDataOutputStream#hsync"); + return -1; + } + return 0; +} + +int hdfsAvailable(hdfsFS fs, hdfsFile f) +{ + // JAVA EQUIVALENT + // fis.available(); + + jobject jInputStream; + jvalue jVal; + jthrowable jthr; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Sanity check + if (!f || f->type != HDFS_STREAM_INPUT) { + errno = EBADF; + return -1; + } + + //Parameters + jInputStream = f->file; + jthr = invokeMethod(env, &jVal, INSTANCE, jInputStream, + JC_FS_DATA_INPUT_STREAM, "available", "()I"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsAvailable: FSDataInputStream#available"); + return -1; + } + return jVal.i; +} + +static int hdfsCopyImpl(hdfsFS srcFS, const char *src, hdfsFS dstFS, + const char *dst, jboolean deleteSource) +{ + //JAVA EQUIVALENT + // FileUtil#copy(srcFS, srcPath, dstFS, dstPath, + // deleteSource = false, conf) + + //Parameters + jobject jSrcFS = (jobject)srcFS; + jobject jDstFS = (jobject)dstFS; + jobject jConfiguration = NULL, jSrcPath = NULL, jDstPath = NULL; + jthrowable jthr; + jvalue jVal; + int ret; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + jthr = constructNewObjectOfPath(env, src, &jSrcPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsCopyImpl(src=%s): constructNewObjectOfPath", src); + goto done; + } + jthr = constructNewObjectOfPath(env, dst, &jDstPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsCopyImpl(dst=%s): constructNewObjectOfPath", dst); + goto done; + } + + //Create the org.apache.hadoop.conf.Configuration object + jthr = constructNewObjectOfCachedClass(env, &jConfiguration, + JC_CONFIGURATION, "()V"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsCopyImpl: Configuration constructor"); + goto done; + } + + //FileUtil#copy + jthr = invokeMethod(env, &jVal, STATIC, NULL, JC_FILE_UTIL, + "copy", + "(Lorg/apache/hadoop/fs/FileSystem;Lorg/apache/hadoop/fs/Path;" + "Lorg/apache/hadoop/fs/FileSystem;Lorg/apache/hadoop/fs/Path;" + "ZLorg/apache/hadoop/conf/Configuration;)Z", + jSrcFS, jSrcPath, jDstFS, jDstPath, deleteSource, + jConfiguration); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsCopyImpl(src=%s, dst=%s, deleteSource=%d): " + "FileUtil#copy", src, dst, deleteSource); + goto done; + } + if (!jVal.z) { + ret = EIO; + goto done; + } + ret = 0; + +done: + destroyLocalReference(env, jConfiguration); + destroyLocalReference(env, jSrcPath); + destroyLocalReference(env, jDstPath); + + if (ret) { + errno = ret; + return -1; + } + return 0; +} + +int hdfsCopy(hdfsFS srcFS, const char *src, hdfsFS dstFS, const char *dst) +{ + return hdfsCopyImpl(srcFS, src, dstFS, dst, 0); +} + +int hdfsMove(hdfsFS srcFS, const char *src, hdfsFS dstFS, const char *dst) +{ + return hdfsCopyImpl(srcFS, src, dstFS, dst, 1); +} + +int hdfsDelete(hdfsFS fs, const char *path, int recursive) +{ + // JAVA EQUIVALENT: + // Path p = new Path(path); + // bool retval = fs.delete(p, recursive); + + jobject jFS = (jobject)fs; + jthrowable jthr; + jobject jPath; + jvalue jVal; + jboolean jRecursive; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsDelete(path=%s): constructNewObjectOfPath", path); + return -1; + } + jRecursive = recursive ? JNI_TRUE : JNI_FALSE; + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, JC_FILE_SYSTEM, + "delete", "(Lorg/apache/hadoop/fs/Path;Z)Z", jPath, + jRecursive); + destroyLocalReference(env, jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsDelete(path=%s, recursive=%d): " + "FileSystem#delete", path, recursive); + return -1; + } + if (!jVal.z) { + errno = EIO; + return -1; + } + return 0; +} + + + +int hdfsRename(hdfsFS fs, const char *oldPath, const char *newPath) +{ + // JAVA EQUIVALENT: + // Path old = new Path(oldPath); + // Path new = new Path(newPath); + // fs.rename(old, new); + + jobject jFS = (jobject)fs; + jthrowable jthr; + jobject jOldPath = NULL, jNewPath = NULL; + int ret = -1; + jvalue jVal; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + jthr = constructNewObjectOfPath(env, oldPath, &jOldPath ); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsRename: constructNewObjectOfPath(%s)", oldPath); + goto done; + } + jthr = constructNewObjectOfPath(env, newPath, &jNewPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsRename: constructNewObjectOfPath(%s)", newPath); + goto done; + } + + // Rename the file + // TODO: use rename2 here? (See HDFS-3592) + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, JC_FILE_SYSTEM, + "rename", JMETHOD2(JPARAM(HADOOP_PATH), JPARAM + (HADOOP_PATH), "Z"), jOldPath, jNewPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsRename(oldPath=%s, newPath=%s): FileSystem#rename", + oldPath, newPath); + goto done; + } + if (!jVal.z) { + errno = EIO; + goto done; + } + ret = 0; + +done: + destroyLocalReference(env, jOldPath); + destroyLocalReference(env, jNewPath); + return ret; +} + + + +char* hdfsGetWorkingDirectory(hdfsFS fs, char* buffer, size_t bufferSize) +{ + // JAVA EQUIVALENT: + // Path p = fs.getWorkingDirectory(); + // return p.toString() + + jobject jPath = NULL; + jstring jPathString = NULL; + jobject jFS = (jobject)fs; + jvalue jVal; + jthrowable jthr; + int ret; + const char *jPathChars = NULL; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return NULL; + } + + //FileSystem#getWorkingDirectory() + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, JC_FILE_SYSTEM, + "getWorkingDirectory", "()Lorg/apache/hadoop/fs/Path;"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetWorkingDirectory: FileSystem#getWorkingDirectory"); + goto done; + } + jPath = jVal.l; + if (!jPath) { + fprintf(stderr, "hdfsGetWorkingDirectory: " + "FileSystem#getWorkingDirectory returned NULL"); + ret = -EIO; + goto done; + } + + //Path#toString() + jthr = invokeMethod(env, &jVal, INSTANCE, jPath, JC_PATH, "toString", + "()Ljava/lang/String;"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetWorkingDirectory: Path#toString"); + goto done; + } + jPathString = jVal.l; + jPathChars = (*env)->GetStringUTFChars(env, jPathString, NULL); + if (!jPathChars) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsGetWorkingDirectory: GetStringUTFChars"); + goto done; + } + + //Copy to user-provided buffer + ret = snprintf(buffer, bufferSize, "%s", jPathChars); + if (ret >= bufferSize) { + ret = ENAMETOOLONG; + goto done; + } + ret = 0; + +done: + if (jPathChars) { + (*env)->ReleaseStringUTFChars(env, jPathString, jPathChars); + } + destroyLocalReference(env, jPath); + destroyLocalReference(env, jPathString); + + if (ret) { + errno = ret; + return NULL; + } + return buffer; +} + + + +int hdfsSetWorkingDirectory(hdfsFS fs, const char *path) +{ + // JAVA EQUIVALENT: + // fs.setWorkingDirectory(Path(path)); + + jobject jFS = (jobject)fs; + jthrowable jthr; + jobject jPath; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Create an object of org.apache.hadoop.fs.Path + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsSetWorkingDirectory(%s): constructNewObjectOfPath", + path); + return -1; + } + + //FileSystem#setWorkingDirectory() + jthr = invokeMethod(env, NULL, INSTANCE, jFS, JC_FILE_SYSTEM, + "setWorkingDirectory", "(Lorg/apache/hadoop/fs/Path;)V", + jPath); + destroyLocalReference(env, jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, NOPRINT_EXC_ILLEGAL_ARGUMENT, + "hdfsSetWorkingDirectory(%s): FileSystem#setWorkingDirectory", + path); + return -1; + } + return 0; +} + + + +int hdfsCreateDirectory(hdfsFS fs, const char *path) +{ + // JAVA EQUIVALENT: + // fs.mkdirs(new Path(path)); + + jobject jFS = (jobject)fs; + jobject jPath; + jthrowable jthr; + jvalue jVal; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Create an object of org.apache.hadoop.fs.Path + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsCreateDirectory(%s): constructNewObjectOfPath", path); + return -1; + } + + //Create the directory + jVal.z = 0; + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, JC_FILE_SYSTEM, + "mkdirs", "(Lorg/apache/hadoop/fs/Path;)Z", jPath); + destroyLocalReference(env, jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, + NOPRINT_EXC_ACCESS_CONTROL | NOPRINT_EXC_FILE_NOT_FOUND | + NOPRINT_EXC_UNRESOLVED_LINK | NOPRINT_EXC_PARENT_NOT_DIRECTORY, + "hdfsCreateDirectory(%s): FileSystem#mkdirs", path); + return -1; + } + if (!jVal.z) { + // It's unclear under exactly which conditions FileSystem#mkdirs + // is supposed to return false (as opposed to throwing an exception.) + // It seems like the current code never actually returns false. + // So we're going to translate this to EIO, since there seems to be + // nothing more specific we can do with it. + errno = EIO; + return -1; + } + return 0; +} + + +int hdfsSetReplication(hdfsFS fs, const char *path, int16_t replication) +{ + // JAVA EQUIVALENT: + // fs.setReplication(new Path(path), replication); + + jobject jFS = (jobject)fs; + jthrowable jthr; + jobject jPath; + jvalue jVal; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Create an object of org.apache.hadoop.fs.Path + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsSetReplication(path=%s): constructNewObjectOfPath", path); + return -1; + } + + //Create the directory + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, JC_FILE_SYSTEM, + "setReplication", "(Lorg/apache/hadoop/fs/Path;S)Z", + jPath, replication); + destroyLocalReference(env, jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsSetReplication(path=%s, replication=%d): " + "FileSystem#setReplication", path, replication); + return -1; + } + if (!jVal.z) { + // setReplication returns false "if file does not exist or is a + // directory." So the nearest translation to that is ENOENT. + errno = ENOENT; + return -1; + } + + return 0; +} + +int hdfsChown(hdfsFS fs, const char *path, const char *owner, const char *group) +{ + // JAVA EQUIVALENT: + // fs.setOwner(path, owner, group) + + jobject jFS = (jobject)fs; + jobject jPath = NULL; + jstring jOwner = NULL, jGroup = NULL; + jthrowable jthr; + int ret; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + if (owner == NULL && group == NULL) { + return 0; + } + + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsChown(path=%s): constructNewObjectOfPath", path); + goto done; + } + + jthr = newJavaStr(env, owner, &jOwner); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsChown(path=%s): newJavaStr(%s)", path, owner); + goto done; + } + jthr = newJavaStr(env, group, &jGroup); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsChown(path=%s): newJavaStr(%s)", path, group); + goto done; + } + + //Create the directory + jthr = invokeMethod(env, NULL, INSTANCE, jFS, JC_FILE_SYSTEM, + "setOwner", JMETHOD3(JPARAM(HADOOP_PATH), + JPARAM(JAVA_STRING), JPARAM(JAVA_STRING), JAVA_VOID), + jPath, jOwner, jGroup); + if (jthr) { + ret = printExceptionAndFree(env, jthr, + NOPRINT_EXC_ACCESS_CONTROL | NOPRINT_EXC_FILE_NOT_FOUND | + NOPRINT_EXC_UNRESOLVED_LINK, + "hdfsChown(path=%s, owner=%s, group=%s): " + "FileSystem#setOwner", path, owner, group); + goto done; + } + ret = 0; + +done: + destroyLocalReference(env, jPath); + destroyLocalReference(env, jOwner); + destroyLocalReference(env, jGroup); + + if (ret) { + errno = ret; + return -1; + } + return 0; +} + +int hdfsChmod(hdfsFS fs, const char *path, short mode) +{ + int ret; + // JAVA EQUIVALENT: + // fs.setPermission(path, FsPermission) + + jthrowable jthr; + jobject jPath = NULL, jPermObj = NULL; + jobject jFS = (jobject)fs; + jshort jmode = mode; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + // construct jPerm = FsPermission.createImmutable(short mode); + jthr = constructNewObjectOfCachedClass(env, &jPermObj, JC_FS_PERMISSION, + "(S)V",jmode); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "constructNewObjectOfCachedClass(%s)", HADOOP_FSPERM); + goto done; + } + + //Create an object of org.apache.hadoop.fs.Path + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsChmod(%s): constructNewObjectOfPath", path); + goto done; + } + + //Create the directory + jthr = invokeMethod(env, NULL, INSTANCE, jFS, JC_FILE_SYSTEM, + "setPermission", JMETHOD2(JPARAM(HADOOP_PATH), + JPARAM(HADOOP_FSPERM), JAVA_VOID), jPath, jPermObj); + if (jthr) { + ret = printExceptionAndFree(env, jthr, + NOPRINT_EXC_ACCESS_CONTROL | NOPRINT_EXC_FILE_NOT_FOUND | + NOPRINT_EXC_UNRESOLVED_LINK, + "hdfsChmod(%s): FileSystem#setPermission", path); + goto done; + } + ret = 0; + +done: + destroyLocalReference(env, jPath); + destroyLocalReference(env, jPermObj); + + if (ret) { + errno = ret; + return -1; + } + return 0; +} + +int hdfsUtime(hdfsFS fs, const char *path, tTime mtime, tTime atime) +{ + // JAVA EQUIVALENT: + // fs.setTimes(src, mtime, atime) + + jthrowable jthr; + jobject jFS = (jobject)fs; + jobject jPath; + static const tTime NO_CHANGE = -1; + jlong jmtime, jatime; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Create an object of org.apache.hadoop.fs.Path + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsUtime(path=%s): constructNewObjectOfPath", path); + return -1; + } + + jmtime = (mtime == NO_CHANGE) ? -1 : (mtime * (jlong)1000); + jatime = (atime == NO_CHANGE) ? -1 : (atime * (jlong)1000); + + jthr = invokeMethod(env, NULL, INSTANCE, jFS, JC_FILE_SYSTEM, + "setTimes", JMETHOD3(JPARAM(HADOOP_PATH), "J", "J", + JAVA_VOID), jPath, jmtime, jatime); + destroyLocalReference(env, jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, + NOPRINT_EXC_ACCESS_CONTROL | NOPRINT_EXC_FILE_NOT_FOUND | + NOPRINT_EXC_UNRESOLVED_LINK, + "hdfsUtime(path=%s): FileSystem#setTimes", path); + return -1; + } + return 0; +} + +/** + * Zero-copy options. + * + * We cache the EnumSet of ReadOptions which has to be passed into every + * readZero call, to avoid reconstructing it each time. This cache is cleared + * whenever an element changes. + */ +struct hadoopRzOptions +{ + JNIEnv *env; + int skipChecksums; + jobject byteBufferPool; + jobject cachedEnumSet; +}; + +struct hadoopRzOptions *hadoopRzOptionsAlloc(void) +{ + struct hadoopRzOptions *opts; + JNIEnv *env; + + env = getJNIEnv(); + if (!env) { + // Check to make sure the JNI environment is set up properly. + errno = EINTERNAL; + return NULL; + } + opts = calloc(1, sizeof(struct hadoopRzOptions)); + if (!opts) { + errno = ENOMEM; + return NULL; + } + return opts; +} + +static void hadoopRzOptionsClearCached(JNIEnv *env, + struct hadoopRzOptions *opts) +{ + if (!opts->cachedEnumSet) { + return; + } + (*env)->DeleteGlobalRef(env, opts->cachedEnumSet); + opts->cachedEnumSet = NULL; +} + +int hadoopRzOptionsSetSkipChecksum( + struct hadoopRzOptions *opts, int skip) +{ + JNIEnv *env; + env = getJNIEnv(); + if (!env) { + errno = EINTERNAL; + return -1; + } + hadoopRzOptionsClearCached(env, opts); + opts->skipChecksums = !!skip; + return 0; +} + +int hadoopRzOptionsSetByteBufferPool( + struct hadoopRzOptions *opts, const char *className) +{ + JNIEnv *env; + jthrowable jthr; + jobject byteBufferPool = NULL; + jobject globalByteBufferPool = NULL; + int ret; + + env = getJNIEnv(); + if (!env) { + errno = EINTERNAL; + return -1; + } + + if (className) { + // Note: we don't have to call hadoopRzOptionsClearCached in this + // function, since the ByteBufferPool is passed separately from the + // EnumSet of ReadOptions. + + jthr = constructNewObjectOfClass(env, &byteBufferPool, className, "()V"); + if (jthr) { + printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hadoopRzOptionsSetByteBufferPool(className=%s): ", className); + ret = EINVAL; + goto done; + } + // Only set opts->byteBufferPool if creating a global reference is + // successful + globalByteBufferPool = (*env)->NewGlobalRef(env, byteBufferPool); + if (!globalByteBufferPool) { + printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hadoopRzOptionsSetByteBufferPool(className=%s): ", + className); + ret = EINVAL; + goto done; + } + // Delete any previous ByteBufferPool we had before setting a new one. + if (opts->byteBufferPool) { + (*env)->DeleteGlobalRef(env, opts->byteBufferPool); + } + opts->byteBufferPool = globalByteBufferPool; + } else if (opts->byteBufferPool) { + // If the specified className is NULL, delete any previous + // ByteBufferPool we had. + (*env)->DeleteGlobalRef(env, opts->byteBufferPool); + opts->byteBufferPool = NULL; + } + ret = 0; +done: + destroyLocalReference(env, byteBufferPool); + if (ret) { + errno = ret; + return -1; + } + return 0; +} + +void hadoopRzOptionsFree(struct hadoopRzOptions *opts) +{ + JNIEnv *env; + env = getJNIEnv(); + if (!env) { + return; + } + hadoopRzOptionsClearCached(env, opts); + if (opts->byteBufferPool) { + (*env)->DeleteGlobalRef(env, opts->byteBufferPool); + opts->byteBufferPool = NULL; + } + free(opts); +} + +struct hadoopRzBuffer +{ + jobject byteBuffer; + uint8_t *ptr; + int32_t length; + int direct; +}; + +static jthrowable hadoopRzOptionsGetEnumSet(JNIEnv *env, + struct hadoopRzOptions *opts, jobject *enumSet) +{ + jthrowable jthr = NULL; + jobject enumInst = NULL, enumSetObj = NULL; + jvalue jVal; + + if (opts->cachedEnumSet) { + // If we cached the value, return it now. + *enumSet = opts->cachedEnumSet; + goto done; + } + if (opts->skipChecksums) { + jthr = fetchEnumInstance(env, HADOOP_RO, + "SKIP_CHECKSUMS", &enumInst); + if (jthr) { + goto done; + } + jthr = invokeMethod(env, &jVal, STATIC, NULL, JC_ENUM_SET, + "of", "(Ljava/lang/Enum;)Ljava/util/EnumSet;", enumInst); + if (jthr) { + goto done; + } + enumSetObj = jVal.l; + } else { + jclass clazz = (*env)->FindClass(env, HADOOP_RO); + if (!clazz) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + jthr = invokeMethod(env, &jVal, STATIC, NULL, JC_ENUM_SET, + "noneOf", "(Ljava/lang/Class;)Ljava/util/EnumSet;", clazz); + if (jthr) { + goto done; + } + enumSetObj = jVal.l; + } + // create global ref + opts->cachedEnumSet = (*env)->NewGlobalRef(env, enumSetObj); + if (!opts->cachedEnumSet) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + *enumSet = opts->cachedEnumSet; + jthr = NULL; +done: + (*env)->DeleteLocalRef(env, enumInst); + (*env)->DeleteLocalRef(env, enumSetObj); + return jthr; +} + +static int hadoopReadZeroExtractBuffer(JNIEnv *env, + const struct hadoopRzOptions *opts, struct hadoopRzBuffer *buffer) +{ + int ret; + jthrowable jthr; + jvalue jVal; + uint8_t *directStart; + void *mallocBuf = NULL; + jint position; + jarray array = NULL; + + jthr = invokeMethod(env, &jVal, INSTANCE, buffer->byteBuffer, + JC_BYTE_BUFFER, "remaining", "()I"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hadoopReadZeroExtractBuffer: ByteBuffer#remaining failed: "); + goto done; + } + buffer->length = jVal.i; + jthr = invokeMethod(env, &jVal, INSTANCE, buffer->byteBuffer, + JC_BYTE_BUFFER, "position", "()I"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hadoopReadZeroExtractBuffer: ByteBuffer#position failed: "); + goto done; + } + position = jVal.i; + directStart = (*env)->GetDirectBufferAddress(env, buffer->byteBuffer); + if (directStart) { + // Handle direct buffers. + buffer->ptr = directStart + position; + buffer->direct = 1; + ret = 0; + goto done; + } + // Handle indirect buffers. + // The JNI docs don't say that GetDirectBufferAddress throws any exceptions + // when it fails. However, they also don't clearly say that it doesn't. It + // seems safest to clear any pending exceptions here, to prevent problems on + // various JVMs. + (*env)->ExceptionClear(env); + if (!opts->byteBufferPool) { + fputs("hadoopReadZeroExtractBuffer: we read through the " + "zero-copy path, but failed to get the address of the buffer via " + "GetDirectBufferAddress. Please make sure your JVM supports " + "GetDirectBufferAddress.\n", stderr); + ret = ENOTSUP; + goto done; + } + // Get the backing array object of this buffer. + jthr = invokeMethod(env, &jVal, INSTANCE, buffer->byteBuffer, + JC_BYTE_BUFFER, "array", "()[B"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hadoopReadZeroExtractBuffer: ByteBuffer#array failed: "); + goto done; + } + array = jVal.l; + if (!array) { + fputs("hadoopReadZeroExtractBuffer: ByteBuffer#array returned NULL.", + stderr); + ret = EIO; + goto done; + } + mallocBuf = malloc(buffer->length); + if (!mallocBuf) { + fprintf(stderr, "hadoopReadZeroExtractBuffer: failed to allocate %d bytes of memory\n", + buffer->length); + ret = ENOMEM; + goto done; + } + (*env)->GetByteArrayRegion(env, array, position, buffer->length, mallocBuf); + jthr = (*env)->ExceptionOccurred(env); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hadoopReadZeroExtractBuffer: GetByteArrayRegion failed: "); + goto done; + } + buffer->ptr = mallocBuf; + buffer->direct = 0; + ret = 0; + +done: + free(mallocBuf); + (*env)->DeleteLocalRef(env, array); + return ret; +} + +static int translateZCRException(JNIEnv *env, jthrowable exc) +{ + int ret; + char *className = NULL; + jthrowable jthr = classNameOfObject(exc, env, &className); + + if (jthr) { + fputs("hadoopReadZero: failed to get class name of " + "exception from read().\n", stderr); + destroyLocalReference(env, exc); + destroyLocalReference(env, jthr); + ret = EIO; + goto done; + } + if (!strcmp(className, "java.lang.UnsupportedOperationException")) { + ret = EPROTONOSUPPORT; + destroyLocalReference(env, exc); + goto done; + } + ret = printExceptionAndFree(env, exc, PRINT_EXC_ALL, + "hadoopZeroCopyRead: ZeroCopyCursor#read failed"); +done: + free(className); + return ret; +} + +struct hadoopRzBuffer* hadoopReadZero(hdfsFile file, + struct hadoopRzOptions *opts, int32_t maxLength) +{ + JNIEnv *env; + jthrowable jthr = NULL; + jvalue jVal; + jobject enumSet = NULL, byteBuffer = NULL; + struct hadoopRzBuffer* buffer = NULL; + int ret; + + env = getJNIEnv(); + if (!env) { + errno = EINTERNAL; + return NULL; + } + if (file->type != HDFS_STREAM_INPUT) { + fputs("Cannot read from a non-InputStream object!\n", stderr); + ret = EINVAL; + goto done; + } + buffer = calloc(1, sizeof(struct hadoopRzBuffer)); + if (!buffer) { + ret = ENOMEM; + goto done; + } + jthr = hadoopRzOptionsGetEnumSet(env, opts, &enumSet); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hadoopReadZero: hadoopRzOptionsGetEnumSet failed: "); + goto done; + } + jthr = invokeMethod(env, &jVal, INSTANCE, file->file, + JC_FS_DATA_INPUT_STREAM, "read", + "(Lorg/apache/hadoop/io/ByteBufferPool;ILjava/util/EnumSet;)" + "Ljava/nio/ByteBuffer;", opts->byteBufferPool, maxLength, enumSet); + if (jthr) { + ret = translateZCRException(env, jthr); + goto done; + } + byteBuffer = jVal.l; + if (!byteBuffer) { + buffer->byteBuffer = NULL; + buffer->length = 0; + buffer->ptr = NULL; + } else { + buffer->byteBuffer = (*env)->NewGlobalRef(env, byteBuffer); + if (!buffer->byteBuffer) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hadoopReadZero: failed to create global ref to ByteBuffer"); + goto done; + } + ret = hadoopReadZeroExtractBuffer(env, opts, buffer); + if (ret) { + goto done; + } + } + ret = 0; +done: + (*env)->DeleteLocalRef(env, byteBuffer); + if (ret) { + if (buffer) { + if (buffer->byteBuffer) { + (*env)->DeleteGlobalRef(env, buffer->byteBuffer); + } + free(buffer); + } + errno = ret; + return NULL; + } else { + errno = 0; + } + return buffer; +} + +int32_t hadoopRzBufferLength(const struct hadoopRzBuffer *buffer) +{ + return buffer->length; +} + +const void *hadoopRzBufferGet(const struct hadoopRzBuffer *buffer) +{ + return buffer->ptr; +} + +void hadoopRzBufferFree(hdfsFile file, struct hadoopRzBuffer *buffer) +{ + jvalue jVal; + jthrowable jthr; + JNIEnv* env; + + env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return; + } + if (buffer->byteBuffer) { + jthr = invokeMethod(env, &jVal, INSTANCE, file->file, + JC_FS_DATA_INPUT_STREAM, "releaseBuffer", + "(Ljava/nio/ByteBuffer;)V", buffer->byteBuffer); + if (jthr) { + printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hadoopRzBufferFree: releaseBuffer failed: "); + // even on error, we have to delete the reference. + } + (*env)->DeleteGlobalRef(env, buffer->byteBuffer); + } + if (!buffer->direct) { + free(buffer->ptr); + } + memset(buffer, 0, sizeof(*buffer)); + free(buffer); +} + +char*** +hdfsGetHosts(hdfsFS fs, const char *path, tOffset start, tOffset length) +{ + // JAVA EQUIVALENT: + // fs.getFileBlockLoctions(new Path(path), start, length); + + jobject jFS = (jobject)fs; + jthrowable jthr; + jobject jPath = NULL; + jobject jFileStatus = NULL; + jvalue jFSVal, jVal; + jobjectArray jBlockLocations = NULL, jFileBlockHosts = NULL; + jstring jHost = NULL; + char*** blockHosts = NULL; + int i, j, ret; + jsize jNumFileBlocks = 0; + jobject jFileBlock; + jsize jNumBlockHosts; + const char *hostName; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return NULL; + } + + //Create an object of org.apache.hadoop.fs.Path + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetHosts(path=%s): constructNewObjectOfPath", path); + goto done; + } + jthr = invokeMethod(env, &jFSVal, INSTANCE, jFS, JC_FILE_SYSTEM, + "getFileStatus", "(Lorg/apache/hadoop/fs/Path;)" + "Lorg/apache/hadoop/fs/FileStatus;", jPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, NOPRINT_EXC_FILE_NOT_FOUND, + "hdfsGetHosts(path=%s, start=%"PRId64", length=%"PRId64"):" + "FileSystem#getFileStatus", path, start, length); + destroyLocalReference(env, jPath); + goto done; + } + jFileStatus = jFSVal.l; + + //org.apache.hadoop.fs.FileSystem#getFileBlockLocations + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, JC_FILE_SYSTEM, + "getFileBlockLocations", + "(Lorg/apache/hadoop/fs/FileStatus;JJ)" + "[Lorg/apache/hadoop/fs/BlockLocation;", jFileStatus, start, + length); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetHosts(path=%s, start=%"PRId64", length=%"PRId64"):" + "FileSystem#getFileBlockLocations", path, start, length); + goto done; + } + jBlockLocations = jVal.l; + + //Figure out no of entries in jBlockLocations + //Allocate memory and add NULL at the end + jNumFileBlocks = (*env)->GetArrayLength(env, jBlockLocations); + + blockHosts = calloc(jNumFileBlocks + 1, sizeof(char**)); + if (blockHosts == NULL) { + ret = ENOMEM; + goto done; + } + if (jNumFileBlocks == 0) { + ret = 0; + goto done; + } + + //Now parse each block to get hostnames + for (i = 0; i < jNumFileBlocks; ++i) { + jFileBlock = + (*env)->GetObjectArrayElement(env, jBlockLocations, i); + jthr = (*env)->ExceptionOccurred(env); + if (jthr || !jFileBlock) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetHosts(path=%s, start=%"PRId64", length=%"PRId64"):" + "GetObjectArrayElement(%d)", path, start, length, i); + goto done; + } + + jthr = invokeMethod(env, &jVal, INSTANCE, jFileBlock, + JC_BLOCK_LOCATION, "getHosts", + "()[Ljava/lang/String;"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetHosts(path=%s, start=%"PRId64", length=%"PRId64"):" + "BlockLocation#getHosts", path, start, length); + goto done; + } + jFileBlockHosts = jVal.l; + if (!jFileBlockHosts) { + fprintf(stderr, + "hdfsGetHosts(path=%s, start=%"PRId64", length=%"PRId64"):" + "BlockLocation#getHosts returned NULL", path, start, length); + ret = EINTERNAL; + goto done; + } + //Figure out no of hosts in jFileBlockHosts, and allocate the memory + jNumBlockHosts = (*env)->GetArrayLength(env, jFileBlockHosts); + blockHosts[i] = calloc(jNumBlockHosts + 1, sizeof(char*)); + if (!blockHosts[i]) { + ret = ENOMEM; + goto done; + } + + //Now parse each hostname + for (j = 0; j < jNumBlockHosts; ++j) { + jHost = (*env)->GetObjectArrayElement(env, jFileBlockHosts, j); + jthr = (*env)->ExceptionOccurred(env); + if (jthr || !jHost) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetHosts(path=%s, start=%"PRId64", length=%"PRId64"): " + "NewByteArray", path, start, length); + goto done; + } + hostName = + (const char*)((*env)->GetStringUTFChars(env, jHost, NULL)); + if (!hostName) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsGetHosts(path=%s, start=%"PRId64", length=%"PRId64", " + "j=%d out of %d): GetStringUTFChars", + path, start, length, j, jNumBlockHosts); + goto done; + } + blockHosts[i][j] = strdup(hostName); + (*env)->ReleaseStringUTFChars(env, jHost, hostName); + if (!blockHosts[i][j]) { + ret = ENOMEM; + goto done; + } + destroyLocalReference(env, jHost); + jHost = NULL; + } + + destroyLocalReference(env, jFileBlockHosts); + jFileBlockHosts = NULL; + } + ret = 0; + +done: + destroyLocalReference(env, jPath); + destroyLocalReference(env, jFileStatus); + destroyLocalReference(env, jBlockLocations); + destroyLocalReference(env, jFileBlockHosts); + destroyLocalReference(env, jHost); + if (ret) { + errno = ret; + if (blockHosts) { + hdfsFreeHosts(blockHosts); + } + return NULL; + } + + return blockHosts; +} + + +void hdfsFreeHosts(char ***blockHosts) +{ + int i, j; + for (i=0; blockHosts[i]; i++) { + for (j=0; blockHosts[i][j]; j++) { + free(blockHosts[i][j]); + } + free(blockHosts[i]); + } + free(blockHosts); +} + + +tOffset hdfsGetDefaultBlockSize(hdfsFS fs) +{ + // JAVA EQUIVALENT: + // fs.getDefaultBlockSize(); + + jobject jFS = (jobject)fs; + jvalue jVal; + jthrowable jthr; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //FileSystem#getDefaultBlockSize() + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, JC_FILE_SYSTEM, + "getDefaultBlockSize", "()J"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetDefaultBlockSize: FileSystem#getDefaultBlockSize"); + return -1; + } + return jVal.j; +} + + +tOffset hdfsGetDefaultBlockSizeAtPath(hdfsFS fs, const char *path) +{ + // JAVA EQUIVALENT: + // fs.getDefaultBlockSize(path); + + jthrowable jthr; + jobject jFS = (jobject)fs; + jobject jPath; + tOffset blockSize; + JNIEnv* env = getJNIEnv(); + + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetDefaultBlockSize(path=%s): constructNewObjectOfPath", + path); + return -1; + } + jthr = getDefaultBlockSize(env, jFS, jPath, (jlong *)&blockSize); + (*env)->DeleteLocalRef(env, jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetDefaultBlockSize(path=%s): " + "FileSystem#getDefaultBlockSize", path); + return -1; + } + return blockSize; +} + + +tOffset hdfsGetCapacity(hdfsFS fs) +{ + // JAVA EQUIVALENT: + // FsStatus fss = fs.getStatus(); + // return Fss.getCapacity(); + + jobject jFS = (jobject)fs; + jvalue jVal; + jthrowable jthr; + jobject fss; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //FileSystem#getStatus + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, JC_FILE_SYSTEM, + "getStatus", "()Lorg/apache/hadoop/fs/FsStatus;"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetCapacity: FileSystem#getStatus"); + return -1; + } + fss = (jobject)jVal.l; + jthr = invokeMethod(env, &jVal, INSTANCE, fss, + JC_FS_STATUS, "getCapacity", "()J"); + destroyLocalReference(env, fss); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetCapacity: FsStatus#getCapacity"); + return -1; + } + return jVal.j; +} + + + +tOffset hdfsGetUsed(hdfsFS fs) +{ + // JAVA EQUIVALENT: + // FsStatus fss = fs.getStatus(); + // return Fss.getUsed(); + + jobject jFS = (jobject)fs; + jvalue jVal; + jthrowable jthr; + jobject fss; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //FileSystem#getStatus + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, JC_FILE_SYSTEM, + "getStatus", "()Lorg/apache/hadoop/fs/FsStatus;"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetUsed: FileSystem#getStatus"); + return -1; + } + fss = (jobject)jVal.l; + jthr = invokeMethod(env, &jVal, INSTANCE, fss, JC_FS_STATUS, + "getUsed", "()J"); + destroyLocalReference(env, fss); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetUsed: FsStatus#getUsed"); + return -1; + } + return jVal.j; +} + +/** + * We cannot add new fields to the hdfsFileInfo structure because it would break + * binary compatibility. The reason is because we return an array + * of hdfsFileInfo structures from hdfsListDirectory. So changing the size of + * those structures would break all programs that relied on finding the second + * element in the array at + sizeof(struct hdfsFileInfo). + * + * So instead, we add the new fields to the hdfsExtendedFileInfo structure. + * This structure is contained in the mOwner string found inside the + * hdfsFileInfo. Specifically, the format of mOwner is: + * + * [owner-string] [null byte] [padding] [hdfsExtendedFileInfo structure] + * + * The padding is added so that the hdfsExtendedFileInfo structure starts on an + * 8-byte boundary. + * + * @param str The string to locate the extended info in. + * @return The offset of the hdfsExtendedFileInfo structure. + */ +static size_t getExtendedFileInfoOffset(const char *str) +{ + int num_64_bit_words = ((strlen(str) + 1) + 7) / 8; + return num_64_bit_words * 8; +} + +static struct hdfsExtendedFileInfo *getExtendedFileInfo(hdfsFileInfo *fileInfo) +{ + char *owner = fileInfo->mOwner; + return (struct hdfsExtendedFileInfo *)(owner + + getExtendedFileInfoOffset(owner)); +} + +static jthrowable +getFileInfoFromStat(JNIEnv *env, jobject jStat, hdfsFileInfo *fileInfo) +{ + jvalue jVal; + jthrowable jthr; + jobject jPath = NULL; + jstring jPathName = NULL; + jstring jUserName = NULL; + jstring jGroupName = NULL; + jobject jPermission = NULL; + const char *cPathName; + const char *cUserName; + const char *cGroupName; + struct hdfsExtendedFileInfo *extInfo; + size_t extOffset; + + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, JC_FILE_STATUS, "isDir", + "()Z"); + if (jthr) + goto done; + fileInfo->mKind = jVal.z ? kObjectKindDirectory : kObjectKindFile; + + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, JC_FILE_STATUS, + "getReplication", "()S"); + if (jthr) + goto done; + fileInfo->mReplication = jVal.s; + + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, JC_FILE_STATUS, + "getBlockSize", "()J"); + if (jthr) + goto done; + fileInfo->mBlockSize = jVal.j; + + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, JC_FILE_STATUS, + "getModificationTime", "()J"); + if (jthr) + goto done; + fileInfo->mLastMod = jVal.j / 1000; + + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, JC_FILE_STATUS, + "getAccessTime", "()J"); + if (jthr) + goto done; + fileInfo->mLastAccess = (tTime) (jVal.j / 1000); + + if (fileInfo->mKind == kObjectKindFile) { + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, JC_FILE_STATUS, + "getLen", "()J"); + if (jthr) + goto done; + fileInfo->mSize = jVal.j; + } + + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, JC_FILE_STATUS, + "getPath", "()Lorg/apache/hadoop/fs/Path;"); + if (jthr) + goto done; + jPath = jVal.l; + if (jPath == NULL) { + jthr = newRuntimeError(env, "org.apache.hadoop.fs.FileStatus#" + "getPath returned NULL!"); + goto done; + } + + jthr = invokeMethod(env, &jVal, INSTANCE, jPath, JC_PATH, "toString", + "()Ljava/lang/String;"); + if (jthr) + goto done; + jPathName = jVal.l; + cPathName = + (const char*) ((*env)->GetStringUTFChars(env, jPathName, NULL)); + if (!cPathName) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + fileInfo->mName = strdup(cPathName); + (*env)->ReleaseStringUTFChars(env, jPathName, cPathName); + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, JC_FILE_STATUS, "getOwner", + "()Ljava/lang/String;"); + if (jthr) + goto done; + jUserName = jVal.l; + cUserName = + (const char*) ((*env)->GetStringUTFChars(env, jUserName, NULL)); + if (!cUserName) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + extOffset = getExtendedFileInfoOffset(cUserName); + fileInfo->mOwner = malloc(extOffset + sizeof(struct hdfsExtendedFileInfo)); + if (!fileInfo->mOwner) { + jthr = newRuntimeError(env, "getFileInfo: OOM allocating mOwner"); + goto done; + } + strcpy(fileInfo->mOwner, cUserName); + (*env)->ReleaseStringUTFChars(env, jUserName, cUserName); + extInfo = getExtendedFileInfo(fileInfo); + memset(extInfo, 0, sizeof(*extInfo)); + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, JC_FILE_STATUS, + "isEncrypted", "()Z"); + if (jthr) { + goto done; + } + if (jVal.z == JNI_TRUE) { + extInfo->flags |= HDFS_EXTENDED_FILE_INFO_ENCRYPTED; + } + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, JC_FILE_STATUS, + "getGroup", "()Ljava/lang/String;"); + if (jthr) + goto done; + jGroupName = jVal.l; + cGroupName = (const char*) ((*env)->GetStringUTFChars(env, jGroupName, NULL)); + if (!cGroupName) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + fileInfo->mGroup = strdup(cGroupName); + (*env)->ReleaseStringUTFChars(env, jGroupName, cGroupName); + + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, JC_FILE_STATUS, + "getPermission", + "()Lorg/apache/hadoop/fs/permission/FsPermission;"); + if (jthr) + goto done; + if (jVal.l == NULL) { + jthr = newRuntimeError(env, "%s#getPermission returned NULL!", + HADOOP_FILESTAT); + goto done; + } + jPermission = jVal.l; + jthr = invokeMethod(env, &jVal, INSTANCE, jPermission, + JC_FS_PERMISSION, "toShort", "()S"); + if (jthr) + goto done; + fileInfo->mPermissions = jVal.s; + jthr = NULL; + +done: + if (jthr) + hdfsFreeFileInfoEntry(fileInfo); + destroyLocalReference(env, jPath); + destroyLocalReference(env, jPathName); + destroyLocalReference(env, jUserName); + destroyLocalReference(env, jGroupName); + destroyLocalReference(env, jPermission); + return jthr; +} + +static jthrowable +getFileInfo(JNIEnv *env, jobject jFS, jobject jPath, hdfsFileInfo **fileInfo) +{ + // JAVA EQUIVALENT: + // fs.isDirectory(f) + // fs.getModificationTime() + // fs.getAccessTime() + // fs.getLength(f) + // f.getPath() + // f.getOwner() + // f.getGroup() + // f.getPermission().toShort() + jobject jStat; + jvalue jVal; + jthrowable jthr; + + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, JC_FILE_SYSTEM, "exists", + JMETHOD1(JPARAM(HADOOP_PATH), "Z"), jPath); + if (jthr) + return jthr; + if (jVal.z == 0) { + *fileInfo = NULL; + return NULL; + } + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, JC_FILE_SYSTEM, + "getFileStatus", JMETHOD1(JPARAM(HADOOP_PATH), JPARAM + (HADOOP_FILESTAT)), jPath); + if (jthr) + return jthr; + jStat = jVal.l; + *fileInfo = calloc(1, sizeof(hdfsFileInfo)); + if (!*fileInfo) { + destroyLocalReference(env, jStat); + return newRuntimeError(env, "getFileInfo: OOM allocating hdfsFileInfo"); + } + jthr = getFileInfoFromStat(env, jStat, *fileInfo); + destroyLocalReference(env, jStat); + return jthr; +} + + + +hdfsFileInfo* hdfsListDirectory(hdfsFS fs, const char *path, int *numEntries) +{ + // JAVA EQUIVALENT: + // Path p(path); + // Path []pathList = fs.listPaths(p) + // foreach path in pathList + // getFileInfo(path) + + jobject jFS = (jobject)fs; + jthrowable jthr; + jobject jPath = NULL; + hdfsFileInfo *pathList = NULL; + jobjectArray jPathList = NULL; + jvalue jVal; + jsize jPathListSize = 0; + int ret; + jsize i; + jobject tmpStat; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return NULL; + } + + //Create an object of org.apache.hadoop.fs.Path + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsListDirectory(%s): constructNewObjectOfPath", path); + goto done; + } + + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, + JC_DISTRIBUTED_FILE_SYSTEM, "listStatus", + JMETHOD1(JPARAM(HADOOP_PATH), JARRPARAM(HADOOP_FILESTAT)), jPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, + NOPRINT_EXC_ACCESS_CONTROL | NOPRINT_EXC_FILE_NOT_FOUND | + NOPRINT_EXC_UNRESOLVED_LINK, + "hdfsListDirectory(%s): FileSystem#listStatus", path); + goto done; + } + jPathList = jVal.l; + + //Figure out the number of entries in that directory + jPathListSize = (*env)->GetArrayLength(env, jPathList); + if (jPathListSize == 0) { + ret = 0; + goto done; + } + + //Allocate memory + pathList = calloc(jPathListSize, sizeof(hdfsFileInfo)); + if (pathList == NULL) { + ret = ENOMEM; + goto done; + } + + //Save path information in pathList + for (i=0; i < jPathListSize; ++i) { + tmpStat = (*env)->GetObjectArrayElement(env, jPathList, i); + jthr = (*env)->ExceptionOccurred(env); + if (jthr || !tmpStat) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsListDirectory(%s): GetObjectArrayElement(%d out of %d)", + path, i, jPathListSize); + goto done; + } + jthr = getFileInfoFromStat(env, tmpStat, &pathList[i]); + destroyLocalReference(env, tmpStat); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsListDirectory(%s): getFileInfoFromStat(%d out of %d)", + path, i, jPathListSize); + goto done; + } + } + ret = 0; + +done: + destroyLocalReference(env, jPath); + destroyLocalReference(env, jPathList); + + if (ret) { + hdfsFreeFileInfo(pathList, jPathListSize); + errno = ret; + return NULL; + } + *numEntries = jPathListSize; + errno = 0; + return pathList; +} + + + +hdfsFileInfo *hdfsGetPathInfo(hdfsFS fs, const char *path) +{ + // JAVA EQUIVALENT: + // File f(path); + // fs.isDirectory(f) + // fs.lastModified() ?? + // fs.getLength(f) + // f.getPath() + + jobject jFS = (jobject)fs; + jobject jPath; + jthrowable jthr; + hdfsFileInfo *fileInfo; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return NULL; + } + + //Create an object of org.apache.hadoop.fs.Path + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetPathInfo(%s): constructNewObjectOfPath", path); + return NULL; + } + jthr = getFileInfo(env, jFS, jPath, &fileInfo); + destroyLocalReference(env, jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, + NOPRINT_EXC_ACCESS_CONTROL | NOPRINT_EXC_FILE_NOT_FOUND | + NOPRINT_EXC_UNRESOLVED_LINK, + "hdfsGetPathInfo(%s): getFileInfo", path); + return NULL; + } + if (!fileInfo) { + errno = ENOENT; + return NULL; + } + return fileInfo; +} + +static void hdfsFreeFileInfoEntry(hdfsFileInfo *hdfsFileInfo) +{ + free(hdfsFileInfo->mName); + free(hdfsFileInfo->mOwner); + free(hdfsFileInfo->mGroup); + memset(hdfsFileInfo, 0, sizeof(*hdfsFileInfo)); +} + +void hdfsFreeFileInfo(hdfsFileInfo *hdfsFileInfo, int numEntries) +{ + //Free the mName, mOwner, and mGroup + int i; + for (i=0; i < numEntries; ++i) { + hdfsFreeFileInfoEntry(hdfsFileInfo + i); + } + + //Free entire block + free(hdfsFileInfo); +} + +int hdfsFileIsEncrypted(hdfsFileInfo *fileInfo) +{ + struct hdfsExtendedFileInfo *extInfo; + + extInfo = getExtendedFileInfo(fileInfo); + return !!(extInfo->flags & HDFS_EXTENDED_FILE_INFO_ENCRYPTED); +} + +char* hdfsGetLastExceptionRootCause() +{ + return getLastTLSExceptionRootCause(); +} + +char* hdfsGetLastExceptionStackTrace() +{ + return getLastTLSExceptionStackTrace(); +} + +/** + * vim: ts=4: sw=4: et: + */ diff --git a/native/hdfs-sys/libhdfs/hdfs_3_3/include/hdfs/hdfs.h b/native/hdfs-sys/libhdfs/hdfs_3_3/include/hdfs/hdfs.h new file mode 100644 index 0000000000..e58a6232d2 --- /dev/null +++ b/native/hdfs-sys/libhdfs/hdfs_3_3/include/hdfs/hdfs.h @@ -0,0 +1,1105 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFS_HDFS_H +#define LIBHDFS_HDFS_H + +#include /* for EINTERNAL, etc. */ +#include /* for O_RDONLY, O_WRONLY */ +#include /* for uint64_t, etc. */ +#include /* for time_t */ + +/* + * Support export of DLL symbols during libhdfs build, and import of DLL symbols + * during client application build. A client application may optionally define + * symbol LIBHDFS_DLL_IMPORT in its build. This is not strictly required, but + * the compiler can produce more efficient code with it. + */ +#ifdef WIN32 + #ifdef LIBHDFS_DLL_EXPORT + #define LIBHDFS_EXTERNAL __declspec(dllexport) + #elif LIBHDFS_DLL_IMPORT + #define LIBHDFS_EXTERNAL __declspec(dllimport) + #else + #define LIBHDFS_EXTERNAL + #endif +#else + #ifdef LIBHDFS_DLL_EXPORT + #define LIBHDFS_EXTERNAL __attribute__((visibility("default"))) + #elif LIBHDFS_DLL_IMPORT + #define LIBHDFS_EXTERNAL __attribute__((visibility("default"))) + #else + #define LIBHDFS_EXTERNAL + #endif +#endif + +#ifndef O_RDONLY +#define O_RDONLY 1 +#endif + +#ifndef O_WRONLY +#define O_WRONLY 2 +#endif + +#ifndef EINTERNAL +#define EINTERNAL 255 +#endif + +#define ELASTIC_BYTE_BUFFER_POOL_CLASS \ + "org/apache/hadoop/io/ElasticByteBufferPool" + +/** All APIs set errno to meaningful values */ + +#ifdef __cplusplus +extern "C" { +#endif + /** + * Some utility decls used in libhdfs. + */ + struct hdfsBuilder; + typedef int32_t tSize; /// size of data for read/write io ops + typedef time_t tTime; /// time type in seconds + typedef int64_t tOffset;/// offset within the file + typedef uint16_t tPort; /// port + typedef enum tObjectKind { + kObjectKindFile = 'F', + kObjectKindDirectory = 'D', + } tObjectKind; + struct hdfsStreamBuilder; + + + /** + * The C reflection of org.apache.org.hadoop.FileSystem . + */ + struct hdfs_internal; + typedef struct hdfs_internal* hdfsFS; + + struct hdfsFile_internal; + typedef struct hdfsFile_internal* hdfsFile; + + struct hadoopRzOptions; + + struct hadoopRzBuffer; + + /** + * Determine if a file is open for read. + * + * @param file The HDFS file + * @return 1 if the file is open for read; 0 otherwise + */ + LIBHDFS_EXTERNAL + int hdfsFileIsOpenForRead(hdfsFile file); + + /** + * Determine if a file is open for write. + * + * @param file The HDFS file + * @return 1 if the file is open for write; 0 otherwise + */ + LIBHDFS_EXTERNAL + int hdfsFileIsOpenForWrite(hdfsFile file); + + struct hdfsReadStatistics { + uint64_t totalBytesRead; + uint64_t totalLocalBytesRead; + uint64_t totalShortCircuitBytesRead; + uint64_t totalZeroCopyBytesRead; + }; + + /** + * Get read statistics about a file. This is only applicable to files + * opened for reading. + * + * @param file The HDFS file + * @param stats (out parameter) on a successful return, the read + * statistics. Unchanged otherwise. You must free the + * returned statistics with hdfsFileFreeReadStatistics. + * @return 0 if the statistics were successfully returned, + * -1 otherwise. On a failure, please check errno against + * ENOTSUP. webhdfs, LocalFilesystem, and so forth may + * not support read statistics. + */ + LIBHDFS_EXTERNAL + int hdfsFileGetReadStatistics(hdfsFile file, + struct hdfsReadStatistics **stats); + + /** + * @param stats HDFS read statistics for a file. + * + * @return the number of remote bytes read. + */ + LIBHDFS_EXTERNAL + int64_t hdfsReadStatisticsGetRemoteBytesRead( + const struct hdfsReadStatistics *stats); + + /** + * Clear the read statistics for a file. + * + * @param file The file to clear the read statistics of. + * + * @return 0 on success; the error code otherwise. + * EINVAL: the file is not open for reading. + * ENOTSUP: the file does not support clearing the read + * statistics. + * Errno will also be set to this code on failure. + */ + LIBHDFS_EXTERNAL + int hdfsFileClearReadStatistics(hdfsFile file); + + /** + * Free some HDFS read statistics. + * + * @param stats The HDFS read statistics to free. + */ + LIBHDFS_EXTERNAL + void hdfsFileFreeReadStatistics(struct hdfsReadStatistics *stats); + + struct hdfsHedgedReadMetrics { + uint64_t hedgedReadOps; + uint64_t hedgedReadOpsWin; + uint64_t hedgedReadOpsInCurThread; + }; + + /** + * Get cluster wide hedged read metrics. + * + * @param fs The configured filesystem handle + * @param metrics (out parameter) on a successful return, the hedged read + * metrics. Unchanged otherwise. You must free the returned + * statistics with hdfsFreeHedgedReadMetrics. + * @return 0 if the metrics were successfully returned, -1 otherwise. + * On a failure, please check errno against + * ENOTSUP. webhdfs, LocalFilesystem, and so forth may + * not support hedged read metrics. + */ + LIBHDFS_EXTERNAL + int hdfsGetHedgedReadMetrics(hdfsFS fs, struct hdfsHedgedReadMetrics **metrics); + + /** + * Free HDFS Hedged read metrics. + * + * @param metrics The HDFS Hedged read metrics to free + */ + LIBHDFS_EXTERNAL + void hdfsFreeHedgedReadMetrics(struct hdfsHedgedReadMetrics *metrics); + + /** + * hdfsConnectAsUser - Connect to a hdfs file system as a specific user + * Connect to the hdfs. + * @param nn The NameNode. See hdfsBuilderSetNameNode for details. + * @param port The port on which the server is listening. + * @param user the user name (this is hadoop domain user). Or NULL is equivelant to hhdfsConnect(host, port) + * @return Returns a handle to the filesystem or NULL on error. + * @deprecated Use hdfsBuilderConnect instead. + */ + LIBHDFS_EXTERNAL + hdfsFS hdfsConnectAsUser(const char* nn, tPort port, const char *user); + + /** + * hdfsConnect - Connect to a hdfs file system. + * Connect to the hdfs. + * @param nn The NameNode. See hdfsBuilderSetNameNode for details. + * @param port The port on which the server is listening. + * @return Returns a handle to the filesystem or NULL on error. + * @deprecated Use hdfsBuilderConnect instead. + */ + LIBHDFS_EXTERNAL + hdfsFS hdfsConnect(const char* nn, tPort port); + + /** + * hdfsConnect - Connect to an hdfs file system. + * + * Forces a new instance to be created + * + * @param nn The NameNode. See hdfsBuilderSetNameNode for details. + * @param port The port on which the server is listening. + * @param user The user name to use when connecting + * @return Returns a handle to the filesystem or NULL on error. + * @deprecated Use hdfsBuilderConnect instead. + */ + LIBHDFS_EXTERNAL + hdfsFS hdfsConnectAsUserNewInstance(const char* nn, tPort port, const char *user ); + + /** + * hdfsConnect - Connect to an hdfs file system. + * + * Forces a new instance to be created + * + * @param nn The NameNode. See hdfsBuilderSetNameNode for details. + * @param port The port on which the server is listening. + * @return Returns a handle to the filesystem or NULL on error. + * @deprecated Use hdfsBuilderConnect instead. + */ + LIBHDFS_EXTERNAL + hdfsFS hdfsConnectNewInstance(const char* nn, tPort port); + + /** + * Connect to HDFS using the parameters defined by the builder. + * + * The HDFS builder will be freed, whether or not the connection was + * successful. + * + * Every successful call to hdfsBuilderConnect should be matched with a call + * to hdfsDisconnect, when the hdfsFS is no longer needed. + * + * @param bld The HDFS builder + * @return Returns a handle to the filesystem, or NULL on error. + */ + LIBHDFS_EXTERNAL + hdfsFS hdfsBuilderConnect(struct hdfsBuilder *bld); + + /** + * Create an HDFS builder. + * + * @return The HDFS builder, or NULL on error. + */ + LIBHDFS_EXTERNAL + struct hdfsBuilder *hdfsNewBuilder(void); + + /** + * Force the builder to always create a new instance of the FileSystem, + * rather than possibly finding one in the cache. + * + * @param bld The HDFS builder + */ + LIBHDFS_EXTERNAL + void hdfsBuilderSetForceNewInstance(struct hdfsBuilder *bld); + + /** + * Set the HDFS NameNode to connect to. + * + * @param bld The HDFS builder + * @param nn The NameNode to use. + * + * If the string given is 'default', the default NameNode + * configuration will be used (from the XML configuration files) + * + * If NULL is given, a LocalFileSystem will be created. + * + * If the string starts with a protocol type such as file:// or + * hdfs://, this protocol type will be used. If not, the + * hdfs:// protocol type will be used. + * + * You may specify a NameNode port in the usual way by + * passing a string of the format hdfs://:. + * Alternately, you may set the port with + * hdfsBuilderSetNameNodePort. However, you must not pass the + * port in two different ways. + */ + LIBHDFS_EXTERNAL + void hdfsBuilderSetNameNode(struct hdfsBuilder *bld, const char *nn); + + /** + * Set the port of the HDFS NameNode to connect to. + * + * @param bld The HDFS builder + * @param port The port. + */ + LIBHDFS_EXTERNAL + void hdfsBuilderSetNameNodePort(struct hdfsBuilder *bld, tPort port); + + /** + * Set the username to use when connecting to the HDFS cluster. + * + * @param bld The HDFS builder + * @param userName The user name. The string will be shallow-copied. + */ + LIBHDFS_EXTERNAL + void hdfsBuilderSetUserName(struct hdfsBuilder *bld, const char *userName); + + /** + * Set the path to the Kerberos ticket cache to use when connecting to + * the HDFS cluster. + * + * @param bld The HDFS builder + * @param kerbTicketCachePath The Kerberos ticket cache path. The string + * will be shallow-copied. + */ + LIBHDFS_EXTERNAL + void hdfsBuilderSetKerbTicketCachePath(struct hdfsBuilder *bld, + const char *kerbTicketCachePath); + + /** + * Free an HDFS builder. + * + * It is normally not necessary to call this function since + * hdfsBuilderConnect frees the builder. + * + * @param bld The HDFS builder + */ + LIBHDFS_EXTERNAL + void hdfsFreeBuilder(struct hdfsBuilder *bld); + + /** + * Set a configuration string for an HdfsBuilder. + * + * @param key The key to set. + * @param val The value, or NULL to set no value. + * This will be shallow-copied. You are responsible for + * ensuring that it remains valid until the builder is + * freed. + * + * @return 0 on success; nonzero error code otherwise. + */ + LIBHDFS_EXTERNAL + int hdfsBuilderConfSetStr(struct hdfsBuilder *bld, const char *key, + const char *val); + + /** + * Get a configuration string. + * + * @param key The key to find + * @param val (out param) The value. This will be set to NULL if the + * key isn't found. You must free this string with + * hdfsConfStrFree. + * + * @return 0 on success; nonzero error code otherwise. + * Failure to find the key is not an error. + */ + LIBHDFS_EXTERNAL + int hdfsConfGetStr(const char *key, char **val); + + /** + * Get a configuration integer. + * + * @param key The key to find + * @param val (out param) The value. This will NOT be changed if the + * key isn't found. + * + * @return 0 on success; nonzero error code otherwise. + * Failure to find the key is not an error. + */ + LIBHDFS_EXTERNAL + int hdfsConfGetInt(const char *key, int32_t *val); + + /** + * Free a configuration string found with hdfsConfGetStr. + * + * @param val A configuration string obtained from hdfsConfGetStr + */ + LIBHDFS_EXTERNAL + void hdfsConfStrFree(char *val); + + /** + * hdfsDisconnect - Disconnect from the hdfs file system. + * Disconnect from hdfs. + * @param fs The configured filesystem handle. + * @return Returns 0 on success, -1 on error. + * Even if there is an error, the resources associated with the + * hdfsFS will be freed. + */ + LIBHDFS_EXTERNAL + int hdfsDisconnect(hdfsFS fs); + + /** + * hdfsOpenFile - Open a hdfs file in given mode. + * @deprecated Use the hdfsStreamBuilder functions instead. + * This function does not support setting block sizes bigger than 2 GB. + * + * @param fs The configured filesystem handle. + * @param path The full path to the file. + * @param flags - an | of bits/fcntl.h file flags - supported flags are O_RDONLY, O_WRONLY (meaning create or overwrite i.e., implies O_TRUNCAT), + * O_WRONLY|O_APPEND. Other flags are generally ignored other than (O_RDWR || (O_EXCL & O_CREAT)) which return NULL and set errno equal ENOTSUP. + * @param bufferSize Size of buffer for read/write - pass 0 if you want + * to use the default configured values. + * @param replication Block replication - pass 0 if you want to use + * the default configured values. + * @param blocksize Size of block - pass 0 if you want to use the + * default configured values. Note that if you want a block size bigger + * than 2 GB, you must use the hdfsStreamBuilder API rather than this + * deprecated function. + * @return Returns the handle to the open file or NULL on error. + */ + LIBHDFS_EXTERNAL + hdfsFile hdfsOpenFile(hdfsFS fs, const char* path, int flags, + int bufferSize, short replication, tSize blocksize); + + /** + * hdfsStreamBuilderAlloc - Allocate an HDFS stream builder. + * + * @param fs The configured filesystem handle. + * @param path The full path to the file. Will be deep-copied. + * @param flags The open flags, as in hdfsOpenFile. + * @return Returns the hdfsStreamBuilder, or NULL on error. + */ + LIBHDFS_EXTERNAL + struct hdfsStreamBuilder *hdfsStreamBuilderAlloc(hdfsFS fs, + const char *path, int flags); + + /** + * hdfsStreamBuilderFree - Free an HDFS file builder. + * + * It is normally not necessary to call this function since + * hdfsStreamBuilderBuild frees the builder. + * + * @param bld The hdfsStreamBuilder to free. + */ + LIBHDFS_EXTERNAL + void hdfsStreamBuilderFree(struct hdfsStreamBuilder *bld); + + /** + * hdfsStreamBuilderSetBufferSize - Set the stream buffer size. + * + * @param bld The hdfs stream builder. + * @param bufferSize The buffer size to set. + * + * @return 0 on success, or -1 on error. Errno will be set on error. + */ + LIBHDFS_EXTERNAL + int hdfsStreamBuilderSetBufferSize(struct hdfsStreamBuilder *bld, + int32_t bufferSize); + + /** + * hdfsStreamBuilderSetReplication - Set the replication for the stream. + * This is only relevant for output streams, which will create new blocks. + * + * @param bld The hdfs stream builder. + * @param replication The replication to set. + * + * @return 0 on success, or -1 on error. Errno will be set on error. + * If you call this on an input stream builder, you will get + * EINVAL, because this configuration is not relevant to input + * streams. + */ + LIBHDFS_EXTERNAL + int hdfsStreamBuilderSetReplication(struct hdfsStreamBuilder *bld, + int16_t replication); + + /** + * hdfsStreamBuilderSetDefaultBlockSize - Set the default block size for + * the stream. This is only relevant for output streams, which will create + * new blocks. + * + * @param bld The hdfs stream builder. + * @param defaultBlockSize The default block size to set. + * + * @return 0 on success, or -1 on error. Errno will be set on error. + * If you call this on an input stream builder, you will get + * EINVAL, because this configuration is not relevant to input + * streams. + */ + LIBHDFS_EXTERNAL + int hdfsStreamBuilderSetDefaultBlockSize(struct hdfsStreamBuilder *bld, + int64_t defaultBlockSize); + + /** + * hdfsStreamBuilderBuild - Build the stream by calling open or create. + * + * @param bld The hdfs stream builder. This pointer will be freed, whether + * or not the open succeeds. + * + * @return the stream pointer on success, or NULL on error. Errno will be + * set on error. + */ + LIBHDFS_EXTERNAL + hdfsFile hdfsStreamBuilderBuild(struct hdfsStreamBuilder *bld); + + /** + * hdfsTruncateFile - Truncate a hdfs file to given lenght. + * @param fs The configured filesystem handle. + * @param path The full path to the file. + * @param newlength The size the file is to be truncated to + * @return 1 if the file has been truncated to the desired newlength + * and is immediately available to be reused for write operations + * such as append. + * 0 if a background process of adjusting the length of the last + * block has been started, and clients should wait for it to + * complete before proceeding with further file updates. + * -1 on error. + */ + LIBHDFS_EXTERNAL + int hdfsTruncateFile(hdfsFS fs, const char* path, tOffset newlength); + + /** + * hdfsUnbufferFile - Reduce the buffering done on a file. + * + * @param file The file to unbuffer. + * @return 0 on success + * ENOTSUP if the file does not support unbuffering + * Errno will also be set to this value. + */ + LIBHDFS_EXTERNAL + int hdfsUnbufferFile(hdfsFile file); + + /** + * hdfsCloseFile - Close an open file. + * @param fs The configured filesystem handle. + * @param file The file handle. + * @return Returns 0 on success, -1 on error. + * On error, errno will be set appropriately. + * If the hdfs file was valid, the memory associated with it will + * be freed at the end of this call, even if there was an I/O + * error. + */ + LIBHDFS_EXTERNAL + int hdfsCloseFile(hdfsFS fs, hdfsFile file); + + + /** + * hdfsExists - Checks if a given path exsits on the filesystem + * @param fs The configured filesystem handle. + * @param path The path to look for + * @return Returns 0 on success, -1 on error. + */ + LIBHDFS_EXTERNAL + int hdfsExists(hdfsFS fs, const char *path); + + + /** + * hdfsSeek - Seek to given offset in file. + * This works only for files opened in read-only mode. + * @param fs The configured filesystem handle. + * @param file The file handle. + * @param desiredPos Offset into the file to seek into. + * @return Returns 0 on success, -1 on error. + */ + LIBHDFS_EXTERNAL + int hdfsSeek(hdfsFS fs, hdfsFile file, tOffset desiredPos); + + + /** + * hdfsTell - Get the current offset in the file, in bytes. + * @param fs The configured filesystem handle. + * @param file The file handle. + * @return Current offset, -1 on error. + */ + LIBHDFS_EXTERNAL + tOffset hdfsTell(hdfsFS fs, hdfsFile file); + + + /** + * hdfsRead - Read data from an open file. + * @param fs The configured filesystem handle. + * @param file The file handle. + * @param buffer The buffer to copy read bytes into. + * @param length The length of the buffer. + * @return On success, a positive number indicating how many bytes + * were read. + * On end-of-file, 0. + * On error, -1. Errno will be set to the error code. + * Just like the POSIX read function, hdfsRead will return -1 + * and set errno to EINTR if data is temporarily unavailable, + * but we are not yet at the end of the file. + */ + LIBHDFS_EXTERNAL + tSize hdfsRead(hdfsFS fs, hdfsFile file, void* buffer, tSize length); + + /** + * hdfsPread - Positional read of data from an open file. Reads up to the + * number of specified bytes in length. + * @param fs The configured filesystem handle. + * @param file The file handle. + * @param position Position from which to read + * @param buffer The buffer to copy read bytes into. + * @param length The length of the buffer. + * @return See hdfsRead + */ + LIBHDFS_EXTERNAL + tSize hdfsPread(hdfsFS fs, hdfsFile file, tOffset position, + void* buffer, tSize length); + + /** + * hdfsPreadFully - Positional read of data from an open file. Reads the + * number of specified bytes in length, or until the end of the data is + * reached. Unlike hdfsRead and hdfsPread, this method does not return + * the number of bytes read because either (1) the entire length of the + * buffer is filled, or (2) the end of the file is reached. If the eof is + * reached, an exception is thrown and errno is set to EINTR. + * @param fs The configured filesystem handle. + * @param file The file handle. + * @param position Position from which to read + * @param buffer The buffer to copy read bytes into. + * @param length The length of the buffer. + * @return Returns 0 on success, -1 on error. + */ + LIBHDFS_EXTERNAL + int hdfsPreadFully(hdfsFS fs, hdfsFile file, tOffset position, + void* buffer, tSize length); + + + /** + * hdfsWrite - Write data into an open file. + * @param fs The configured filesystem handle. + * @param file The file handle. + * @param buffer The data. + * @param length The no. of bytes to write. + * @return Returns the number of bytes written, -1 on error. + */ + LIBHDFS_EXTERNAL + tSize hdfsWrite(hdfsFS fs, hdfsFile file, const void* buffer, + tSize length); + + + /** + * hdfsWrite - Flush the data. + * @param fs The configured filesystem handle. + * @param file The file handle. + * @return Returns 0 on success, -1 on error. + */ + LIBHDFS_EXTERNAL + int hdfsFlush(hdfsFS fs, hdfsFile file); + + + /** + * hdfsHFlush - Flush out the data in client's user buffer. After the + * return of this call, new readers will see the data. + * @param fs configured filesystem handle + * @param file file handle + * @return 0 on success, -1 on error and sets errno + */ + LIBHDFS_EXTERNAL + int hdfsHFlush(hdfsFS fs, hdfsFile file); + + + /** + * hdfsHSync - Similar to posix fsync, Flush out the data in client's + * user buffer. all the way to the disk device (but the disk may have + * it in its cache). + * @param fs configured filesystem handle + * @param file file handle + * @return 0 on success, -1 on error and sets errno + */ + LIBHDFS_EXTERNAL + int hdfsHSync(hdfsFS fs, hdfsFile file); + + + /** + * hdfsAvailable - Number of bytes that can be read from this + * input stream without blocking. + * @param fs The configured filesystem handle. + * @param file The file handle. + * @return Returns available bytes; -1 on error. + */ + LIBHDFS_EXTERNAL + int hdfsAvailable(hdfsFS fs, hdfsFile file); + + + /** + * hdfsCopy - Copy file from one filesystem to another. + * @param srcFS The handle to source filesystem. + * @param src The path of source file. + * @param dstFS The handle to destination filesystem. + * @param dst The path of destination file. + * @return Returns 0 on success, -1 on error. + */ + LIBHDFS_EXTERNAL + int hdfsCopy(hdfsFS srcFS, const char* src, hdfsFS dstFS, const char* dst); + + + /** + * hdfsMove - Move file from one filesystem to another. + * @param srcFS The handle to source filesystem. + * @param src The path of source file. + * @param dstFS The handle to destination filesystem. + * @param dst The path of destination file. + * @return Returns 0 on success, -1 on error. + */ + LIBHDFS_EXTERNAL + int hdfsMove(hdfsFS srcFS, const char* src, hdfsFS dstFS, const char* dst); + + + /** + * hdfsDelete - Delete file. + * @param fs The configured filesystem handle. + * @param path The path of the file. + * @param recursive if path is a directory and set to + * non-zero, the directory is deleted else throws an exception. In + * case of a file the recursive argument is irrelevant. + * @return Returns 0 on success, -1 on error. + */ + LIBHDFS_EXTERNAL + int hdfsDelete(hdfsFS fs, const char* path, int recursive); + + /** + * hdfsRename - Rename file. + * @param fs The configured filesystem handle. + * @param oldPath The path of the source file. + * @param newPath The path of the destination file. + * @return Returns 0 on success, -1 on error. + */ + LIBHDFS_EXTERNAL + int hdfsRename(hdfsFS fs, const char* oldPath, const char* newPath); + + + /** + * hdfsGetWorkingDirectory - Get the current working directory for + * the given filesystem. + * @param fs The configured filesystem handle. + * @param buffer The user-buffer to copy path of cwd into. + * @param bufferSize The length of user-buffer. + * @return Returns buffer, NULL on error. + */ + LIBHDFS_EXTERNAL + char* hdfsGetWorkingDirectory(hdfsFS fs, char *buffer, size_t bufferSize); + + + /** + * hdfsSetWorkingDirectory - Set the working directory. All relative + * paths will be resolved relative to it. + * @param fs The configured filesystem handle. + * @param path The path of the new 'cwd'. + * @return Returns 0 on success, -1 on error. + */ + LIBHDFS_EXTERNAL + int hdfsSetWorkingDirectory(hdfsFS fs, const char* path); + + + /** + * hdfsCreateDirectory - Make the given file and all non-existent + * parents into directories. + * @param fs The configured filesystem handle. + * @param path The path of the directory. + * @return Returns 0 on success, -1 on error. + */ + LIBHDFS_EXTERNAL + int hdfsCreateDirectory(hdfsFS fs, const char* path); + + + /** + * hdfsSetReplication - Set the replication of the specified + * file to the supplied value + * @param fs The configured filesystem handle. + * @param path The path of the file. + * @return Returns 0 on success, -1 on error. + */ + LIBHDFS_EXTERNAL + int hdfsSetReplication(hdfsFS fs, const char* path, int16_t replication); + + + /** + * hdfsFileInfo - Information about a file/directory. + */ + typedef struct { + tObjectKind mKind; /* file or directory */ + char *mName; /* the name of the file */ + tTime mLastMod; /* the last modification time for the file in seconds */ + tOffset mSize; /* the size of the file in bytes */ + short mReplication; /* the count of replicas */ + tOffset mBlockSize; /* the block size for the file */ + char *mOwner; /* the owner of the file */ + char *mGroup; /* the group associated with the file */ + short mPermissions; /* the permissions associated with the file */ + tTime mLastAccess; /* the last access time for the file in seconds */ + } hdfsFileInfo; + + + /** + * hdfsListDirectory - Get list of files/directories for a given + * directory-path. hdfsFreeFileInfo should be called to deallocate memory. + * @param fs The configured filesystem handle. + * @param path The path of the directory. + * @param numEntries Set to the number of files/directories in path. + * @return Returns a dynamically-allocated array of hdfsFileInfo + * objects; NULL on error or empty directory. + * errno is set to non-zero on error or zero on success. + */ + LIBHDFS_EXTERNAL + hdfsFileInfo *hdfsListDirectory(hdfsFS fs, const char* path, + int *numEntries); + + + /** + * hdfsGetPathInfo - Get information about a path as a (dynamically + * allocated) single hdfsFileInfo struct. hdfsFreeFileInfo should be + * called when the pointer is no longer needed. + * @param fs The configured filesystem handle. + * @param path The path of the file. + * @return Returns a dynamically-allocated hdfsFileInfo object; + * NULL on error. + */ + LIBHDFS_EXTERNAL + hdfsFileInfo *hdfsGetPathInfo(hdfsFS fs, const char* path); + + + /** + * hdfsFreeFileInfo - Free up the hdfsFileInfo array (including fields) + * @param hdfsFileInfo The array of dynamically-allocated hdfsFileInfo + * objects. + * @param numEntries The size of the array. + */ + LIBHDFS_EXTERNAL + void hdfsFreeFileInfo(hdfsFileInfo *hdfsFileInfo, int numEntries); + + /** + * hdfsFileIsEncrypted: determine if a file is encrypted based on its + * hdfsFileInfo. + * @return -1 if there was an error (errno will be set), 0 if the file is + * not encrypted, 1 if the file is encrypted. + */ + LIBHDFS_EXTERNAL + int hdfsFileIsEncrypted(hdfsFileInfo *hdfsFileInfo); + + + /** + * hdfsGetHosts - Get hostnames where a particular block (determined by + * pos & blocksize) of a file is stored. The last element in the array + * is NULL. Due to replication, a single block could be present on + * multiple hosts. + * @param fs The configured filesystem handle. + * @param path The path of the file. + * @param start The start of the block. + * @param length The length of the block. + * @return Returns a dynamically-allocated 2-d array of blocks-hosts; + * NULL on error. + */ + LIBHDFS_EXTERNAL + char*** hdfsGetHosts(hdfsFS fs, const char* path, + tOffset start, tOffset length); + + + /** + * hdfsFreeHosts - Free up the structure returned by hdfsGetHosts + * @param hdfsFileInfo The array of dynamically-allocated hdfsFileInfo + * objects. + * @param numEntries The size of the array. + */ + LIBHDFS_EXTERNAL + void hdfsFreeHosts(char ***blockHosts); + + + /** + * hdfsGetDefaultBlockSize - Get the default blocksize. + * + * @param fs The configured filesystem handle. + * @deprecated Use hdfsGetDefaultBlockSizeAtPath instead. + * + * @return Returns the default blocksize, or -1 on error. + */ + LIBHDFS_EXTERNAL + tOffset hdfsGetDefaultBlockSize(hdfsFS fs); + + + /** + * hdfsGetDefaultBlockSizeAtPath - Get the default blocksize at the + * filesystem indicated by a given path. + * + * @param fs The configured filesystem handle. + * @param path The given path will be used to locate the actual + * filesystem. The full path does not have to exist. + * + * @return Returns the default blocksize, or -1 on error. + */ + LIBHDFS_EXTERNAL + tOffset hdfsGetDefaultBlockSizeAtPath(hdfsFS fs, const char *path); + + + /** + * hdfsGetCapacity - Return the raw capacity of the filesystem. + * @param fs The configured filesystem handle. + * @return Returns the raw-capacity; -1 on error. + */ + LIBHDFS_EXTERNAL + tOffset hdfsGetCapacity(hdfsFS fs); + + + /** + * hdfsGetUsed - Return the total raw size of all files in the filesystem. + * @param fs The configured filesystem handle. + * @return Returns the total-size; -1 on error. + */ + LIBHDFS_EXTERNAL + tOffset hdfsGetUsed(hdfsFS fs); + + /** + * Change the user and/or group of a file or directory. + * + * @param fs The configured filesystem handle. + * @param path the path to the file or directory + * @param owner User string. Set to NULL for 'no change' + * @param group Group string. Set to NULL for 'no change' + * @return 0 on success else -1 + */ + LIBHDFS_EXTERNAL + int hdfsChown(hdfsFS fs, const char* path, const char *owner, + const char *group); + + /** + * hdfsChmod + * @param fs The configured filesystem handle. + * @param path the path to the file or directory + * @param mode the bitmask to set it to + * @return 0 on success else -1 + */ + LIBHDFS_EXTERNAL + int hdfsChmod(hdfsFS fs, const char* path, short mode); + + /** + * hdfsUtime + * @param fs The configured filesystem handle. + * @param path the path to the file or directory + * @param mtime new modification time or -1 for no change + * @param atime new access time or -1 for no change + * @return 0 on success else -1 + */ + LIBHDFS_EXTERNAL + int hdfsUtime(hdfsFS fs, const char* path, tTime mtime, tTime atime); + + /** + * Allocate a zero-copy options structure. + * + * You must free all options structures allocated with this function using + * hadoopRzOptionsFree. + * + * @return A zero-copy options structure, or NULL if one could + * not be allocated. If NULL is returned, errno will + * contain the error number. + */ + LIBHDFS_EXTERNAL + struct hadoopRzOptions *hadoopRzOptionsAlloc(void); + + /** + * Determine whether we should skip checksums in read0. + * + * @param opts The options structure. + * @param skip Nonzero to skip checksums sometimes; zero to always + * check them. + * + * @return 0 on success; -1 plus errno on failure. + */ + LIBHDFS_EXTERNAL + int hadoopRzOptionsSetSkipChecksum( + struct hadoopRzOptions *opts, int skip); + + /** + * Set the ByteBufferPool to use with read0. + * + * @param opts The options structure. + * @param className If this is NULL, we will not use any + * ByteBufferPool. If this is non-NULL, it will be + * treated as the name of the pool class to use. + * For example, you can use + * ELASTIC_BYTE_BUFFER_POOL_CLASS. + * + * @return 0 if the ByteBufferPool class was found and + * instantiated; + * -1 plus errno otherwise. + */ + LIBHDFS_EXTERNAL + int hadoopRzOptionsSetByteBufferPool( + struct hadoopRzOptions *opts, const char *className); + + /** + * Free a hadoopRzOptionsFree structure. + * + * @param opts The options structure to free. + * Any associated ByteBufferPool will also be freed. + */ + LIBHDFS_EXTERNAL + void hadoopRzOptionsFree(struct hadoopRzOptions *opts); + + /** + * Perform a byte buffer read. + * If possible, this will be a zero-copy (mmap) read. + * + * @param file The file to read from. + * @param opts An options structure created by hadoopRzOptionsAlloc. + * @param maxLength The maximum length to read. We may read fewer bytes + * than this length. + * + * @return On success, we will return a new hadoopRzBuffer. + * This buffer will continue to be valid and readable + * until it is released by readZeroBufferFree. Failure to + * release a buffer will lead to a memory leak. + * You can access the data within the hadoopRzBuffer with + * hadoopRzBufferGet. If you have reached EOF, the data + * within the hadoopRzBuffer will be NULL. You must still + * free hadoopRzBuffer instances containing NULL. + * + * On failure, we will return NULL plus an errno code. + * errno = EOPNOTSUPP indicates that we could not do a + * zero-copy read, and there was no ByteBufferPool + * supplied. + */ + LIBHDFS_EXTERNAL + struct hadoopRzBuffer* hadoopReadZero(hdfsFile file, + struct hadoopRzOptions *opts, int32_t maxLength); + + /** + * Determine the length of the buffer returned from readZero. + * + * @param buffer a buffer returned from readZero. + * @return the length of the buffer. + */ + LIBHDFS_EXTERNAL + int32_t hadoopRzBufferLength(const struct hadoopRzBuffer *buffer); + + /** + * Get a pointer to the raw buffer returned from readZero. + * + * To find out how many bytes this buffer contains, call + * hadoopRzBufferLength. + * + * @param buffer a buffer returned from readZero. + * @return a pointer to the start of the buffer. This will be + * NULL when end-of-file has been reached. + */ + LIBHDFS_EXTERNAL + const void *hadoopRzBufferGet(const struct hadoopRzBuffer *buffer); + + /** + * Release a buffer obtained through readZero. + * + * @param file The hdfs stream that created this buffer. This must be + * the same stream you called hadoopReadZero on. + * @param buffer The buffer to release. + */ + LIBHDFS_EXTERNAL + void hadoopRzBufferFree(hdfsFile file, struct hadoopRzBuffer *buffer); + + /** + * Get the last exception root cause that happened in the context of the + * current thread, i.e. the thread that called into libHDFS. + * + * The pointer returned by this function is guaranteed to be valid until + * the next call into libHDFS by the current thread. + * Users of this function should not free the pointer. + * + * A NULL will be returned if no exception information could be retrieved + * for the previous call. + * + * @return The root cause as a C-string. + */ + LIBHDFS_EXTERNAL + char* hdfsGetLastExceptionRootCause(); + + /** + * Get the last exception stack trace that happened in the context of the + * current thread, i.e. the thread that called into libHDFS. + * + * The pointer returned by this function is guaranteed to be valid until + * the next call into libHDFS by the current thread. + * Users of this function should not free the pointer. + * + * A NULL will be returned if no exception information could be retrieved + * for the previous call. + * + * @return The stack trace as a C-string. + */ + LIBHDFS_EXTERNAL + char* hdfsGetLastExceptionStackTrace(); + +#ifdef __cplusplus +} +#endif + +#undef LIBHDFS_EXTERNAL +#endif /*LIBHDFS_HDFS_H*/ + +/** + * vim: ts=4: sw=4: et + */ diff --git a/native/hdfs-sys/libhdfs/hdfs_3_3/jclasses.c b/native/hdfs-sys/libhdfs/hdfs_3_3/jclasses.c new file mode 100644 index 0000000000..cf880e91b7 --- /dev/null +++ b/native/hdfs-sys/libhdfs/hdfs_3_3/jclasses.c @@ -0,0 +1,136 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "exception.h" +#include "jclasses.h" +#include "jni_helper.h" +#include "os/mutexes.h" + +#include + +/** + * Whether initCachedClasses has been called or not. Protected by the mutex + * jclassInitMutex. + */ +static int jclassesInitialized = 0; + +typedef struct { + jclass javaClass; + const char *className; +} javaClassAndName; + +/** + * A collection of commonly used jclass objects that are used throughout + * libhdfs. The jclasses are loaded immediately after the JVM is created (see + * initCachedClasses). The array is indexed using CachedJavaClass. + */ +javaClassAndName cachedJavaClasses[NUM_CACHED_CLASSES]; + +/** + * Helper method that creates and sets a jclass object given a class name. + * Returns a jthrowable on error, NULL otherwise. + */ +static jthrowable initCachedClass(JNIEnv *env, const char *className, + jclass *cachedJclass) { + assert(className != NULL && "Found a CachedJavaClass without a class " + "name"); + jthrowable jthr = NULL; + jclass tempLocalClassRef; + tempLocalClassRef = (*env)->FindClass(env, className); + if (!tempLocalClassRef) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + *cachedJclass = (jclass) (*env)->NewGlobalRef(env, tempLocalClassRef); + if (!*cachedJclass) { + jthr = getPendingExceptionAndClear(env); + goto done; + } +done: + destroyLocalReference(env, tempLocalClassRef); + return jthr; +} + +jthrowable initCachedClasses(JNIEnv* env) { + mutexLock(&jclassInitMutex); + if (!jclassesInitialized) { + // Set all the class names + cachedJavaClasses[JC_CONFIGURATION].className = + "org/apache/hadoop/conf/Configuration"; + cachedJavaClasses[JC_PATH].className = + "org/apache/hadoop/fs/Path"; + cachedJavaClasses[JC_FILE_SYSTEM].className = + "org/apache/hadoop/fs/FileSystem"; + cachedJavaClasses[JC_FS_STATUS].className = + "org/apache/hadoop/fs/FsStatus"; + cachedJavaClasses[JC_FILE_UTIL].className = + "org/apache/hadoop/fs/FileUtil"; + cachedJavaClasses[JC_BLOCK_LOCATION].className = + "org/apache/hadoop/fs/BlockLocation"; + cachedJavaClasses[JC_DFS_HEDGED_READ_METRICS].className = + "org/apache/hadoop/hdfs/DFSHedgedReadMetrics"; + cachedJavaClasses[JC_DISTRIBUTED_FILE_SYSTEM].className = + "org/apache/hadoop/hdfs/DistributedFileSystem"; + cachedJavaClasses[JC_FS_DATA_INPUT_STREAM].className = + "org/apache/hadoop/fs/FSDataInputStream"; + cachedJavaClasses[JC_FS_DATA_OUTPUT_STREAM].className = + "org/apache/hadoop/fs/FSDataOutputStream"; + cachedJavaClasses[JC_FILE_STATUS].className = + "org/apache/hadoop/fs/FileStatus"; + cachedJavaClasses[JC_FS_PERMISSION].className = + "org/apache/hadoop/fs/permission/FsPermission"; + cachedJavaClasses[JC_READ_STATISTICS].className = + "org/apache/hadoop/hdfs/ReadStatistics"; + cachedJavaClasses[JC_HDFS_DATA_INPUT_STREAM].className = + "org/apache/hadoop/hdfs/client/HdfsDataInputStream"; + cachedJavaClasses[JC_DOMAIN_SOCKET].className = + "org/apache/hadoop/net/unix/DomainSocket"; + cachedJavaClasses[JC_URI].className = + "java/net/URI"; + cachedJavaClasses[JC_BYTE_BUFFER].className = + "java/nio/ByteBuffer"; + cachedJavaClasses[JC_ENUM_SET].className = + "java/util/EnumSet"; + cachedJavaClasses[JC_EXCEPTION_UTILS].className = + "org/apache/commons/lang3/exception/ExceptionUtils"; + + // Create and set the jclass objects based on the class names set above + jthrowable jthr; + int numCachedClasses = + sizeof(cachedJavaClasses) / sizeof(javaClassAndName); + for (int i = 0; i < numCachedClasses; i++) { + jthr = initCachedClass(env, cachedJavaClasses[i].className, + &cachedJavaClasses[i].javaClass); + if (jthr) { + mutexUnlock(&jclassInitMutex); + return jthr; + } + } + jclassesInitialized = 1; + } + mutexUnlock(&jclassInitMutex); + return NULL; +} + +jclass getJclass(CachedJavaClass cachedJavaClass) { + return cachedJavaClasses[cachedJavaClass].javaClass; +} + +const char *getClassName(CachedJavaClass cachedJavaClass) { + return cachedJavaClasses[cachedJavaClass].className; +} diff --git a/native/hdfs-sys/libhdfs/hdfs_3_3/jclasses.h b/native/hdfs-sys/libhdfs/hdfs_3_3/jclasses.h new file mode 100644 index 0000000000..92cdd542e2 --- /dev/null +++ b/native/hdfs-sys/libhdfs/hdfs_3_3/jclasses.h @@ -0,0 +1,112 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFS_JCLASSES_H +#define LIBHDFS_JCLASSES_H + +#include + +/** + * Encapsulates logic to cache jclass objects so they can re-used across + * calls to FindClass. Creating jclass objects every time libhdfs has to + * invoke a method can hurt performance. By cacheing jclass objects we avoid + * this overhead. + * + * We use the term "cached" here loosely; jclasses are not truly cached, + * instead they are created once during JVM load and are kept alive until the + * process shutdowns. There is no eviction of jclass objects. + * + * @see https://www.ibm.com/developerworks/library/j-jni/index.html#notc + */ + +/** + * Each enum value represents one jclass that is cached. Enum values should + * be passed to getJclass or getName to get the jclass object or class name + * represented by the enum value. + */ +typedef enum { + JC_CONFIGURATION, + JC_PATH, + JC_FILE_SYSTEM, + JC_FS_STATUS, + JC_FILE_UTIL, + JC_BLOCK_LOCATION, + JC_DFS_HEDGED_READ_METRICS, + JC_DISTRIBUTED_FILE_SYSTEM, + JC_FS_DATA_INPUT_STREAM, + JC_FS_DATA_OUTPUT_STREAM, + JC_FILE_STATUS, + JC_FS_PERMISSION, + JC_READ_STATISTICS, + JC_HDFS_DATA_INPUT_STREAM, + JC_DOMAIN_SOCKET, + JC_URI, + JC_BYTE_BUFFER, + JC_ENUM_SET, + JC_EXCEPTION_UTILS, + // A special marker enum that counts the number of cached jclasses + NUM_CACHED_CLASSES +} CachedJavaClass; + +/** + * Internally initializes all jclass objects listed in the CachedJavaClass + * enum. This method is idempotent and thread-safe. + */ +jthrowable initCachedClasses(JNIEnv* env); + +/** + * Return the jclass object represented by the given CachedJavaClass + */ +jclass getJclass(CachedJavaClass cachedJavaClass); + +/** + * Return the class name represented by the given CachedJavaClass + */ +const char *getClassName(CachedJavaClass cachedJavaClass); + +/* Some frequently used HDFS class names */ +#define HADOOP_CONF "org/apache/hadoop/conf/Configuration" +#define HADOOP_PATH "org/apache/hadoop/fs/Path" +#define HADOOP_LOCALFS "org/apache/hadoop/fs/LocalFileSystem" +#define HADOOP_FS "org/apache/hadoop/fs/FileSystem" +#define HADOOP_FSSTATUS "org/apache/hadoop/fs/FsStatus" +#define HADOOP_FILEUTIL "org/apache/hadoop/fs/FileUtil" +#define HADOOP_BLK_LOC "org/apache/hadoop/fs/BlockLocation" +#define HADOOP_DFS_HRM "org/apache/hadoop/hdfs/DFSHedgedReadMetrics" +#define HADOOP_DFS "org/apache/hadoop/hdfs/DistributedFileSystem" +#define HADOOP_FSDISTRM "org/apache/hadoop/fs/FSDataInputStream" +#define HADOOP_FSDOSTRM "org/apache/hadoop/fs/FSDataOutputStream" +#define HADOOP_FILESTAT "org/apache/hadoop/fs/FileStatus" +#define HADOOP_FSPERM "org/apache/hadoop/fs/permission/FsPermission" +#define HADOOP_RSTAT "org/apache/hadoop/hdfs/ReadStatistics" +#define HADOOP_HDISTRM "org/apache/hadoop/hdfs/client/HdfsDataInputStream" +#define HADOOP_RO "org/apache/hadoop/fs/ReadOption" +#define HADOOP_DS "org/apache/hadoop/net/unix/DomainSocket" + +/* Some frequently used Java class names */ +#define JAVA_NET_ISA "java/net/InetSocketAddress" +#define JAVA_NET_URI "java/net/URI" +#define JAVA_BYTEBUFFER "java/nio/ByteBuffer" +#define JAVA_STRING "java/lang/String" +#define JAVA_ENUMSET "java/util/EnumSet" + +/* Some frequently used third-party class names */ + +#define EXCEPTION_UTILS "org/apache/commons/lang3/exception/ExceptionUtils" + +#endif /*LIBHDFS_JCLASSES_H*/ diff --git a/native/hdfs-sys/libhdfs/hdfs_3_3/jni_helper.c b/native/hdfs-sys/libhdfs/hdfs_3_3/jni_helper.c new file mode 100644 index 0000000000..badb041df7 --- /dev/null +++ b/native/hdfs-sys/libhdfs/hdfs_3_3/jni_helper.c @@ -0,0 +1,994 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * This file was modified by the Apache DataFusion Comet project. See + * native/hdfs-sys/README.md for the provenance of this directory and for the + * full list of modifications. The change here is that getGlobalJNIEnv reports + * whether it attached the current thread, and reuses an existing attachment + * instead of calling AttachCurrentThread on a thread that already has one. + */ + +#include "config.h" +#include "exception.h" +#include "jclasses.h" +#include "jni_helper.h" +#include "platform.h" +#include "os/mutexes.h" +#include "os/thread_local_storage.h" + +#include +#include +#include +#include + +/** The Native return types that methods could return */ +#define JVOID 'V' +#define JOBJECT 'L' +#define JARRAYOBJECT '[' +#define JBOOLEAN 'Z' +#define JBYTE 'B' +#define JCHAR 'C' +#define JSHORT 'S' +#define JINT 'I' +#define JLONG 'J' +#define JFLOAT 'F' +#define JDOUBLE 'D' + +/** + * Length of buffer for retrieving created JVMs. (We only ever create one.) + */ +#define VM_BUF_LENGTH 1 + +void destroyLocalReference(JNIEnv *env, jobject jObject) +{ + if (jObject) + (*env)->DeleteLocalRef(env, jObject); +} + +static jthrowable validateMethodType(JNIEnv *env, MethType methType) +{ + if (methType != STATIC && methType != INSTANCE) { + return newRuntimeError(env, "validateMethodType(methType=%d): " + "illegal method type.\n", methType); + } + return NULL; +} + +jthrowable newJavaStr(JNIEnv *env, const char *str, jstring *out) +{ + jstring jstr; + + if (!str) { + /* Can't pass NULL to NewStringUTF: the result would be + * implementation-defined. */ + *out = NULL; + return NULL; + } + jstr = (*env)->NewStringUTF(env, str); + if (!jstr) { + /* If NewStringUTF returns NULL, an exception has been thrown, + * which we need to handle. Probaly an OOM. */ + return getPendingExceptionAndClear(env); + } + *out = jstr; + return NULL; +} + +jthrowable newCStr(JNIEnv *env, jstring jstr, char **out) +{ + const char *tmp; + + if (!jstr) { + *out = NULL; + return NULL; + } + tmp = (*env)->GetStringUTFChars(env, jstr, NULL); + if (!tmp) { + return getPendingExceptionAndClear(env); + } + *out = strdup(tmp); + (*env)->ReleaseStringUTFChars(env, jstr, tmp); + return NULL; +} + +/** + * Does the work to actually execute a Java method. Takes in an existing jclass + * object and a va_list of arguments for the Java method to be invoked. + */ +static jthrowable invokeMethodOnJclass(JNIEnv *env, jvalue *retval, + MethType methType, jobject instObj, jclass cls, const char *className, + const char *methName, const char *methSignature, va_list args) +{ + jmethodID mid; + jthrowable jthr; + const char *str; + char returnType; + + jthr = methodIdFromClass(cls, className, methName, methSignature, methType, + env, &mid); + if (jthr) + return jthr; + str = methSignature; + while (*str != ')') str++; + str++; + returnType = *str; + if (returnType == JOBJECT || returnType == JARRAYOBJECT) { + jobject jobj = NULL; + if (methType == STATIC) { + jobj = (*env)->CallStaticObjectMethodV(env, cls, mid, args); + } + else if (methType == INSTANCE) { + jobj = (*env)->CallObjectMethodV(env, instObj, mid, args); + } + retval->l = jobj; + } + else if (returnType == JVOID) { + if (methType == STATIC) { + (*env)->CallStaticVoidMethodV(env, cls, mid, args); + } + else if (methType == INSTANCE) { + (*env)->CallVoidMethodV(env, instObj, mid, args); + } + } + else if (returnType == JBOOLEAN) { + jboolean jbool = 0; + if (methType == STATIC) { + jbool = (*env)->CallStaticBooleanMethodV(env, cls, mid, args); + } + else if (methType == INSTANCE) { + jbool = (*env)->CallBooleanMethodV(env, instObj, mid, args); + } + retval->z = jbool; + } + else if (returnType == JSHORT) { + jshort js = 0; + if (methType == STATIC) { + js = (*env)->CallStaticShortMethodV(env, cls, mid, args); + } + else if (methType == INSTANCE) { + js = (*env)->CallShortMethodV(env, instObj, mid, args); + } + retval->s = js; + } + else if (returnType == JLONG) { + jlong jl = -1; + if (methType == STATIC) { + jl = (*env)->CallStaticLongMethodV(env, cls, mid, args); + } + else if (methType == INSTANCE) { + jl = (*env)->CallLongMethodV(env, instObj, mid, args); + } + retval->j = jl; + } + else if (returnType == JINT) { + jint ji = -1; + if (methType == STATIC) { + ji = (*env)->CallStaticIntMethodV(env, cls, mid, args); + } + else if (methType == INSTANCE) { + ji = (*env)->CallIntMethodV(env, instObj, mid, args); + } + retval->i = ji; + } + + jthr = (*env)->ExceptionOccurred(env); + if (jthr) { + (*env)->ExceptionClear(env); + return jthr; + } + return NULL; +} + +jthrowable findClassAndInvokeMethod(JNIEnv *env, jvalue *retval, + MethType methType, jobject instObj, const char *className, + const char *methName, const char *methSignature, ...) +{ + jclass cls = NULL; + jthrowable jthr = NULL; + + va_list args; + va_start(args, methSignature); + + jthr = validateMethodType(env, methType); + if (jthr) { + goto done; + } + + cls = (*env)->FindClass(env, className); + if (!cls) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + + jthr = invokeMethodOnJclass(env, retval, methType, instObj, cls, + className, methName, methSignature, args); + +done: + va_end(args); + destroyLocalReference(env, cls); + return jthr; +} + +jthrowable invokeMethod(JNIEnv *env, jvalue *retval, MethType methType, + jobject instObj, CachedJavaClass class, + const char *methName, const char *methSignature, ...) +{ + jthrowable jthr; + + va_list args; + va_start(args, methSignature); + + jthr = invokeMethodOnJclass(env, retval, methType, instObj, + getJclass(class), getClassName(class), methName, methSignature, + args); + + va_end(args); + return jthr; +} + +static jthrowable constructNewObjectOfJclass(JNIEnv *env, + jobject *out, jclass cls, const char *className, + const char *ctorSignature, va_list args) { + jmethodID mid; + jobject jobj; + jthrowable jthr; + + jthr = methodIdFromClass(cls, className, "", ctorSignature, INSTANCE, + env, &mid); + if (jthr) + return jthr; + jobj = (*env)->NewObjectV(env, cls, mid, args); + if (!jobj) + return getPendingExceptionAndClear(env); + *out = jobj; + return NULL; +} + +jthrowable constructNewObjectOfClass(JNIEnv *env, jobject *out, + const char *className, const char *ctorSignature, ...) +{ + va_list args; + jclass cls; + jthrowable jthr = NULL; + + cls = (*env)->FindClass(env, className); + if (!cls) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + + va_start(args, ctorSignature); + jthr = constructNewObjectOfJclass(env, out, cls, className, + ctorSignature, args); + va_end(args); +done: + destroyLocalReference(env, cls); + return jthr; +} + +jthrowable constructNewObjectOfCachedClass(JNIEnv *env, jobject *out, + CachedJavaClass cachedJavaClass, const char *ctorSignature, ...) +{ + jthrowable jthr = NULL; + va_list args; + va_start(args, ctorSignature); + + jthr = constructNewObjectOfJclass(env, out, + getJclass(cachedJavaClass), getClassName(cachedJavaClass), + ctorSignature, args); + + va_end(args); + return jthr; +} + +jthrowable methodIdFromClass(jclass cls, const char *className, + const char *methName, const char *methSignature, MethType methType, + JNIEnv *env, jmethodID *out) +{ + jthrowable jthr; + jmethodID mid = 0; + + jthr = validateMethodType(env, methType); + if (jthr) + return jthr; + if (methType == STATIC) { + mid = (*env)->GetStaticMethodID(env, cls, methName, methSignature); + } + else if (methType == INSTANCE) { + mid = (*env)->GetMethodID(env, cls, methName, methSignature); + } + if (mid == NULL) { + fprintf(stderr, "could not find method %s from class %s with " + "signature %s\n", methName, className, methSignature); + return getPendingExceptionAndClear(env); + } + *out = mid; + return NULL; +} + +jthrowable classNameOfObject(jobject jobj, JNIEnv *env, char **name) +{ + jthrowable jthr; + jclass cls, clsClass = NULL; + jmethodID mid; + jstring str = NULL; + const char *cstr = NULL; + char *newstr; + + cls = (*env)->GetObjectClass(env, jobj); + if (cls == NULL) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + clsClass = (*env)->FindClass(env, "java/lang/Class"); + if (clsClass == NULL) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + mid = (*env)->GetMethodID(env, clsClass, "getName", "()Ljava/lang/String;"); + if (mid == NULL) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + str = (*env)->CallObjectMethod(env, cls, mid); + jthr = (*env)->ExceptionOccurred(env); + if (jthr) { + (*env)->ExceptionClear(env); + goto done; + } + if (str == NULL) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + cstr = (*env)->GetStringUTFChars(env, str, NULL); + if (!cstr) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + newstr = strdup(cstr); + if (newstr == NULL) { + jthr = newRuntimeError(env, "classNameOfObject: out of memory"); + goto done; + } + *name = newstr; + jthr = NULL; + +done: + destroyLocalReference(env, cls); + destroyLocalReference(env, clsClass); + if (str) { + if (cstr) + (*env)->ReleaseStringUTFChars(env, str, cstr); + (*env)->DeleteLocalRef(env, str); + } + return jthr; +} + +/** + * For the given path, expand it by filling in with all *.jar or *.JAR files, + * separated by PATH_SEPARATOR. Assumes that expanded is big enough to hold the + * string, eg allocated after using this function with expanded=NULL to get the + * right size. Also assumes that the path ends with a "/.". The length of the + * expanded path is returned, which includes space at the end for either a + * PATH_SEPARATOR or null terminator. + */ +static ssize_t wildcard_expandPath(const char* path, char* expanded) +{ + struct dirent* file; + char* dest = expanded; + ssize_t length = 0; + size_t pathLength = strlen(path); + DIR* dir; + + dir = opendir(path); + if (dir != NULL) { + // can open dir so try to match with all *.jar and *.JAR entries + +#ifdef _LIBHDFS_JNI_HELPER_DEBUGGING_ON_ + printf("wildcard_expandPath: %s\n", path); +#endif + + errno = 0; + while ((file = readdir(dir)) != NULL) { + const char* filename = file->d_name; + const size_t filenameLength = strlen(filename); + const char* jarExtension; + + // If filename is smaller than 4 characters then it can not possibly + // have extension ".jar" or ".JAR" + if (filenameLength < 4) { + continue; + } + + jarExtension = &filename[filenameLength-4]; + if ((strcmp(jarExtension, ".jar") == 0) || + (strcmp(jarExtension, ".JAR") == 0)) { + + // pathLength includes an extra '.' which we'll use for either + // separator or null termination + length += pathLength + filenameLength; + +#ifdef _LIBHDFS_JNI_HELPER_DEBUGGING_ON_ + printf("wildcard_scanPath:\t%s\t:\t%zd\n", filename, length); +#endif + + if (expanded != NULL) { + // pathLength includes an extra '.' + strncpy(dest, path, pathLength-1); + dest += pathLength - 1; + strncpy(dest, filename, filenameLength); + dest += filenameLength; + *dest = PATH_SEPARATOR; + dest++; + +#ifdef _LIBHDFS_JNI_HELPER_DEBUGGING_ON_ + printf("wildcard_expandPath:\t%s\t:\t%s\n", + filename, expanded); +#endif + } + } + } + + if (errno != 0) { + fprintf(stderr, "wildcard_expandPath: on readdir %s: %s\n", + path, strerror(errno)); + length = -1; + } + + if (closedir(dir) != 0) { + fprintf(stderr, "wildcard_expandPath: on closedir %s: %s\n", + path, strerror(errno)); + } + } else if ((errno != EACCES) && (errno != ENOENT) && (errno != ENOTDIR)) { + // can not opendir due to an error we can not handle + fprintf(stderr, "wildcard_expandPath: on opendir %s: %s\n", path, + strerror(errno)); + length = -1; + } + + if (length == 0) { + // either we failed to open dir due to EACCESS, ENOENT, or ENOTDIR, or + // we did not find any file that matches *.jar or *.JAR + +#ifdef _LIBHDFS_JNI_HELPER_DEBUGGING_ON_ + fprintf(stderr, "wildcard_expandPath: can not expand %.*s*: %s\n", + (int)(pathLength-1), path, strerror(errno)); +#endif + + // in this case, the wildcard expansion is the same as the original + // +1 for PATH_SEPARTOR or null termination + length = pathLength + 1; + if (expanded != NULL) { + // pathLength includes an extra '.' + strncpy(dest, path, pathLength-1); + dest += pathLength-1; + *dest = '*'; // restore wildcard + dest++; + *dest = PATH_SEPARATOR; + dest++; + } + } + + return length; +} + +/** + * Helper to expand classpaths. Returns the total length of the expanded + * classpath. If expandedClasspath is not NULL, then fills that with the + * expanded classpath. It assumes that expandedClasspath is of correct size, eg + * allocated after using this function with expandedClasspath=NULL to get the + * right size. + */ +static ssize_t getClassPath_helper(const char *classpath, char* expandedClasspath) +{ + ssize_t length; + ssize_t retval; + char* expandedCP_curr; + char* cp_token; + char* classpath_dup; + + classpath_dup = strdup(classpath); + if (classpath_dup == NULL) { + fprintf(stderr, "getClassPath_helper: failed strdup: %s\n", + strerror(errno)); + return -1; + } + + length = 0; + + // expandedCP_curr is the current pointer + expandedCP_curr = expandedClasspath; + + cp_token = strtok(classpath_dup, PATH_SEPARATOR_STR); + while (cp_token != NULL) { + size_t tokenlen; + +#ifdef _LIBHDFS_JNI_HELPER_DEBUGGING_ON_ + printf("%s\n", cp_token); +#endif + + tokenlen = strlen(cp_token); + // We only expand if token ends with "/*" + if ((tokenlen > 1) && + (cp_token[tokenlen-1] == '*') && (cp_token[tokenlen-2] == '/')) { + // replace the '*' with '.' so that we don't have to allocate another + // string for passing to opendir() in wildcard_expandPath() + cp_token[tokenlen-1] = '.'; + retval = wildcard_expandPath(cp_token, expandedCP_curr); + if (retval < 0) { + free(classpath_dup); + return -1; + } + + length += retval; + if (expandedCP_curr != NULL) { + expandedCP_curr += retval; + } + } else { + // +1 for path separator or null terminator + length += tokenlen + 1; + if (expandedCP_curr != NULL) { + strncpy(expandedCP_curr, cp_token, tokenlen); + expandedCP_curr += tokenlen; + *expandedCP_curr = PATH_SEPARATOR; + expandedCP_curr++; + } + } + + cp_token = strtok(NULL, PATH_SEPARATOR_STR); + } + + // Fix the last ':' and use it to null terminate + if (expandedCP_curr != NULL) { + expandedCP_curr--; + *expandedCP_curr = '\0'; + } + + free(classpath_dup); + return length; +} + +/** + * Gets the classpath. Wild card entries are resolved only if the entry ends + * with "/\*" (backslash to escape commenting) to match against .jar and .JAR. + * All other wild card entries (eg /path/to/dir/\*foo*) are not resolved, + * following JAVA default behavior, see: + * https://docs.oracle.com/javase/8/docs/technotes/tools/unix/classpath.html + */ +static char* getClassPath() +{ + char* classpath; + char* expandedClasspath; + ssize_t length; + ssize_t retval; + + classpath = getenv("CLASSPATH"); + if (classpath == NULL) { + return NULL; + } + + // First, get the total size of the string we will need for the expanded + // classpath + length = getClassPath_helper(classpath, NULL); + if (length < 0) { + return NULL; + } + +#ifdef _LIBHDFS_JNI_HELPER_DEBUGGING_ON_ + printf("+++++++++++++++++\n"); +#endif + + // we don't have to do anything if classpath has no valid wildcards + // we get length = 0 when CLASSPATH is set but empty + // if CLASSPATH is not empty, then length includes null terminator + // if length of expansion is same as original, then return a duplicate of + // original since expansion can only be longer + if ((length == 0) || ((length - 1) == strlen(classpath))) { + +#ifdef _LIBHDFS_JNI_HELPER_DEBUGGING_ON_ + if ((length == 0) && (strlen(classpath) != 0)) { + fprintf(stderr, "Something went wrong with getting the wildcard \ + expansion length\n" ); + } +#endif + + expandedClasspath = strdup(classpath); + +#ifdef _LIBHDFS_JNI_HELPER_DEBUGGING_ON_ + printf("Expanded classpath=%s\n", expandedClasspath); +#endif + + return expandedClasspath; + } + + // Allocte memory for expanded classpath string + expandedClasspath = calloc(length, sizeof(char)); + if (expandedClasspath == NULL) { + fprintf(stderr, "getClassPath: failed calloc: %s\n", strerror(errno)); + return NULL; + } + + // Actual expansion + retval = getClassPath_helper(classpath, expandedClasspath); + if (retval < 0) { + free(expandedClasspath); + return NULL; + } + + // This should not happen, but dotting i's and crossing t's + if (retval != length) { + fprintf(stderr, + "Expected classpath expansion length to be %zu but instead got %zu\n", + length, retval); + free(expandedClasspath); + return NULL; + } + +#ifdef _LIBHDFS_JNI_HELPER_DEBUGGING_ON_ + printf("===============\n"); + printf("Allocated %zd for expanding classpath\n", length); + printf("Used %zu for expanding classpath\n", strlen(expandedClasspath) + 1); + printf("Expanded classpath=%s\n", expandedClasspath); +#endif + + return expandedClasspath; +} + + +/** + * Get the global JNI environemnt. + * + * We only have to create the JVM once. After that, we can use it in + * every thread. You must be holding the jvmMutex when you call this + * function. + * + * @param[out] attachedByLibhdfs Set to true if this call attached the current + * thread to the JVM, false if the thread was + * already attached by someone else. Only the + * former may be detached at thread exit. + * + * @return The JNIEnv on success; error code otherwise + */ +static JNIEnv* getGlobalJNIEnv(bool *attachedByLibhdfs) +{ + JavaVM* vmBuf[VM_BUF_LENGTH]; + JNIEnv *env; + jint rv = 0; + jint noVMs = 0; + jthrowable jthr; + char *hadoopClassPath; + const char *hadoopClassPathVMArg = "-Djava.class.path="; + size_t optHadoopClassPathLen; + char *optHadoopClassPath; + int noArgs = 1; + char *hadoopJvmArgs; + char jvmArgDelims[] = " "; + char *str, *token, *savePtr; + JavaVMInitArgs vm_args; + JavaVM *vm; + JavaVMOption *options; + + *attachedByLibhdfs = false; + rv = JNI_GetCreatedJavaVMs(&(vmBuf[0]), VM_BUF_LENGTH, &noVMs); + if (rv != 0) { + fprintf(stderr, "JNI_GetCreatedJavaVMs failed with error: %d\n", rv); + return NULL; + } + + if (noVMs == 0) { + //Get the environment variables for initializing the JVM + hadoopClassPath = getClassPath(); + if (hadoopClassPath == NULL) { + fprintf(stderr, "Environment variable CLASSPATH not set!\n"); + return NULL; + } + optHadoopClassPathLen = strlen(hadoopClassPath) + + strlen(hadoopClassPathVMArg) + 1; + optHadoopClassPath = malloc(sizeof(char)*optHadoopClassPathLen); + snprintf(optHadoopClassPath, optHadoopClassPathLen, + "%s%s", hadoopClassPathVMArg, hadoopClassPath); + + free(hadoopClassPath); + + // Determine the # of LIBHDFS_OPTS args + hadoopJvmArgs = getenv("LIBHDFS_OPTS"); + if (hadoopJvmArgs != NULL) { + hadoopJvmArgs = strdup(hadoopJvmArgs); + for (noArgs = 1, str = hadoopJvmArgs; ; noArgs++, str = NULL) { + token = strtok_r(str, jvmArgDelims, &savePtr); + if (NULL == token) { + break; + } + } + free(hadoopJvmArgs); + } + + // Now that we know the # args, populate the options array + options = calloc(noArgs, sizeof(JavaVMOption)); + if (!options) { + fputs("Call to calloc failed\n", stderr); + free(optHadoopClassPath); + return NULL; + } + options[0].optionString = optHadoopClassPath; + hadoopJvmArgs = getenv("LIBHDFS_OPTS"); + if (hadoopJvmArgs != NULL) { + hadoopJvmArgs = strdup(hadoopJvmArgs); + for (noArgs = 1, str = hadoopJvmArgs; ; noArgs++, str = NULL) { + token = strtok_r(str, jvmArgDelims, &savePtr); + if (NULL == token) { + break; + } + options[noArgs].optionString = token; + } + } + + //Create the VM + vm_args.version = JNI_VERSION_1_2; + vm_args.options = options; + vm_args.nOptions = noArgs; + vm_args.ignoreUnrecognized = 1; + + rv = JNI_CreateJavaVM(&vm, (void*)&env, &vm_args); + + if (hadoopJvmArgs != NULL) { + free(hadoopJvmArgs); + } + free(optHadoopClassPath); + free(options); + + if (rv != 0) { + fprintf(stderr, "Call to JNI_CreateJavaVM failed " + "with error: %d\n", rv); + return NULL; + } + *attachedByLibhdfs = true; + + // We use findClassAndInvokeMethod here because the jclasses in + // jclasses.h have not loaded yet + jthr = findClassAndInvokeMethod(env, NULL, STATIC, NULL, HADOOP_FS, + "loadFileSystems", "()V"); + if (jthr) { + printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "FileSystem: loadFileSystems failed"); + return NULL; + } + } else { + vm = vmBuf[0]; + // Reuse an existing attachment rather than creating one. On a thread + // the JVM or the embedding application already attached, + // AttachCurrentThread succeeds and hands back the same JNIEnv, which + // would leave libhdfs believing it owns an attachment it did not make + // and detaching it at thread exit. Comet attaches its own Tokio worker + // threads, so that is the common case here rather than a corner one. + rv = (*vm)->GetEnv(vm, (void**)&env, JNI_VERSION_1_2); + if (rv == JNI_OK) { + return env; + } + if (rv != JNI_EDETACHED) { + fprintf(stderr, "Call to GetEnv failed with error: %d\n", rv); + return NULL; + } + //Attach this thread to the VM + rv = (*vm)->AttachCurrentThread(vm, (void*)&env, 0); + if (rv != 0) { + fprintf(stderr, "Call to AttachCurrentThread " + "failed with error: %d\n", rv); + return NULL; + } + *attachedByLibhdfs = true; + } + + return env; +} + +/** + * getJNIEnv: A helper function to get the JNIEnv* for the given thread. + * If no JVM exists, then one will be created. JVM command line arguments + * are obtained from the LIBHDFS_OPTS environment variable. + * + * Implementation note: we rely on POSIX thread-local storage (tls). + * This allows us to associate a destructor function with each thread, that + * will detach the thread from the Java VM when the thread terminates. If we + * failt to do this, it will cause a memory leak. + * + * However, POSIX TLS is not the most efficient way to do things. It requires a + * key to be initialized before it can be used. Since we don't know if this key + * is initialized at the start of this function, we have to lock a mutex first + * and check. Luckily, most operating systems support the more efficient + * __thread construct, which is initialized by the linker. + * + * @param: None. + * @return The JNIEnv* corresponding to the thread. + */ +JNIEnv* getJNIEnv(void) +{ + struct ThreadLocalState *state = NULL; + THREAD_LOCAL_STORAGE_GET_QUICK(&state); + if (state) return state->env; + + mutexLock(&jvmMutex); + if (threadLocalStorageGet(&state)) { + mutexUnlock(&jvmMutex); + return NULL; + } + if (state) { + mutexUnlock(&jvmMutex); + + // Free any stale exception strings. + free(state->lastExceptionRootCause); + free(state->lastExceptionStackTrace); + state->lastExceptionRootCause = NULL; + state->lastExceptionStackTrace = NULL; + + return state->env; + } + + /* Create a ThreadLocalState for this thread */ + state = threadLocalStorageCreate(); + if (!state) { + mutexUnlock(&jvmMutex); + fprintf(stderr, "getJNIEnv: Unable to create ThreadLocalState\n"); + return NULL; + } + if (threadLocalStorageSet(state)) { + mutexUnlock(&jvmMutex); + goto fail; + } + THREAD_LOCAL_STORAGE_SET_QUICK(state); + + state->env = getGlobalJNIEnv(&state->attachedByLibhdfs); + mutexUnlock(&jvmMutex); + + if (!state->env) { + goto fail; + } + + jthrowable jthr = NULL; + jthr = initCachedClasses(state->env); + if (jthr) { + printExceptionAndFree(state->env, jthr, PRINT_EXC_ALL, + "initCachedClasses failed"); + goto fail; + } + return state->env; + +fail: + fprintf(stderr, "getJNIEnv: getGlobalJNIEnv failed\n"); + hdfsThreadDestructor(state); + return NULL; +} + +char* getLastTLSExceptionRootCause() +{ + struct ThreadLocalState *state = NULL; + THREAD_LOCAL_STORAGE_GET_QUICK(&state); + if (!state) { + mutexLock(&jvmMutex); + if (threadLocalStorageGet(&state)) { + mutexUnlock(&jvmMutex); + return NULL; + } + mutexUnlock(&jvmMutex); + } + return state->lastExceptionRootCause; +} + +char* getLastTLSExceptionStackTrace() +{ + struct ThreadLocalState *state = NULL; + THREAD_LOCAL_STORAGE_GET_QUICK(&state); + if (!state) { + mutexLock(&jvmMutex); + if (threadLocalStorageGet(&state)) { + mutexUnlock(&jvmMutex); + return NULL; + } + mutexUnlock(&jvmMutex); + } + return state->lastExceptionStackTrace; +} + +void setTLSExceptionStrings(const char *rootCause, const char *stackTrace) +{ + struct ThreadLocalState *state = NULL; + THREAD_LOCAL_STORAGE_GET_QUICK(&state); + if (!state) { + mutexLock(&jvmMutex); + if (threadLocalStorageGet(&state)) { + mutexUnlock(&jvmMutex); + return; + } + mutexUnlock(&jvmMutex); + } + + free(state->lastExceptionRootCause); + free(state->lastExceptionStackTrace); + state->lastExceptionRootCause = (char*)rootCause; + state->lastExceptionStackTrace = (char*)stackTrace; +} + +int javaObjectIsOfClass(JNIEnv *env, jobject obj, const char *name) +{ + jclass clazz; + int ret; + + clazz = (*env)->FindClass(env, name); + if (!clazz) { + printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "javaObjectIsOfClass(%s)", name); + return -1; + } + ret = (*env)->IsInstanceOf(env, obj, clazz); + (*env)->DeleteLocalRef(env, clazz); + return ret == JNI_TRUE ? 1 : 0; +} + +jthrowable hadoopConfSetStr(JNIEnv *env, jobject jConfiguration, + const char *key, const char *value) +{ + jthrowable jthr; + jstring jkey = NULL, jvalue = NULL; + + jthr = newJavaStr(env, key, &jkey); + if (jthr) + goto done; + jthr = newJavaStr(env, value, &jvalue); + if (jthr) + goto done; + jthr = invokeMethod(env, NULL, INSTANCE, jConfiguration, + JC_CONFIGURATION, "set", "(Ljava/lang/String;Ljava/lang/String;)V", + jkey, jvalue); + if (jthr) + goto done; +done: + (*env)->DeleteLocalRef(env, jkey); + (*env)->DeleteLocalRef(env, jvalue); + return jthr; +} + +jthrowable fetchEnumInstance(JNIEnv *env, const char *className, + const char *valueName, jobject *out) +{ + jclass clazz; + jfieldID fieldId; + jobject jEnum; + char prettyClass[256]; + + clazz = (*env)->FindClass(env, className); + if (!clazz) { + return getPendingExceptionAndClear(env); + } + if (snprintf(prettyClass, sizeof(prettyClass), "L%s;", className) + >= sizeof(prettyClass)) { + return newRuntimeError(env, "fetchEnum(%s, %s): class name too long.", + className, valueName); + } + fieldId = (*env)->GetStaticFieldID(env, clazz, valueName, prettyClass); + if (!fieldId) { + return getPendingExceptionAndClear(env); + } + jEnum = (*env)->GetStaticObjectField(env, clazz, fieldId); + if (!jEnum) { + return getPendingExceptionAndClear(env); + } + *out = jEnum; + return NULL; +} + diff --git a/native/hdfs-sys/libhdfs/hdfs_3_3/jni_helper.h b/native/hdfs-sys/libhdfs/hdfs_3_3/jni_helper.h new file mode 100644 index 0000000000..41d6fab2a7 --- /dev/null +++ b/native/hdfs-sys/libhdfs/hdfs_3_3/jni_helper.h @@ -0,0 +1,221 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFS_JNI_HELPER_H +#define LIBHDFS_JNI_HELPER_H + +#include "jclasses.h" + +#include +#include + +#include +#include +#include + +#ifdef WIN32 + #define PATH_SEPARATOR ';' + #define PATH_SEPARATOR_STR ";" +#else + #define PATH_SEPARATOR ':' + #define PATH_SEPARATOR_STR ":" +#endif + +// #define _LIBHDFS_JNI_HELPER_DEBUGGING_ON_ + +/** Denote the method we want to invoke as STATIC or INSTANCE */ +typedef enum { + STATIC, + INSTANCE +} MethType; + +/** + * Create a new malloc'ed C string from a Java string. + * + * @param env The JNI environment + * @param jstr The Java string + * @param out (out param) the malloc'ed C string + * + * @return NULL on success; the exception otherwise + */ +jthrowable newCStr(JNIEnv *env, jstring jstr, char **out); + +/** + * Create a new Java string from a C string. + * + * @param env The JNI environment + * @param str The C string + * @param out (out param) the java string + * + * @return NULL on success; the exception otherwise + */ +jthrowable newJavaStr(JNIEnv *env, const char *str, jstring *out); + +/** + * Helper function to destroy a local reference of java.lang.Object + * @param env: The JNIEnv pointer. + * @param jFile: The local reference of java.lang.Object object + * @return None. + */ +void destroyLocalReference(JNIEnv *env, jobject jObject); + +/** invokeMethod: Invoke a Static or Instance method. + * methName: Name of the method + * methSignature: the signature of the method "(arg-types)ret-type" + * methType: The type of the method (STATIC or INSTANCE) + * instObj: Required if the methType is INSTANCE. The object to invoke + the method on. + * class: The CachedJavaClass to call the method on. + * env: The JNIEnv pointer + * retval: The pointer to a union type which will contain the result of the + method invocation, e.g. if the method returns an Object, retval will be + set to that, if the method returns boolean, retval will be set to the + value (JNI_TRUE or JNI_FALSE), etc. + * exc: If the methods throws any exception, this will contain the reference + * Arguments (the method arguments) must be passed after methSignature + * RETURNS: -1 on error and 0 on success. If -1 is returned, exc will have + a valid exception reference, and the result stored at retval is undefined. + */ +jthrowable invokeMethod(JNIEnv *env, jvalue *retval, MethType methType, + jobject instObj, CachedJavaClass class, + const char *methName, const char *methSignature, ...); + +/** + * findClassAndInvokeMethod: Same as invokeMethod, but it calls FindClass on + * the given className first and then calls invokeMethod. This method exists + * mainly for test infrastructure, any production code should use + * invokeMethod. Calling FindClass repeatedly can introduce performance + * overhead, so users should prefer invokeMethod and supply a CachedJavaClass. + */ +jthrowable findClassAndInvokeMethod(JNIEnv *env, jvalue *retval, + MethType methType, jobject instObj, const char *className, + const char *methName, const char *methSignature, ...); + +jthrowable constructNewObjectOfClass(JNIEnv *env, jobject *out, + const char *className, const char *ctorSignature, ...); + +/** + * Same as constructNewObjectOfClass but it takes in a CachedJavaClass + * rather than a className. This avoids an extra call to FindClass. + */ +jthrowable constructNewObjectOfCachedClass(JNIEnv *env, jobject *out, + CachedJavaClass cachedJavaClass, const char *ctorSignature, ...); + +jthrowable methodIdFromClass(jclass cls, const char *className, + const char *methName, const char *methSignature, MethType methType, + JNIEnv *env, jmethodID *out); + +/** classNameOfObject: Get an object's class name. + * @param jobj: The object. + * @param env: The JNIEnv pointer. + * @param name: (out param) On success, will contain a string containing the + * class name. This string must be freed by the caller. + * @return NULL on success, or the exception + */ +jthrowable classNameOfObject(jobject jobj, JNIEnv *env, char **name); + +/** getJNIEnv: A helper function to get the JNIEnv* for the given thread. + * It gets this from the ThreadLocalState if it exists. If a ThreadLocalState + * does not exist, one will be created. + * If no JVM exists, then one will be created. JVM command line arguments + * are obtained from the LIBHDFS_OPTS environment variable. + * @param: None. + * @return The JNIEnv* corresponding to the thread. + * */ +JNIEnv* getJNIEnv(void); + +/** + * Get the last exception root cause that happened in the context of the + * current thread. + * + * The pointer returned by this function is guaranteed to be valid until + * the next call to invokeMethod() by the current thread. + * Users of this function should not free the pointer. + * + * @return The root cause as a C-string. + */ +char* getLastTLSExceptionRootCause(); + +/** + * Get the last exception stack trace that happened in the context of the + * current thread. + * + * The pointer returned by this function is guaranteed to be valid until + * the next call to invokeMethod() by the current thread. + * Users of this function should not free the pointer. + * + * @return The stack trace as a C-string. + */ +char* getLastTLSExceptionStackTrace(); + +/** setTLSExceptionStrings: Sets the 'rootCause' and 'stackTrace' in the + * ThreadLocalState if one exists for the current thread. + * + * @param rootCause A string containing the root cause of an exception. + * @param stackTrace A string containing the stack trace of an exception. + * @return None. + */ +void setTLSExceptionStrings(const char *rootCause, const char *stackTrace); + +/** + * Figure out if a Java object is an instance of a particular class. + * + * @param env The Java environment. + * @param obj The object to check. + * @param name The class name to check. + * + * @return -1 if we failed to find the referenced class name. + * 0 if the object is not of the given class. + * 1 if the object is of the given class. + */ +int javaObjectIsOfClass(JNIEnv *env, jobject obj, const char *name); + +/** + * Set a value in a configuration object. + * + * @param env The JNI environment + * @param jConfiguration The configuration object to modify + * @param key The key to modify + * @param value The value to set the key to + * + * @return NULL on success; exception otherwise + */ +jthrowable hadoopConfSetStr(JNIEnv *env, jobject jConfiguration, + const char *key, const char *value); + +/** + * Fetch an instance of an Enum. + * + * @param env The JNI environment. + * @param className The enum class name. + * @param valueName The name of the enum value + * @param out (out param) on success, a local reference to an + * instance of the enum object. (Since Java enums are + * singletones, this is also the only instance.) + * + * @return NULL on success; exception otherwise + */ +jthrowable fetchEnumInstance(JNIEnv *env, const char *className, + const char *valueName, jobject *out); + +#endif /*LIBHDFS_JNI_HELPER_H*/ + +/** + * vim: ts=4: sw=4: et: + */ + diff --git a/native/hdfs-sys/libhdfs/hdfs_3_3/os/mutexes.h b/native/hdfs-sys/libhdfs/hdfs_3_3/os/mutexes.h new file mode 100644 index 0000000000..92afabd7c7 --- /dev/null +++ b/native/hdfs-sys/libhdfs/hdfs_3_3/os/mutexes.h @@ -0,0 +1,55 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFS_MUTEXES_H +#define LIBHDFS_MUTEXES_H + +/* + * Defines abstraction over platform-specific mutexes. libhdfs has no formal + * initialization function that users would call from a single-threaded context + * to initialize the library. This creates a challenge for bootstrapping the + * mutexes. To address this, all required mutexes are pre-defined here with + * external storage. Platform-specific implementations must guarantee that the + * mutexes are initialized via static initialization. + */ + +#include "platform.h" + +/** Mutex protecting singleton JVM instance. */ +extern mutex jvmMutex; + +/** Mutex protecting initialization of jclasses in jclasses.h. */ +extern mutex jclassInitMutex; + +/** + * Locks a mutex. + * + * @param m mutex + * @return 0 if successful, non-zero otherwise + */ +int mutexLock(mutex *m); + +/** + * Unlocks a mutex. + * + * @param m mutex + * @return 0 if successful, non-zero otherwise + */ +int mutexUnlock(mutex *m); + +#endif diff --git a/native/hdfs-sys/libhdfs/hdfs_3_3/os/posix/mutexes.c b/native/hdfs-sys/libhdfs/hdfs_3_3/os/posix/mutexes.c new file mode 100644 index 0000000000..5c6b429d5e --- /dev/null +++ b/native/hdfs-sys/libhdfs/hdfs_3_3/os/posix/mutexes.c @@ -0,0 +1,50 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "os/mutexes.h" + +#include +#include + +mutex jvmMutex; +mutex jclassInitMutex = PTHREAD_MUTEX_INITIALIZER; +pthread_mutexattr_t jvmMutexAttr; + +__attribute__((constructor)) static void init() { + pthread_mutexattr_init(&jvmMutexAttr); + pthread_mutexattr_settype(&jvmMutexAttr, PTHREAD_MUTEX_RECURSIVE); + pthread_mutex_init(&jvmMutex, &jvmMutexAttr); +} + +int mutexLock(mutex *m) { + int ret = pthread_mutex_lock(m); + if (ret) { + fprintf(stderr, "mutexLock: pthread_mutex_lock failed with error %d\n", + ret); + } + return ret; +} + +int mutexUnlock(mutex *m) { + int ret = pthread_mutex_unlock(m); + if (ret) { + fprintf(stderr, "mutexUnlock: pthread_mutex_unlock failed with error %d\n", + ret); + } + return ret; +} diff --git a/native/hdfs-sys/libhdfs/hdfs_3_3/os/posix/platform.h b/native/hdfs-sys/libhdfs/hdfs_3_3/os/posix/platform.h new file mode 100644 index 0000000000..c63bbf9e0e --- /dev/null +++ b/native/hdfs-sys/libhdfs/hdfs_3_3/os/posix/platform.h @@ -0,0 +1,34 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFS_PLATFORM_H +#define LIBHDFS_PLATFORM_H + +#include + +/* Use gcc type-checked format arguments. */ +#define TYPE_CHECKED_PRINTF_FORMAT(formatArg, varArgs) \ + __attribute__((format(printf, formatArg, varArgs))) + +/* + * Mutex and thread data types defined by pthreads. + */ +typedef pthread_mutex_t mutex; +typedef pthread_t threadId; + +#endif diff --git a/native/hdfs-sys/libhdfs/hdfs_3_3/os/posix/thread.c b/native/hdfs-sys/libhdfs/hdfs_3_3/os/posix/thread.c new file mode 100644 index 0000000000..af0c61f03d --- /dev/null +++ b/native/hdfs-sys/libhdfs/hdfs_3_3/os/posix/thread.c @@ -0,0 +1,52 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "os/thread.h" + +#include +#include + +/** + * Defines a helper function that adapts function pointer provided by caller to + * the type required by pthread_create. + * + * @param toRun thread to run + * @return void* result of running thread (always NULL) + */ +static void* runThread(void *toRun) { + const thread *t = toRun; + t->start(t->arg); + return NULL; +} + +int threadCreate(thread *t) { + int ret; + ret = pthread_create(&t->id, NULL, runThread, t); + if (ret) { + fprintf(stderr, "threadCreate: pthread_create failed with error %d\n", ret); + } + return ret; +} + +int threadJoin(const thread *t) { + int ret = pthread_join(t->id, NULL); + if (ret) { + fprintf(stderr, "threadJoin: pthread_join failed with error %d\n", ret); + } + return ret; +} diff --git a/native/hdfs-sys/libhdfs/hdfs_3_3/os/posix/thread_local_storage.c b/native/hdfs-sys/libhdfs/hdfs_3_3/os/posix/thread_local_storage.c new file mode 100644 index 0000000000..0e187e9865 --- /dev/null +++ b/native/hdfs-sys/libhdfs/hdfs_3_3/os/posix/thread_local_storage.c @@ -0,0 +1,207 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * This file was modified by the Apache DataFusion Comet project. See + * native/hdfs-sys/README.md for the provenance of this directory and for the + * full list of modifications. The changes here are the HDFS-16021 + * thread-ownership guard in hdfsThreadDestructor and the initialisation of the + * two fields threadLocalStorageCreate previously left uninitialised. + */ + +#include "os/thread_local_storage.h" + +#include +#include +#include +#include + +#include "exception.h" +#include "jni_helper.h" + +#define UNKNOWN "UNKNOWN" +#define MAXTHRID 256 + +/** Key that allows us to retrieve thread-local storage */ +static pthread_key_t gTlsKey; + +/** nonzero if we succeeded in initializing gTlsKey. Protected by the jvmMutex */ +static int gTlsKeyInitialized = 0; + +static void get_current_thread_id(JNIEnv* env, char* id, int max); + +/** + * The function that is called whenever a thread with libhdfs thread local data + * is destroyed. + * + * @param v The thread-local data + */ +void hdfsThreadDestructor(void *v) +{ + JavaVM *vm; + struct ThreadLocalState *state = (struct ThreadLocalState*)v; + JNIEnv *env = state->env;; + jint ret; + jthrowable jthr; + char thr_name[MAXTHRID]; + + /* Detach only threads that libhdfs attached to the JVM. Detaching a thread + * that the JVM (or an embedding application) attached frees a JNIEnv its + * owner still holds, and by the time this destructor runs that env may + * already have been freed, so the dereference below reads freed memory. + * See HDFS-16021. */ + if (state->attachedByLibhdfs && (env != NULL) && (*env != NULL)) { + ret = (*env)->GetJavaVM(env, &vm); + + if (ret != 0) { + fprintf(stderr, "hdfsThreadDestructor: GetJavaVM failed with error %d\n", + ret); + jthr = (*env)->ExceptionOccurred(env); + if (jthr) { + (*env)->ExceptionDescribe(env); + (*env)->ExceptionClear(env); + } + } else { + ret = (*vm)->DetachCurrentThread(vm); + + if (ret != JNI_OK) { + jthr = (*env)->ExceptionOccurred(env); + if (jthr) { + (*env)->ExceptionDescribe(env); + (*env)->ExceptionClear(env); + } + get_current_thread_id(env, thr_name, MAXTHRID); + + fprintf(stderr, "hdfsThreadDestructor: Unable to detach thread %s " + "from the JVM. Error code: %d\n", thr_name, ret); + } + } + } + + /* Free exception strings */ + if (state->lastExceptionStackTrace) free(state->lastExceptionStackTrace); + if (state->lastExceptionRootCause) free(state->lastExceptionRootCause); + + /* Free the state itself */ + free(state); +} + +static void get_current_thread_id(JNIEnv* env, char* id, int max) { + jvalue jVal; + jobject thr = NULL; + jstring thr_name = NULL; + jlong thr_id = 0; + jthrowable jthr = NULL; + const char *thr_name_str; + + jthr = findClassAndInvokeMethod(env, &jVal, STATIC, NULL, "java/lang/Thread", + "currentThread", "()Ljava/lang/Thread;"); + if (jthr) { + snprintf(id, max, "%s", UNKNOWN); + printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "get_current_thread_id: Thread#currentThread failed: "); + goto done; + } + thr = jVal.l; + + jthr = findClassAndInvokeMethod(env, &jVal, INSTANCE, thr, + "java/lang/Thread", "getId", "()J"); + if (jthr) { + snprintf(id, max, "%s", UNKNOWN); + printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "get_current_thread_id: Thread#getId failed: "); + goto done; + } + thr_id = jVal.j; + + jthr = findClassAndInvokeMethod(env, &jVal, INSTANCE, thr, + "java/lang/Thread", "toString", "()Ljava/lang/String;"); + if (jthr) { + snprintf(id, max, "%s:%ld", UNKNOWN, thr_id); + printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "get_current_thread_id: Thread#toString failed: "); + goto done; + } + thr_name = jVal.l; + + thr_name_str = (*env)->GetStringUTFChars(env, thr_name, NULL); + if (!thr_name_str) { + printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "get_current_thread_id: GetStringUTFChars failed: "); + snprintf(id, max, "%s:%ld", UNKNOWN, thr_id); + goto done; + } + + // Treating the jlong as a long *should* be safe + snprintf(id, max, "%s:%ld", thr_name_str, thr_id); + + // Release the char* + (*env)->ReleaseStringUTFChars(env, thr_name, thr_name_str); + +done: + destroyLocalReference(env, thr); + destroyLocalReference(env, thr_name); + + // Make sure the id is null terminated in case we overflow the max length + id[max - 1] = '\0'; +} + +struct ThreadLocalState* threadLocalStorageCreate() +{ + struct ThreadLocalState *state; + state = (struct ThreadLocalState*)malloc(sizeof(struct ThreadLocalState)); + if (state == NULL) { + fprintf(stderr, + "threadLocalStorageCreate: OOM - Unable to allocate thread local state\n"); + return NULL; + } + state->attachedByLibhdfs = false; + state->env = NULL; + state->lastExceptionStackTrace = NULL; + state->lastExceptionRootCause = NULL; + return state; +} + +int threadLocalStorageGet(struct ThreadLocalState **state) +{ + int ret = 0; + if (!gTlsKeyInitialized) { + ret = pthread_key_create(&gTlsKey, hdfsThreadDestructor); + if (ret) { + fprintf(stderr, + "threadLocalStorageGet: pthread_key_create failed with error %d\n", + ret); + return ret; + } + gTlsKeyInitialized = 1; + } + *state = pthread_getspecific(gTlsKey); + return ret; +} + +int threadLocalStorageSet(struct ThreadLocalState *state) +{ + int ret = pthread_setspecific(gTlsKey, state); + if (ret) { + fprintf(stderr, + "threadLocalStorageSet: pthread_setspecific failed with error %d\n", + ret); + hdfsThreadDestructor(state); + } + return ret; +} diff --git a/native/hdfs-sys/libhdfs/hdfs_3_3/os/thread.h b/native/hdfs-sys/libhdfs/hdfs_3_3/os/thread.h new file mode 100644 index 0000000000..ae425d3564 --- /dev/null +++ b/native/hdfs-sys/libhdfs/hdfs_3_3/os/thread.h @@ -0,0 +1,54 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFS_THREAD_H +#define LIBHDFS_THREAD_H + +/* + * Defines abstraction over platform-specific threads. + */ + +#include "platform.h" + +/** Pointer to function to run in thread. */ +typedef void (*threadProcedure)(void *); + +/** Structure containing a thread's ID, starting address and argument. */ +typedef struct { + threadId id; + threadProcedure start; + void *arg; +} thread; + +/** + * Creates and immediately starts a new thread. + * + * @param t thread to create + * @return 0 if successful, non-zero otherwise + */ +int threadCreate(thread *t); + +/** + * Joins to the given thread, blocking if necessary. + * + * @param t thread to join + * @return 0 if successful, non-zero otherwise + */ +int threadJoin(const thread *t); + +#endif diff --git a/native/hdfs-sys/libhdfs/hdfs_3_3/os/thread_local_storage.h b/native/hdfs-sys/libhdfs/hdfs_3_3/os/thread_local_storage.h new file mode 100644 index 0000000000..4c7128a713 --- /dev/null +++ b/native/hdfs-sys/libhdfs/hdfs_3_3/os/thread_local_storage.h @@ -0,0 +1,110 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * This file was modified by the Apache DataFusion Comet project. See + * native/hdfs-sys/README.md for the provenance of this directory and for the + * full list of modifications. The change here is the `attachedByLibhdfs` field, + * which carries the HDFS-16021 thread-ownership fix. + */ + +#ifndef LIBHDFS_THREAD_LOCAL_STORAGE_H +#define LIBHDFS_THREAD_LOCAL_STORAGE_H + +/* + * Defines abstraction over platform-specific thread-local storage. libhdfs + * currently only needs thread-local storage for a single piece of data: the + * thread's JNIEnv. For simplicity, this interface is defined in terms of + * JNIEnv, not general-purpose thread-local storage of any arbitrary data. + */ + +#include +#include + +/* + * Most operating systems support the more efficient __thread construct, which + * is initialized by the linker. The following macros use this technique on the + * operating systems that support it. + */ +#ifdef HAVE_BETTER_TLS + #define THREAD_LOCAL_STORAGE_GET_QUICK(state) \ + static __thread struct ThreadLocalState *quickTlsEnv = NULL; \ + { \ + if (quickTlsEnv) { \ + *state = quickTlsEnv; \ + } \ + } + + #define THREAD_LOCAL_STORAGE_SET_QUICK(state) \ + { \ + quickTlsEnv = (state); \ + } +#else + #define THREAD_LOCAL_STORAGE_GET_QUICK(state) + #define THREAD_LOCAL_STORAGE_SET_QUICK(state) +#endif + +struct ThreadLocalState { + /* Whether libhdfs attached this thread to the JVM. */ + bool attachedByLibhdfs; + /* The JNIEnv associated with the current thread */ + JNIEnv *env; + /* The last exception stack trace that occured on this thread */ + char *lastExceptionStackTrace; + /* The last exception root cause that occured on this thread */ + char *lastExceptionRootCause; +}; + +/** + * The function that is called whenever a thread with libhdfs thread local data + * is destroyed. + * + * @param v The thread-local data + */ +void hdfsThreadDestructor(void *v); + +/** + * Creates an object of ThreadLocalState. + * + * @return The newly created object if successful, NULL otherwise. + */ +struct ThreadLocalState* threadLocalStorageCreate(); + +/** + * Gets the ThreadLocalState in thread-local storage for the current thread. + * If the call succeeds, and there is a ThreadLocalState associated with this + * thread, then returns 0 and populates 'state'. If the call succeeds, but + * there is no ThreadLocalState associated with this thread, then returns 0 + * and sets ThreadLocalState to NULL. If the call fails, then returns non-zero. + * Only one thread at a time may execute this function. The caller is + * responsible for enforcing mutual exclusion. + * + * @param env ThreadLocalState out parameter + * @return 0 if successful, non-zero otherwise + */ +int threadLocalStorageGet(struct ThreadLocalState **state); + +/** + * Sets the ThreadLocalState in thread-local storage for the current thread. + * + * @param env ThreadLocalState to set + * @return 0 if successful, non-zero otherwise + */ +int threadLocalStorageSet(struct ThreadLocalState *state); + +#endif diff --git a/native/hdfs-sys/src/lib.rs b/native/hdfs-sys/src/lib.rs new file mode 100644 index 0000000000..9fda3a5681 --- /dev/null +++ b/native/hdfs-sys/src/lib.rs @@ -0,0 +1,191 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Raw bindings to the `libhdfs` C API, built from the vendored Apache Hadoop +//! sources under `libhdfs/hdfs_3_3/`. +//! +//! This crate stands in for the crates.io `hdfs-sys` crate through a +//! `[patch.crates-io]` entry in `native/Cargo.toml`. It exists only so Comet can +//! carry the HDFS-16021 thread-ownership fix, which no released `hdfs-sys` +//! contains. See `README.md` for the provenance and the removal condition. +//! +//! The declarations below are transcribed from the vendored +//! `libhdfs/hdfs_3_3/include/hdfs/hdfs.h` and cover the surface `hdrs`, the only +//! consumer in Comet's dependency graph, actually uses. Names match the +//! bindgen-style spelling that `hdrs` expects, notably the flattened +//! `tObjectKind_kObjectKind*` constants. + +#![allow(non_snake_case)] +#![allow(non_camel_case_types)] +#![allow(non_upper_case_globals)] + +use std::os::raw::{c_char, c_int, c_long, c_short, c_void}; + +/// `typedef int32_t tSize` -- size of data for read/write io ops. +pub type tSize = i32; +/// `typedef time_t tTime` -- time type in seconds. +pub type tTime = c_long; +/// `typedef int64_t tOffset` -- offset within the file. +pub type tOffset = i64; +/// `typedef uint16_t tPort` -- port. +pub type tPort = u16; + +/// `typedef enum tObjectKind`. A C enum with values that fit in an `unsigned +/// int`, which is how `hdrs` consumes `hdfsFileInfo::mKind`. +pub type tObjectKind = ::std::os::raw::c_uint; +/// `kObjectKindFile = 'F'` +pub const tObjectKind_kObjectKindFile: tObjectKind = 70; +/// `kObjectKindDirectory = 'D'` +pub const tObjectKind_kObjectKindDirectory: tObjectKind = 68; + +/// Opaque `struct hdfsBuilder`. +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct hdfsBuilder { + _unused: [u8; 0], +} + +/// Opaque `struct hdfs_internal`. +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct hdfs_internal { + _unused: [u8; 0], +} + +/// `typedef struct hdfs_internal* hdfsFS`. +pub type hdfsFS = *mut hdfs_internal; + +/// Opaque `struct hdfsFile_internal`. +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct hdfsFile_internal { + _unused: [u8; 0], +} + +/// `typedef struct hdfsFile_internal* hdfsFile`. +pub type hdfsFile = *mut hdfsFile_internal; + +/// Field order and widths mirror the `hdfsFileInfo` typedef in `hdfs.h`. +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct hdfsFileInfo { + /// file or directory + pub mKind: tObjectKind, + /// the name of the file + pub mName: *mut c_char, + /// the last modification time for the file in seconds + pub mLastMod: tTime, + /// the size of the file in bytes + pub mSize: tOffset, + /// the count of replicas + pub mReplication: c_short, + /// the block size for the file + pub mBlockSize: tOffset, + /// the owner of the file + pub mOwner: *mut c_char, + /// the group associated with the file + pub mGroup: *mut c_char, + /// the permissions associated with the file + pub mPermissions: c_short, + /// the last access time for the file in seconds + pub mLastAccess: tTime, +} + +unsafe extern "C" { + pub fn hdfsNewBuilder() -> *mut hdfsBuilder; + pub fn hdfsFreeBuilder(bld: *mut hdfsBuilder); + pub fn hdfsBuilderSetNameNode(bld: *mut hdfsBuilder, nn: *const c_char); + pub fn hdfsBuilderSetNameNodePort(bld: *mut hdfsBuilder, port: tPort); + pub fn hdfsBuilderSetUserName(bld: *mut hdfsBuilder, userName: *const c_char); + pub fn hdfsBuilderSetKerbTicketCachePath( + bld: *mut hdfsBuilder, + kerbTicketCachePath: *const c_char, + ); + pub fn hdfsBuilderConnect(bld: *mut hdfsBuilder) -> hdfsFS; + + pub fn hdfsConnect(nn: *const c_char, port: tPort) -> hdfsFS; + pub fn hdfsDisconnect(fs: hdfsFS) -> c_int; + + pub fn hdfsOpenFile( + fs: hdfsFS, + path: *const c_char, + flags: c_int, + bufferSize: c_int, + replication: c_short, + blocksize: tSize, + ) -> hdfsFile; + pub fn hdfsCloseFile(fs: hdfsFS, file: hdfsFile) -> c_int; + pub fn hdfsExists(fs: hdfsFS, path: *const c_char) -> c_int; + + pub fn hdfsSeek(fs: hdfsFS, file: hdfsFile, desiredPos: tOffset) -> c_int; + pub fn hdfsTell(fs: hdfsFS, file: hdfsFile) -> tOffset; + pub fn hdfsRead(fs: hdfsFS, file: hdfsFile, buffer: *mut c_void, length: tSize) -> tSize; + pub fn hdfsPread( + fs: hdfsFS, + file: hdfsFile, + position: tOffset, + buffer: *mut c_void, + length: tSize, + ) -> tSize; + pub fn hdfsWrite(fs: hdfsFS, file: hdfsFile, buffer: *const c_void, length: tSize) -> tSize; + pub fn hdfsFlush(fs: hdfsFS, file: hdfsFile) -> c_int; + + pub fn hdfsDelete(fs: hdfsFS, path: *const c_char, recursive: c_int) -> c_int; + pub fn hdfsRename(fs: hdfsFS, oldPath: *const c_char, newPath: *const c_char) -> c_int; + pub fn hdfsCreateDirectory(fs: hdfsFS, path: *const c_char) -> c_int; + + pub fn hdfsGetPathInfo(fs: hdfsFS, path: *const c_char) -> *mut hdfsFileInfo; + pub fn hdfsListDirectory( + fs: hdfsFS, + path: *const c_char, + numEntries: *mut c_int, + ) -> *mut hdfsFileInfo; + pub fn hdfsFreeFileInfo(hdfsFileInfo: *mut hdfsFileInfo, numEntries: c_int); +} + +/// `hdrs` reads `hdfsFileInfo` fields out of an array libhdfs allocated, so a layout mismatch +/// between these declarations and the C typedef they transcribe would be silent memory corruption +/// rather than a link error. These are compile-time assertions rather than `#[test]`s so that every +/// build of the crate enforces them; the crate is not a default workspace member, so `cargo test` +/// would not otherwise reach it. +/// +/// The expected offsets follow from the C field order under the usual 8-byte alignment: +/// `mKind`(u32) + 4 pad, `mName`(8), `mLastMod`(8), `mSize`(8), `mReplication`(2) + 6 pad, +/// `mBlockSize`(8), `mOwner`(8), `mGroup`(8), `mPermissions`(2) + 6 pad, `mLastAccess`(8). +const _: () = { + use std::mem::{align_of, offset_of, size_of}; + + assert!(size_of::() == 80); + assert!(align_of::() == 8); + assert!(offset_of!(hdfsFileInfo, mKind) == 0); + assert!(offset_of!(hdfsFileInfo, mName) == 8); + assert!(offset_of!(hdfsFileInfo, mLastMod) == 16); + assert!(offset_of!(hdfsFileInfo, mSize) == 24); + assert!(offset_of!(hdfsFileInfo, mReplication) == 32); + assert!(offset_of!(hdfsFileInfo, mBlockSize) == 40); + assert!(offset_of!(hdfsFileInfo, mOwner) == 48); + assert!(offset_of!(hdfsFileInfo, mGroup) == 56); + assert!(offset_of!(hdfsFileInfo, mPermissions) == 64); + assert!(offset_of!(hdfsFileInfo, mLastAccess) == 72); + + // `hdfs.h` spells these as the character literals 'F' and 'D'. + assert!(tObjectKind_kObjectKindFile == b'F' as tObjectKind); + assert!(tObjectKind_kObjectKindDirectory == b'D' as tObjectKind); + + // `tTime` is `time_t`, and `hdrs` binds the two timestamp fields to `i64`. + assert!(size_of::() == 8); +};