libhdfs JNI FindClass 使用 java.io.FileNotFoundException 返回 null

Question

我正在尝试在 windows 上测试 libhdfs，它使用 JNI 调用 java 函数，创建了 JVM，但是在加载 hadoop 类时，findclass 总是失败。

我用谷歌搜索了stackoverflow，无法为我找出问题所在，请帮忙。

系统：

Windows 7 32-bit
Visual Studio 2013 Express
JDK 1.6.0_32
Hadoop 2.5.0 (it's working fine using command)

问题：

when program executed to getGlobalJNIEnv -> globalClassReference -> FindClass

    local_clazz = (*env)->FindClass(env, className); // classname = org/apache/hadoop/fs/FileSystem

org.apache.hadoop.fs.FileSystem is in C:\hadoop\share\hadoop\common\hadoop-common-2.5.0.jar, which is in CLASSPATH

FindClass always return null.

libhdfs 代码：

static JNIEnv* getGlobalJNIEnv(void)
{
    JavaVM* vmBuf[VM_BUF_LENGTH];
    JNIEnv *env;
    jint rv = 0;
    jint noVMs = 0;
    jthrowable jthr;
    char *hadoopClassPath;
    const char *hadoopClassPathVMArg = "-Djava.class.path=";
    size_t optHadoopClassPathLen;
    char *optHadoopClassPath;
    const int noArgs = 6;
    char *hadoopJvmArgs;
    char jvmArgDelims[] = " ";
    char *str, *token, *savePtr;
    JavaVMInitArgs vm_args;
    JavaVM *vm;
    JavaVMOption *options;

    rv = JNI_GetCreatedJavaVMs(&(vmBuf[0]), VM_BUF_LENGTH, &noVMs);
    if (rv != 0) {
        fprintf(stderr, "JNI_GetCreatedJavaVMs failed with error: %d\n", rv);
        return NULL;
    }

    if (noVMs == 0) {
        //Get the environment variables for initializing the JVM
        hadoopClassPath = getenv("CLASSPATH");
        if (hadoopClassPath == NULL) {
            fprintf(stderr, "Environment variable CLASSPATH not set!\n");
            return NULL;
        }
        optHadoopClassPathLen = strlen(hadoopClassPath) +
            strlen(hadoopClassPathVMArg) + 1;
        optHadoopClassPath = malloc(sizeof(char)*optHadoopClassPathLen);
        snprintf(optHadoopClassPath, optHadoopClassPathLen,
            "%s%s", hadoopClassPathVMArg, hadoopClassPath);



        // Now that we know the # args, populate the options array
        options = calloc(noArgs, sizeof(JavaVMOption));
        if (!options) {
            fputs("Call to calloc failed\n", stderr);
            free(optHadoopClassPath);
            return NULL;
        }

        options[0].optionString = optHadoopClassPath;
        options[1].optionString = "-Djava.library.path=C:\\Progra~1\\Java\\jdk1.6.0_32\\lib";
        options[2].optionString = "-Djava.compiler=NONE";
        options[3].optionString = "-verbose:jni";
        options[4].optionString = "-verbose:class";
        options[5].optionString = "-verbose:gc";



        //Create the VM
        vm_args.version = JNI_VERSION_1_6;
        vm_args.options = options;
        vm_args.nOptions = noArgs;
        vm_args.ignoreUnrecognized = JNI_FALSE;

        rv = JNI_CreateJavaVM(&vm, (void*)&env, &vm_args);

        if (rv != 0) {
            fprintf(stderr, "Call to JNI_CreateJavaVM failed "
                "with error: %d\n", rv);
            return NULL;
        }
        jthr = invokeMethod(env, NULL, STATIC, NULL,
            "org/apache/hadoop/fs/FileSystem",
            "loadFileSystems", "()V");

        free(optHadoopClassPath);
        free(options);

        if (jthr) {
            printExceptionAndFree(env, jthr, PRINT_EXC_ALL, "loadFileSystems");
        }

    }
    else {
        //Attach this thread to the VM
        vm = vmBuf[0];
        rv = (*vm)->AttachCurrentThread(vm, (void*)&env, 0);
        if (rv != 0) {
            fprintf(stderr, "Call to AttachCurrentThread "
                "failed with error: %d\n", rv);
            return NULL;
        }
    }

    return env;
}


jthrowable globalClassReference(const char *className, JNIEnv *env, jclass *out)
{
    jthrowable jthr = NULL;
    jclass local_clazz = NULL;
    jclass clazz = NULL;
    int ret;

    mutexLock(&hdfsHashMutex);
    if (!gClassRefHTable) {
        gClassRefHTable = htable_alloc(MAX_HASH_TABLE_ELEM, ht_hash_string,
            ht_compare_string);
        if (!gClassRefHTable) {
            jthr = newRuntimeError(env, "htable_alloc failed\n");
            goto done;
        }
    }
    clazz = htable_get(gClassRefHTable, className);
    if (clazz) {
        *out = clazz;
        goto done;
    }

    local_clazz = (*env)->FindClass(env, className);

    if (!local_clazz) {

        (*env)->ExceptionDescribe(env);

        jthr = getPendingExceptionAndClear(env);
        goto done;
    }
    clazz = (*env)->NewGlobalRef(env, local_clazz);
    if (!clazz) {
        jthr = getPendingExceptionAndClear(env);
        goto done;
    }
    ret = htable_put(gClassRefHTable, (void*)className, clazz);
    if (ret) {
        jthr = newRuntimeError(env, "htable_put failed with error "
            "code %d\n", ret);
        goto done;
    }
    *out = clazz;
    jthr = NULL;
done:
    mutexUnlock(&hdfsHashMutex);
    (*env)->DeleteLocalRef(env, local_clazz);
    if (jthr && clazz) {
        (*env)->DeleteGlobalRef(env, clazz);
    }
    return jthr;
}

jthrowable getPendingExceptionAndClear(JNIEnv *env)
{
    jthrowable jthr = (*env)->ExceptionOccurred(env);
    if (!jthr)
        return NULL;
    (*env)->ExceptionClear(env);
    return jthr;
}

int printExceptionAndFreeV(JNIEnv *env, jthrowable exc, int noPrintFlags,
        const char *fmt, va_list ap)
{
    int i, noPrint, excErrno;
    char *className = NULL;
    jstring jStr = NULL;
    jvalue jVal;
    jthrowable jthr;

    jthr = classNameOfObject(exc, env, &className);
    if (jthr) {
        fprintf(stderr, "PrintExceptionAndFree: error determining class name "
            "of exception.\n");
        className = strdup("(unknown)");
        destroyLocalReference(env, jthr);
    }
    for (i = 0; i < EXCEPTION_INFO_LEN; i++) {
        if (!strcmp(gExceptionInfo[i].name, className)) {
            break;
        }
    }
    if (i < EXCEPTION_INFO_LEN) {
        noPrint = (gExceptionInfo[i].noPrintFlag & noPrintFlags);
        excErrno = gExceptionInfo[i].excErrno;
    } else {
        noPrint = 0;
        excErrno = EINTERNAL;
    }
    if (!noPrint) {
        vfprintf(stderr, fmt, ap);
        fprintf(stderr, " error:\n");

        // We don't want to  use ExceptionDescribe here, because that requires a
        // pending exception.  Instead, use ExceptionUtils.
        jthr = invokeMethod(env, &jVal, STATIC, NULL, 
            "org/apache/commons/lang/exception/ExceptionUtils",
            "getStackTrace", "(Ljava/lang/Throwable;)Ljava/lang/String;", exc);
        if (jthr) {
            fprintf(stderr, "(unable to get stack trace for %s exception: "
                    "ExceptionUtils::getStackTrace error.)\n", className);
            destroyLocalReference(env, jthr);
        } else {
            jStr = jVal.l;
            const char *stackTrace = (*env)->GetStringUTFChars(env, jStr, NULL);
            if (!stackTrace) {
                fprintf(stderr, "(unable to get stack trace for %s exception: "
                        "GetStringUTFChars error.)\n", className);
            } else {
                fprintf(stderr, "%s", stackTrace);
                (*env)->ReleaseStringUTFChars(env, jStr, stackTrace);
            }
        }
    }
    destroyLocalReference(env, jStr);
    destroyLocalReference(env, exc);
    free(className);
    return excErrno;
}

int printExceptionAndFree(JNIEnv *env, jthrowable exc, int noPrintFlags,
        const char *fmt, ...)
{
    va_list ap;
    int ret;

    va_start(ap, fmt);
    ret = printExceptionAndFreeV(env, exc, noPrintFlags, fmt, ap);
    va_end(ap);
    return ret;
}

环境变量：

C:\>hadoop classpath
c:\hadoop\etc\hadoop;C:\hadoop\share\hadoop\common\lib\*;C:\hadoop\share\hadoop\
common\*;C:\hadoop\share\hadoop\hdfs;C:\hadoop\share\hadoop\hdfs\lib\*;C:\hadoop
\share\hadoop\hdfs\*;C:\hadoop\share\hadoop\yarn\lib\*;C:\hadoop\share\hadoop\ya
rn\*;C:\hadoop\share\hadoop\mapreduce\lib\*;C:\hadoop\share\hadoop\mapreduce\*

CLASSPATH = c:\Progra~1\Java\jdk1.6.0_32\lib;c:\Progra~1\Java\jdk1.6.0_32\jre\lib;c:\hadoop\etc\hadoop;C:\hadoop\share\hadoop\common\lib\*;C:\hadoop\share\hadoop\common\*;C:\hadoop\share\hadoop\hdfs;C:\hadoop\share\hadoop\hdfs\lib\*;C:\hadoop\share\hadoop\hdfs\*;C:\hadoop\share\hadoop\yarn\lib\*;C:\hadoop\share\hadoop\yarn\*;C:\hadoop\share\hadoop\mapreduce\lib\*;C:\hadoop\share\hadoop\mapreduce\*

JAVA_HOME = c:\Progra~1\Java\jdk1.6.0_32

控制台输出：（在 findclass 之后添加 ExceptionDescribe 后更新）

Exception in thread "main" java.lang.NoClassDefFoundError: org/apache/hadoop/fs/
FileSystem
Caused by: java.lang.ClassNotFoundException: org.apache.hadoop.fs.FileSystem
        at java.net.URLClassLoader$1.run(URLClassLoader.java:202)
        at java.security.AccessController.doPrivileged(Native Method)
        at java.net.URLClassLoader.findClass(URLClassLoader.java:190)
        at java.lang.ClassLoader.loadClass(ClassLoader.java:306)
        at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:301)
        at java.lang.ClassLoader.loadClass(ClassLoader.java:247)
Exception in thread "main" java.lang.NoClassDefFoundError: org/apache/hadoop/fs/
FileSystem
Caused by: java.lang.ClassNotFoundException: org.apache.hadoop.fs.FileSystem
        at java.net.URLClassLoader$1.run(URLClassLoader.java:202)
        at java.security.AccessController.doPrivileged(Native Method)
        at java.net.URLClassLoader.findClass(URLClassLoader.java:190)
        at java.lang.ClassLoader.loadClass(ClassLoader.java:306)
        at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:301)
        at java.lang.ClassLoader.loadClass(ClassLoader.java:247)
could not find method loadFileSystems from class org/apache/hadoop/fs/FileSystem
 with signature ()V
loadFileSystems error:
Exception in thread "main" java.lang.NoClassDefFoundError: org/apache/commons/la
ng/exception/ExceptionUtils
Caused by: java.lang.ClassNotFoundException: org.apache.commons.lang.exception.E
xceptionUtils
        at java.net.URLClassLoader$1.run(URLClassLoader.java:202)
        at java.security.AccessController.doPrivileged(Native Method)
        at java.net.URLClassLoader.findClass(URLClassLoader.java:190)
        at java.lang.ClassLoader.loadClass(ClassLoader.java:306)
        at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:301)
        at java.lang.ClassLoader.loadClass(ClassLoader.java:247)
Exception in thread "main" java.lang.NoClassDefFoundError: org/apache/commons/la
ng/exception/ExceptionUtils
Caused by: java.lang.ClassNotFoundException: org.apache.commons.lang.exception.E
xceptionUtils
        at java.net.URLClassLoader$1.run(URLClassLoader.java:202)
        at java.security.AccessController.doPrivileged(Native Method)
        at java.net.URLClassLoader.findClass(URLClassLoader.java:190)
        at java.lang.ClassLoader.loadClass(ClassLoader.java:306)
        at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:301)
        at java.lang.ClassLoader.loadClass(ClassLoader.java:247)
could not find method getStackTrace from class org/apache/commons/lang/exception
/ExceptionUtils with signature (Ljava/lang/Throwable;)Ljava/lang/String;
(unable to get stack trace for java.lang.NoSuchMethodError exception: ExceptionU
tils::getStackTrace error.)
Exception in thread "main" java.lang.NoClassDefFoundError: org/apache/hadoop/con
f/Configuration
Caused by: java.lang.ClassNotFoundException: org.apache.hadoop.conf.Configuratio
n
        at java.net.URLClassLoader$1.run(URLClassLoader.java:202)
        at java.security.AccessController.doPrivileged(Native Method)
        at java.net.URLClassLoader.findClass(URLClassLoader.java:190)
        at java.lang.ClassLoader.loadClass(ClassLoader.java:306)
        at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:301)
        at java.lang.ClassLoader.loadClass(ClassLoader.java:247)
Exception in thread "main" java.lang.NoClassDefFoundError: org/apache/hadoop/con
f/Configuration
Caused by: java.lang.ClassNotFoundException: org.apache.hadoop.conf.Configuratio
n
        at java.net.URLClassLoader$1.run(URLClassLoader.java:202)
        at java.security.AccessController.doPrivileged(Native Method)
        at java.net.URLClassLoader.findClass(URLClassLoader.java:190)
        at java.lang.ClassLoader.loadClass(ClassLoader.java:306)
        at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:301)
        at java.lang.ClassLoader.loadClass(ClassLoader.java:247)

Answer 1

Caused by: java.lang.ClassNotFoundException: org.apache.hadoop.fs.FileSystem

CLASSPATH 上没有该类。