#3693: Compile natives as C instead of C++, check malloc/calloc return values for null

native/compile-native.sh

@@ -8,7 +8,20 @@ echo "Compiling mbedtls"
 echo "Compiling zlib"
 (cd zlib && CFLAGS=-fPIC ./configure --static && make)
 
-CXX="g++ -shared -fPIC -O3 -Wall -Werror -I$JAVA_HOME/include/ -I$JAVA_HOME/include/linux/"
+CC="gcc"
+CFLAGS="-c -fPIC -O3 -Wall -Werror -I$JAVA_HOME/include/ -I$JAVA_HOME/include/linux/"
+LDFLAGS="-shared"
 
-$CXX -Imbedtls/include src/main/c/NativeCipherImpl.cpp -o src/main/resources/native-cipher.so mbedtls/library/libmbedcrypto.a
-$CXX -Izlib src/main/c/NativeCompressImpl.cpp -o src/main/resources/native-compress.so zlib/libz.a
+echo "Compiling bungee"
+$CC $CFLAGS -o shared.o src/main/c/shared.c 
+$CC $CFLAGS -Imbedtls/include -o NativeCipherImpl.o src/main/c/NativeCipherImpl.c
+$CC $CFLAGS -Izlib -o NativeCompressImpl.o src/main/c/NativeCompressImpl.c
+
+echo "Linking native-cipher.so"
+$CC $LDFLAGS -o src/main/resources/native-cipher.so shared.o NativeCipherImpl.o mbedtls/library/libmbedcrypto.a
+
+echo "Linking native-compress.so"
+$CC $LDFLAGS -o src/main/resources/native-compress.so shared.o NativeCompressImpl.o zlib/libz.a
+
+echo "Cleaning up"
+rm shared.o NativeCipherImpl.o NativeCompressImpl.o

native/src/main/c/NativeCipherImpl.c

@@ -2,31 +2,33 @@
 #include <string.h>
 
 #include <mbedtls/aes.h>
+#include "shared.h"
 #include "net_md_5_bungee_jni_cipher_NativeCipherImpl.h"
 
 typedef unsigned char byte;
 
-struct crypto_context {
+typedef struct crypto_context {
     int mode;
     mbedtls_aes_context cipher;
-    byte *key;
-};
+    byte key[];
+} crypto_context;
 
 jlong JNICALL Java_net_md_15_bungee_jni_cipher_NativeCipherImpl_init(JNIEnv* env, jobject obj, jboolean forEncryption, jbyteArray key) {
-    jsize keyLen = env->GetArrayLength(key);
-    jbyte *keyBytes = env->GetByteArrayElements(key, NULL);
+    jsize keyLen = (*env)->GetArrayLength(env, key);
+
+    crypto_context *crypto = (crypto_context*) malloc(sizeof (crypto_context) + (size_t) keyLen);
+    if (!crypto) {
+        throwOutOfMemoryError(env, "Failed to malloc new crypto_context");
+        return 0;
+    }
+
+    (*env)->GetByteArrayRegion(env, key, 0, keyLen, (jbyte*) &crypto->key);
 
-    crypto_context *crypto = (crypto_context*) malloc(sizeof (crypto_context));
     mbedtls_aes_init(&crypto->cipher);
-
-    mbedtls_aes_setkey_enc(&crypto->cipher, (byte*) keyBytes, keyLen * 8);
-
-    crypto->key = (byte*) malloc(keyLen);
-    memcpy(crypto->key, keyBytes, keyLen);
+    mbedtls_aes_setkey_enc(&crypto->cipher, (byte*) &crypto->key, keyLen * 8);
 
     crypto->mode = (forEncryption) ? MBEDTLS_AES_ENCRYPT : MBEDTLS_AES_DECRYPT;
 
-    env->ReleaseByteArrayElements(key, keyBytes, JNI_ABORT);
     return (jlong) crypto;
 }
 
@@ -34,7 +36,6 @@ void Java_net_md_15_bungee_jni_cipher_NativeCipherImpl_free(JNIEnv* env, jobject
     crypto_context *crypto = (crypto_context*) ctx;
 
     mbedtls_aes_free(&crypto->cipher);
-    free(crypto->key);
     free(crypto);
 }
 

native/src/main/c/NativeCompressImpl.c

@@ -2,28 +2,28 @@
 #include <string.h>
 
 #include <zlib.h>
+#include "shared.h"
 #include "net_md_5_bungee_jni_zlib_NativeCompressImpl.h"
 
 typedef unsigned char byte;
 
+static jclass classID;
 static jfieldID consumedID;
 static jfieldID finishedID;
+static jmethodID makeExceptionID;
 
 void JNICALL Java_net_md_15_bungee_jni_zlib_NativeCompressImpl_initFields(JNIEnv* env, jclass clazz) {
-    // We trust that these fields will be there
-    consumedID = env->GetFieldID(clazz, "consumed", "I");
-    finishedID = env->GetFieldID(clazz, "finished", "Z");
+    classID = clazz;
+    // We trust that these will be there
+    consumedID = (*env)->GetFieldID(env, clazz, "consumed", "I");
+    finishedID = (*env)->GetFieldID(env, clazz, "finished", "Z");
+    makeExceptionID = (*env)->GetMethodID(env, clazz, "makeException", "(Ljava/lang/String;I)Lnet/md_5/bungee/jni/NativeCodeException;");
 }
 
 jint throwException(JNIEnv *env, const char* message, int err) {
-    // These can't be static for some unknown reason
-    jclass exceptionClass = env->FindClass("net/md_5/bungee/jni/NativeCodeException");
-    jmethodID exceptionInitID = env->GetMethodID(exceptionClass, "<init>", "(Ljava/lang/String;I)V");
-
-    jstring jMessage = env->NewStringUTF(message);
-
-    jthrowable throwable = (jthrowable) env->NewObject(exceptionClass, exceptionInitID, jMessage, err);
-    return env->Throw(throwable);
+    jstring jMessage = (*env)->NewStringUTF(env, message);
+    jthrowable throwable = (jthrowable) (*env)->CallStaticObjectMethod(env, classID, makeExceptionID, jMessage, err);
+    return (*env)->Throw(env, throwable);
 }
 
 void JNICALL Java_net_md_15_bungee_jni_zlib_NativeCompressImpl_reset(JNIEnv* env, jobject obj, jlong ctx, jboolean compress) {
@@ -48,10 +48,17 @@ void JNICALL Java_net_md_15_bungee_jni_zlib_NativeCompressImpl_end(JNIEnv* env,
 
 jlong JNICALL Java_net_md_15_bungee_jni_zlib_NativeCompressImpl_init(JNIEnv* env, jobject obj, jboolean compress, jint level) {
     z_stream* stream = (z_stream*) calloc(1, sizeof (z_stream));
+    if (!stream) {
+        throwOutOfMemoryError(env, "Failed to calloc new z_stream");
+        return 0;
+    }
+
     int ret = (compress) ? deflateInit(stream, level) : inflateInit(stream);
 
     if (ret != Z_OK) {
+        free(stream);
         throwException(env, "Could not init z_stream", ret);
+        return 0;
     }
 
     return (jlong) stream;
@@ -70,15 +77,16 @@ jint JNICALL Java_net_md_15_bungee_jni_zlib_NativeCompressImpl_process(JNIEnv* e
 
     switch (ret) {
         case Z_STREAM_END:
-            env->SetBooleanField(obj, finishedID, true);
+            (*env)->SetBooleanField(env, obj, finishedID, JNI_TRUE);
             break;
         case Z_OK:
             break;
         default:
             throwException(env, "Unknown z_stream return code", ret);
+            return -1;
     }
 
-    env->SetIntField(obj, consumedID, inLength - stream->avail_in);
+    (*env)->SetIntField(env, obj, consumedID, inLength - stream->avail_in);
 
     return outLength - stream->avail_out;
 }

native/src/main/c/shared.c

@@ -0,0 +1,15 @@
+#include "shared.h"
+#include <stdlib.h>
+#include <stdio.h>
+
+void throwOutOfMemoryError(JNIEnv* env, const char* msg) {
+    jclass exceptionClass = (*env)->FindClass(env, "java/lang/OutOfMemoryError");
+    if (!exceptionClass) {
+        // If the proxy ran out of memory, loading this class may fail
+        fprintf(stderr, "OUT OF MEMORY: %s\n", msg);
+        fprintf(stderr, "Could not load class java.lang.OutOfMemoryError!\n");
+        exit(-1);
+        return;
+    }
+    (*env)->ThrowNew(env, exceptionClass, msg);
+}

native/src/main/c/shared.h

@@ -0,0 +1,10 @@
+// This header contains functions to be shared between both native libraries
+
+#include <jni.h>
+
+#ifndef _INCLUDE_SHARED_H
+#define _INCLUDE_SHARED_H
+
+void throwOutOfMemoryError(JNIEnv* env, const char* msg);
+
+#endif

native/src/main/java/net/md_5/bungee/jni/NativeCodeException.java

@@ -1,6 +1,6 @@
 package net.md_5.bungee.jni;
 
-public class NativeCodeException extends Exception
+public class NativeCodeException extends RuntimeException
 {
 
     public NativeCodeException(String message, int reason)

native/src/main/java/net/md_5/bungee/jni/zlib/NativeCompressImpl.java

@@ -1,5 +1,7 @@
 package net.md_5.bungee.jni.zlib;
 
+import net.md_5.bungee.jni.NativeCodeException;
+
 public class NativeCompressImpl
 {
 
@@ -20,4 +22,9 @@ public class NativeCompressImpl
     native long init(boolean compress, int compressionLevel);
 
     native int process(long ctx, long in, int inLength, long out, int outLength, boolean compress);
+
+    NativeCodeException makeException(String message, int err)
+    {
+        return new NativeCodeException( message, err );
+    }
 }

native/src/main/java/net/md_5/bungee/jni/zlib/NativeZlib.java

@@ -4,6 +4,7 @@ import com.google.common.base.Preconditions;
 import io.netty.buffer.ByteBuf;
 import java.util.zip.DataFormatException;
 import lombok.Getter;
+import net.md_5.bungee.jni.NativeCodeException;
 
 public class NativeZlib implements BungeeZlib
 {
@@ -48,7 +49,14 @@ public class NativeZlib implements BungeeZlib
         {
             out.ensureWritable( 8192 );
 
-            int processed = nativeCompress.process( ctx, in.memoryAddress() + in.readerIndex(), in.readableBytes(), out.memoryAddress() + out.writerIndex(), out.writableBytes(), compress );
+            int processed;
+            try
+            {
+                processed = nativeCompress.process( ctx, in.memoryAddress() + in.readerIndex(), in.readableBytes(), out.memoryAddress() + out.writerIndex(), out.writableBytes(), compress );
+            } catch ( NativeCodeException exception )
+            {
+                throw (DataFormatException) new DataFormatException( "Failed to decompress via Zlib!" ).initCause( exception );
+            }
 
             in.readerIndex( in.readerIndex() + nativeCompress.consumed );
             out.writerIndex( out.writerIndex() + processed );

native/src/test/java/net/md_5/bungee/NativeZlibTest.java

@@ -28,6 +28,17 @@ public class NativeZlibTest
         test( new JavaZlib() );
     }
 
+    @Test
+    public void testException() throws DataFormatException
+    {
+        if ( NativeCode.isSupported() )
+        {
+            assertTrue( factory.load(), "Native code failed to load!" );
+            testExceptionImpl( factory.newInstance() );
+        }
+        testExceptionImpl( new JavaZlib() );
+    }
+
     private void test(BungeeZlib zlib) throws DataFormatException
     {
         System.out.println( "Testing: " + zlib );
@@ -66,4 +77,22 @@ public class NativeZlibTest
 
         assertTrue( Arrays.equals( dataBuf, check ), "Results do not match" );
     }
+
+    private void testExceptionImpl(BungeeZlib zlib) throws DataFormatException
+    {
+        System.out.println( "Testing Exception: " + zlib );
+        long start = System.currentTimeMillis();
+
+        byte[] dataBuf = new byte[ 1 << 12 ]; // 4096 random bytes
+        new Random().nextBytes( dataBuf );
+
+        zlib.init( false, 0 );
+
+        ByteBuf originalBuf = Unpooled.directBuffer();
+        originalBuf.writeBytes( dataBuf );
+
+        ByteBuf decompressed = Unpooled.directBuffer();
+
+        assertThrows( DataFormatException.class, () -> zlib.process( originalBuf, decompressed ), "Decompressing random bytes did not result in a DataFormatException!" );
+    }
 }