This is an automated email from the ASF dual-hosted git repository. shaojunwang pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-teaclave-java-tee-sdk.git
commit a5eb6ada1a50c4835c142e81f85a8a5c674fd01a Author: jeffery.wsj <[email protected]> AuthorDate: Thu May 19 14:35:07 2022 +0800 [sdk] Support TEE_SDK enclave mode Summary: 1. Add TEE_SDK host and enclave's feature implementation 2. Add TEE_SDK compile and package toolchains 3. Add TEE_SDK mode for the test project in JavaEnclave Test Plan: all tests pass Reviewers: lei.yul, cengfeng.lzy, sanhong.lsh Issue: https://aone.alibaba-inc.com/task/42325600 CR: https://code.aone.alibaba-inc.com/java-tee/JavaEnclave/codereview/8964556 --- build.sh | 7 +- sdk/enclave/pom.xml | 22 ++++ .../enclave/EnclaveEntry.java | 3 + .../platform/tee_sdk_svm/edge_routines/Makefile | 20 +++ .../platform/tee_sdk_svm/edge_routines/sgx_mmap.c | 50 +++++++ .../platform/tee_sdk_svm/edge_routines/sgx_mmap.h | 15 +++ .../tee_sdk_svm/edge_routines/tee_sdk_symbol.c | 141 ++++++++++++++++++++ .../tee_sdk_svm/edge_routines/tee_sdk_symbol.h | 109 +++++++++++++++ .../cpp/platform/tee_sdk_svm/wrapper/Makefile | 13 ++ .../platform/tee_sdk_svm/wrapper/tee_sdk_wrapper.c | 83 ++++++++++++ .../platform/tee_sdk_svm/wrapper/tee_sdk_wrapper.h | 17 +++ .../native-image/serialization-config.json | 3 + .../src/main/resources/native/enc_environment.h | 20 +-- sdk/host/pom.xml | 2 +- .../host/AbstractEnclave.java | 2 +- .../host/MockInSvmEnclave.java | 7 +- .../confidentialcomputing/host/TeeSdkEnclave.java | 73 +++-------- sdk/host/src/main/native/Makefile | 11 -- sdk/host/src/main/native/config/config.mk | 13 -- .../native/cpp/platform/mock_in_svm/jni/Makefile | 4 +- .../cpp/platform/mock_in_svm/jni/jni_mock_in_svm.c | 72 +++++----- .../main/native/cpp/platform/tee_sdk_svm/Makefile | 23 ++++ .../platform/tee_sdk_svm/edge_routines/Makefile | 18 +++ .../cpp/platform/tee_sdk_svm/edge_routines/ocall.c | 14 ++ .../cpp/platform/tee_sdk_svm/edge_routines/ocall.h | 20 +++ .../native/cpp/platform/tee_sdk_svm/jni/Makefile | 15 +++ .../cpp/platform/tee_sdk_svm/jni/jni_tee_sdk_svm.c | 146 +++++++++++++++++++++ .../cpp/platform/tee_sdk_svm/jni/jni_tee_sdk_svm.h | 66 ++++++++++ sdk/host/src/main/native/include/enc_environment.h | 29 ---- .../native/bin/platform/mock_in_svm/jni/.gitkeep | 0 .../bin/platform/tee_sdk_svm}/jni/.gitkeep | 0 sdk/native/config/config.mk | 6 + .../config/platform/mock_in_svm/jni/config.mk | 0 .../platform/tee_sdk_svm/edl/tee_sdk_enclave.edl | 27 ++++ .../platform/tee_sdk_svm/edl/tee_sdk_enclave.lds | 11 ++ .../config/platform/tee_sdk_svm/jni/config.mk | 61 +++++++++ .../native => native/include}/enc_environment.h | 20 +-- .../main => }/native/include/enc_exported_symbol.h | 0 sdk/native/script/build_app/Makefile | 36 +++++ sdk/native/script/build_app/make.sh | 33 +++++ sdk/native/script/build_enclave_sdk/Makefile | 17 +++ .../script/build_enclave_sdk}/make.sh | 10 +- sdk/native/script/build_host_sdk/Makefile | 23 ++++ .../script/build_host_sdk}/make.sh | 15 ++- sdk/pom.xml | 2 +- .../test/common/ReflectionCallService.java | 1 - test/enclave/pom.xml | 39 ++++++ test/enclave/src/main/resources/tee_sdk_svm.conf | 12 ++ test/host/pom.xml | 1 + .../test/host/TestJavaEnclaveService.java | 3 + test/pom.xml | 2 +- tools/cicd/Dockerfile | 10 +- tools/cicd/make.sh | 4 + 53 files changed, 1167 insertions(+), 184 deletions(-) diff --git a/build.sh b/build.sh index daf8dd6..bed423b 100644 --- a/build.sh +++ b/build.sh @@ -8,5 +8,10 @@ cd "${SHELL_FOLDER}" # workspace dir is the same as build.sh path location. WORKDIR="$PWD" +# Build JavaEnclave SDK cd "${WORKDIR}"/sdk && mvn --settings /root/tools/settings.xml clean install -cd "${WORKDIR}"/test && mvn --settings /root/tools/settings.xml -Pnative -e clean package +# Install JavaEnclave SDK +rm -rf /opt/javaenclave && mkdir -p /opt/javaenclave && cp -r ${SHELL_FOLDER}/sdk/native/bin /opt/javaenclave \ +&& cp -r ${SHELL_FOLDER}/sdk/native/config /opt/javaenclave && cp -r ${SHELL_FOLDER}/sdk/native/script/build_app /opt/javaenclave +# Test unit test cases in JavaEnclave +cd "${WORKDIR}"/test && mvn -X --settings /root/tools/settings.xml -Pnative -e clean package diff --git a/sdk/enclave/pom.xml b/sdk/enclave/pom.xml index d818052..e586e83 100644 --- a/sdk/enclave/pom.xml +++ b/sdk/enclave/pom.xml @@ -65,6 +65,28 @@ </profiles> <build> <plugins> + <plugin> + <artifactId>exec-maven-plugin</artifactId> + <groupId>org.codehaus.mojo</groupId> + <executions> + <execution> + <id>JavaEnclaveNativeCompile</id> + <phase>compile</phase> + <goals> + <goal>exec</goal> + </goals> + <configuration> + <executable>bash</executable> + <arguments> + <argument>${project.basedir}/../native/script/build_enclave_sdk/make.sh</argument> + <argument>${project.basedir}</argument> + <argument>${com.alibaba.enclave.platform}</argument> + <argument>build</argument> + </arguments> + </configuration> + </execution> + </executions> + </plugin> <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-compiler-plugin</artifactId> diff --git a/sdk/enclave/src/main/java/com/alibaba/confidentialcomputing/enclave/EnclaveEntry.java b/sdk/enclave/src/main/java/com/alibaba/confidentialcomputing/enclave/EnclaveEntry.java index e75d542..c46e835 100644 --- a/sdk/enclave/src/main/java/com/alibaba/confidentialcomputing/enclave/EnclaveEntry.java +++ b/sdk/enclave/src/main/java/com/alibaba/confidentialcomputing/enclave/EnclaveEntry.java @@ -22,6 +22,7 @@ public class EnclaveEntry { } @SuppressWarnings("unused") + // Align with head define file enc_exported_symbol.h if it changes. @CEntryPoint(name = "java_loadservice_invoke") @CEntryPointOptions(prologue = EnclavePrologue.class) public static int loadService(Isolate isolate, EncData input, EncData result, CallBacks callBacks) { @@ -36,6 +37,7 @@ public class EnclaveEntry { } @SuppressWarnings("unused") + // Align with head define file enc_exported_symbol.h if it changes. @CEntryPoint(name = "java_unloadservice_invoke") @CEntryPointOptions(prologue = EnclavePrologue.class) public static int unloadService(Isolate isolate, EncData input, EncData result, CallBacks callBacks) { @@ -50,6 +52,7 @@ public class EnclaveEntry { } @SuppressWarnings("unused") + // Align with head define file enc_exported_symbol.h if it changes. @CEntryPoint(name = "java_enclave_invoke") @CEntryPointOptions(prologue = EnclavePrologue.class) public static int javaEnclaveInvoke(Isolate isolate, EncData input, EncData result, CallBacks callBacks) { diff --git a/sdk/enclave/src/main/native/cpp/platform/tee_sdk_svm/edge_routines/Makefile b/sdk/enclave/src/main/native/cpp/platform/tee_sdk_svm/edge_routines/Makefile new file mode 100644 index 0000000..20a2a30 --- /dev/null +++ b/sdk/enclave/src/main/native/cpp/platform/tee_sdk_svm/edge_routines/Makefile @@ -0,0 +1,20 @@ +# Copyright (c) + +include $(NATIVE_BASE_DIR)/config/config.mk +include $(NATIVE_BASE_DIR)/config/platform/tee_sdk_svm/jni/config.mk + +.PHONY: all build clean + +all: build + +build: + $(SGX_EDGER8R) $(CONFIG)/platform/tee_sdk_svm/edl/tee_sdk_enclave.edl --trusted --search-path $(TEE_SDK_PATH)/include + $(CC) -g -c -std=c99 -I$(INCLUDE) $(TS_ENCLAVE_INCDIR) $(TS_ENCLAVE_CFLAGS) tee_sdk_enclave_t.c sgx_mmap.c tee_sdk_symbol.c + + mv *.o $(BIN)/platform/tee_sdk_svm/ + + rm -rf tee_sdk_enclave_t.* + +clean: + rm -rf $(BIN)/platform/tee_sdk_svm/wrapper/*.o + rm -rf tee_sdk_svm_* \ No newline at end of file diff --git a/sdk/enclave/src/main/native/cpp/platform/tee_sdk_svm/edge_routines/sgx_mmap.c b/sdk/enclave/src/main/native/cpp/platform/tee_sdk_svm/edge_routines/sgx_mmap.c new file mode 100644 index 0000000..bc2fb8e --- /dev/null +++ b/sdk/enclave/src/main/native/cpp/platform/tee_sdk_svm/edge_routines/sgx_mmap.c @@ -0,0 +1,50 @@ +#include <stdint.h> +#include <stdio.h> +#include <assert.h> + +#include "tee_sdk_enclave_t.h" +#include "sgx_mmap.h" + +#define PHYSICAL_PAGE_SIZE 4096 +#define VIRTUAL_PAGE_SIZE 4096 + +// get memory physical page size in enclave. +long physical_page_size() { + return PHYSICAL_PAGE_SIZE; +} + +// get memory physical page number in enclave. +long physical_page_number() { + return get_heap_size() / PHYSICAL_PAGE_SIZE; +} + +// get memory virtual page size in enclave. +long virtual_page_size() { + return VIRTUAL_PAGE_SIZE; +} + +// mmap and munmap is only partially supported in tee sdk enclave, and mmap doesn't +// support memory space reserve, but support memory space allocation. +void* mmap(void *hint, int size, int prot, int flags) { + void *ptr = 0; + // flags == 0x4022, svm runtime expects to reserve a memory buffer with giving start address hint; + // flags == 0x22 and hint == 0x0, svm runtime expects to reserve a memory buffer, the start address depends. + // Both the two scene, JavaEnclave SDK view them as enclave memory allocation, while not memory space reserve. + if ((flags == 0x4022) || (flags == 0x22 && hint == 0x0 && prot == 0x3)) { + // fd mapping is not supported in enclave, so the last two parameters of + // (int fd, off_t offset) must be (-1, 0); + // parameter pro = 0x3 (0B0011) indicates allocated buffer could be read and written. + // parameter flags = 0x21, because ts_mmap only support this kind of operation. + ptr = ts_mmap(hint, size, 0x3, 0x21, -1, 0); + } else if (flags == 0x32) { + ptr = hint; + } else { + printf("JavaEnclave Warning: unsupported mmap operation in tee sdk enclave: 0x%lx, ptr is: %p, size is: %d, prot is: 0x%x, flags is: 0x%x.\n", (uint64_t)hint, ptr, size, prot, flags); + assert(-1); + } + return ptr; +} + +int munmap(void *addr, int size) { + return ts_munmap(addr, size); +} \ No newline at end of file diff --git a/sdk/enclave/src/main/native/cpp/platform/tee_sdk_svm/edge_routines/sgx_mmap.h b/sdk/enclave/src/main/native/cpp/platform/tee_sdk_svm/edge_routines/sgx_mmap.h new file mode 100644 index 0000000..34350c7 --- /dev/null +++ b/sdk/enclave/src/main/native/cpp/platform/tee_sdk_svm/edge_routines/sgx_mmap.h @@ -0,0 +1,15 @@ +#include "tee_sdk_enclave_t.h" + +#ifndef _SGX_MMAP_H_ +#define _SGX_MMAP_H_ + +long physical_page_size(); +long physical_page_number(); +long virtual_page_size(); +void* mmap(void *hint, int size, int prot, int flags); +int munmap(void *addr, int size); +extern void* ts_mmap(void *addr, size_t length, int prot, int flags, int fd, int offset); +extern int ts_munmap(void *addr, size_t len); +extern size_t get_heap_size(void); + +#endif /* !_SGX_MMAP_H_ */ \ No newline at end of file diff --git a/sdk/enclave/src/main/native/cpp/platform/tee_sdk_svm/edge_routines/tee_sdk_symbol.c b/sdk/enclave/src/main/native/cpp/platform/tee_sdk_svm/edge_routines/tee_sdk_symbol.c new file mode 100644 index 0000000..d34b495 --- /dev/null +++ b/sdk/enclave/src/main/native/cpp/platform/tee_sdk_svm/edge_routines/tee_sdk_symbol.c @@ -0,0 +1,141 @@ +#include "tee_sdk_enclave_t.h" +#include "tee_sdk_symbol.h" + +//#define ENABLE_TRACE_SYSCALL +#if defined(ENABLE_TRACE_SYSCALL) +#define TRACE_SYMBOL_CALL() printf("JavaEnclave Warning: %s is called in enclave svm.\n", __FUNCTION__); +#else +#define TRACE_SYMBOL_CALL() +#endif + +//#define UNSUPPORTED_SYSCALL_SYMBOL_ASSERT +#if defined(UNSUPPORTED_SYSCALL_SYMBOL_ASSERT) +#define ASSERT() assert(-1); +#else +#define ASSERT() +#endif + +void __fxstat() {TRACE_SYMBOL_CALL(); ASSERT();} +void __fxstat64() {TRACE_SYMBOL_CALL(); ASSERT();} +void __isnan() {TRACE_SYMBOL_CALL(); ASSERT();} +void __libc_current_sigrtmax() {TRACE_SYMBOL_CALL(); ASSERT();} +void __libc_malloc() {TRACE_SYMBOL_CALL(); ASSERT();} +void __lxstat() {TRACE_SYMBOL_CALL(); ASSERT();} +void __lxstat64() {TRACE_SYMBOL_CALL(); ASSERT();} +void __sched_cpucount() {TRACE_SYMBOL_CALL(); ASSERT();} +void __strdup() {TRACE_SYMBOL_CALL(); ASSERT();} +void __xmknod() {TRACE_SYMBOL_CALL(); ASSERT();} +void __xstat() {TRACE_SYMBOL_CALL(); ASSERT();} +void __xstat64() {TRACE_SYMBOL_CALL(); ASSERT();} +void chmod() {TRACE_SYMBOL_CALL(); ASSERT();} +void chown() {TRACE_SYMBOL_CALL(); ASSERT();} +void crc32() {TRACE_SYMBOL_CALL(); ASSERT();} +void deflate() {TRACE_SYMBOL_CALL(); ASSERT();} +void deflateBound() {TRACE_SYMBOL_CALL(); ASSERT();} +void deflateEnd() {TRACE_SYMBOL_CALL(); ASSERT();} +void deflateInit2_() {TRACE_SYMBOL_CALL(); ASSERT();} +void deflateSetHeader() {TRACE_SYMBOL_CALL(); ASSERT();} +void dlopen() {TRACE_SYMBOL_CALL(); ASSERT();} +void dlsym() {TRACE_SYMBOL_CALL(); ASSERT();} +void endmntent() {TRACE_SYMBOL_CALL(); ASSERT();} +void fchmod() {TRACE_SYMBOL_CALL(); ASSERT();} +void fchown() {TRACE_SYMBOL_CALL(); ASSERT();} +void fpathconf() {TRACE_SYMBOL_CALL(); ASSERT();} +void fstatvfs() {TRACE_SYMBOL_CALL(); ASSERT();} +void fstatvfs64() {TRACE_SYMBOL_CALL(); ASSERT();} +void getgrnam_r() {TRACE_SYMBOL_CALL(); ASSERT();} +void getmntent_r() {TRACE_SYMBOL_CALL(); ASSERT();} +void getpwnam_r() {TRACE_SYMBOL_CALL(); ASSERT();} +void inflate() {TRACE_SYMBOL_CALL(); ASSERT();} +void inflateEnd() {TRACE_SYMBOL_CALL(); ASSERT();} +void inflateInit2_() {TRACE_SYMBOL_CALL(); ASSERT();} +void inflateReset() {TRACE_SYMBOL_CALL(); ASSERT();} +void inflateSetDictionary() {TRACE_SYMBOL_CALL(); ASSERT();} +void lchown() {TRACE_SYMBOL_CALL(); ASSERT();} +void lstat() {TRACE_SYMBOL_CALL(); ASSERT();} +void mknod() {TRACE_SYMBOL_CALL(); ASSERT();} +void pathconf() {TRACE_SYMBOL_CALL(); ASSERT();} +void pipe() {TRACE_SYMBOL_CALL(); ASSERT();} +void pthread_attr_init() {TRACE_SYMBOL_CALL(); ASSERT();} +void pthread_attr_setdetachstate() {TRACE_SYMBOL_CALL(); ASSERT();} +void pthread_kill() {TRACE_SYMBOL_CALL(); ASSERT();} +void pthread_setname_np() {TRACE_SYMBOL_CALL(); ASSERT();} +void readlink() {TRACE_SYMBOL_CALL(); ASSERT();} +void realpath() {TRACE_SYMBOL_CALL(); ASSERT();} +void sched_getaffinity() {TRACE_SYMBOL_CALL(); ASSERT();} +void sendfile() {TRACE_SYMBOL_CALL(); ASSERT();} +void sendfile64() {TRACE_SYMBOL_CALL(); ASSERT();} +void setmntent() {TRACE_SYMBOL_CALL(); ASSERT();} +void sigaddset() {TRACE_SYMBOL_CALL(); ASSERT();} +void sigemptyset() {TRACE_SYMBOL_CALL(); ASSERT();} +void sigprocmask() {TRACE_SYMBOL_CALL(); ASSERT();} +void statvfs() {TRACE_SYMBOL_CALL(); ASSERT();} +void statvfs64() {TRACE_SYMBOL_CALL(); ASSERT();} +void symlink() {TRACE_SYMBOL_CALL(); ASSERT();} +void utimes() {TRACE_SYMBOL_CALL(); ASSERT();} + +int posix_memalign(void **memptr, size_t alignment, size_t size) { + TRACE_SYMBOL_CALL(); + void* ptr = malloc(size); + if (ptr == NULL) { return -1; } + *memptr = ptr; + return 0; +} + +unsigned long int pthread_self(void) { + TRACE_SYMBOL_CALL(); + return (unsigned long int)get_thread_data(); +} + +int pthread_attr_getstack(const pthread_attr *a, void ** addr, size_t *size) { + TRACE_SYMBOL_CALL(); + thread_data *self = (thread_data *)get_thread_data(); + uint64_t stack_base_addr = self->__stack_base_addr; + uint64_t stack_limit_addr = self->__stack_limit_addr; + *size = (int)ROUND_TO_PAGE(stack_base_addr - stack_limit_addr); + *addr = (void *)stack_limit_addr; + return 0; +} + +int pthread_attr_getguardsize(const pthread_attr *a, size_t *size) { + TRACE_SYMBOL_CALL(); + *size = 1; + return 0; +} + +int getrlimit(int resource, rlimit* rlim) { + TRACE_SYMBOL_CALL(); + int ret = 0; + ocall_getrlimit(&ret, resource, (void*)rlim); + return ret; +} + +int mprotect() { + TRACE_SYMBOL_CALL(); + return 0; +} + +int pthread_condattr_init() { + TRACE_SYMBOL_CALL(); + return 0; +} + +int pthread_condattr_setclock() { + TRACE_SYMBOL_CALL(); + return 0; +} + +int pthread_getattr_np() { + TRACE_SYMBOL_CALL(); + return 0; +} + +int pthread_attr_destroy() { + TRACE_SYMBOL_CALL(); + return 0; +} + +int setrlimit() { + TRACE_SYMBOL_CALL(); + return 0; +} \ No newline at end of file diff --git a/sdk/enclave/src/main/native/cpp/platform/tee_sdk_svm/edge_routines/tee_sdk_symbol.h b/sdk/enclave/src/main/native/cpp/platform/tee_sdk_svm/edge_routines/tee_sdk_symbol.h new file mode 100644 index 0000000..56d44dd --- /dev/null +++ b/sdk/enclave/src/main/native/cpp/platform/tee_sdk_svm/edge_routines/tee_sdk_symbol.h @@ -0,0 +1,109 @@ +#ifndef _TEE_SDK_SYMBOL_H +#define _TEE_SDK_SYMBOL_H + +#include <stdint.h> +#include <stdio.h> +#include <assert.h> + +void __fxstat(); +void __fxstat64(); +void __isnan(); +void __libc_current_sigrtmax(); +void __libc_malloc(); +void __lxstat(); +void __lxstat64(); +void __sched_cpucount(); +void __strdup(); +void __xmknod(); +void __xstat(); +void __xstat64(); +void chmod(); +void chown(); +void crc32(); +void deflate(); +void deflateBound(); +void deflateEnd(); +void deflateInit2_(); +void deflateSetHeader(); +void dlopen(); +void dlsym(); +void endmntent(); +void fchmod(); +void fchown(); +void fpathconf(); +void fstatvfs(); +void fstatvfs64(); +void getgrnam_r(); +void getmntent_r(); +void getpwnam_r(); +void inflate(); +void inflateEnd(); +void inflateInit2_(); +void inflateReset(); +void inflateSetDictionary(); +void lchown(); +void lstat(); +void mknod(); +void pathconf(); +void pipe(); +void pthread_attr_init(); +void pthread_attr_setdetachstate(); +void pthread_attr_setstacksize(); +void pthread_kill(); +void pthread_setname_np(); +void readlink(); +void realpath(); +void sched_getaffinity(); +void sendfile(); +void sendfile64(); +void setmntent(); +void sigaddset(); +void sigemptyset(); +void sigprocmask(); +void statvfs(); +void statvfs64(); +void symlink(); +void utimes(); + +int posix_memalign(void **memptr, size_t alignment, size_t size); + +unsigned long int pthread_self(); + +typedef struct _thread_data { + uint64_t self_addr; + uint64_t __reserved_0; + uint64_t __stack_base_addr; + uint64_t __stack_limit_addr; + uint64_t __first_ssa_gpr; +} thread_data; + +typedef struct _pthread_attr { + uint64_t __private[7]; +} pthread_attr; + +#define SE_PAGE_SIZE 0x1000 +#define ROUND_TO(x, align) (((x) + ((align)-1)) & ~((align)-1)) +#define ROUND_TO_PAGE(x) ROUND_TO(x, SE_PAGE_SIZE) + +thread_data* get_thread_data(void); +unsigned long int pthread_self(void); +int pthread_attr_getstack(const pthread_attr *a, void ** addr, uint64_t *size); +int pthread_attr_getguardsize(const pthread_attr *a, size_t *size); +int mprotect(); + +// Avoid memory allocation in enclave failed, so we restrict resources by getrlimit +// In enclave. +#define FD_MAX (unsigned long)(64) + +typedef struct { + unsigned long rlim_cur; + unsigned long rlim_max; +} rlimit; + +int getrlimit(int resource, rlimit* rlim); +int setrlimit(); +int pthread_condattr_init(); +int pthread_condattr_setclock(); +int pthread_attr_destroy(); + +#endif /* end of _TEE_SDK_SYMBOL_H */ \ No newline at end of file diff --git a/sdk/enclave/src/main/native/cpp/platform/tee_sdk_svm/wrapper/Makefile b/sdk/enclave/src/main/native/cpp/platform/tee_sdk_svm/wrapper/Makefile new file mode 100644 index 0000000..af20bb3 --- /dev/null +++ b/sdk/enclave/src/main/native/cpp/platform/tee_sdk_svm/wrapper/Makefile @@ -0,0 +1,13 @@ +include $(NATIVE_BASE_DIR)/config/config.mk +include $(NATIVE_BASE_DIR)/config/platform/tee_sdk_svm/jni/config.mk + +.PHONY: all build clean + +all: build + +build: + $(CC) -g -c -I$(INCLUDE) -I$(JAVA_HOME)/lib $(TS_ENCLAVE_INCDIR) $(TS_ENCLAVE_CFLAGS) tee_sdk_wrapper.c \ + -o $(BIN)/platform/tee_sdk_svm/tee_sdk_wrapper.o + +clean: + rm -rf $(BIN)/platform/tee_sdk_svm/tee_sdk_wrapper.o \ No newline at end of file diff --git a/sdk/enclave/src/main/native/cpp/platform/tee_sdk_svm/wrapper/tee_sdk_wrapper.c b/sdk/enclave/src/main/native/cpp/platform/tee_sdk_svm/wrapper/tee_sdk_wrapper.c new file mode 100644 index 0000000..6d651ea --- /dev/null +++ b/sdk/enclave/src/main/native/cpp/platform/tee_sdk_svm/wrapper/tee_sdk_wrapper.c @@ -0,0 +1,83 @@ +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <stdint.h> +#include <sgx_trts.h> + +#include <graal_isolate.h> +#include <enc_environment.h> +#include <enc_exported_symbol.h> + +#include "tee_sdk_wrapper.h" + +typedef int (*enclave_calling_stub)(uint64_t isolate, enc_data_t* input, enc_data_t* output, callbacks_t* callback); + +char* alloc_memory_from_host(char* src, int len) { + int flag = 0; + char *ptr = 0; + ocall_malloc(&flag, len, (void*)&ptr); + // ocall malloc buffer failed. + if (flag != 0x0) { return NULL; } + memcpy(ptr, src, len); + return (char*)ptr; +} + +void tee_sdk_exception_callback(char* err_msg, char* stack_trace, char* exception_name) { + printf("err_msg=%s\n", err_msg); + printf("stack_trace=%s\n", stack_trace); + printf("exception_name=%s\n", exception_name); +} + +int tee_sdk_random(void* data, long size) { + return (int)sgx_read_rand(data, (size_t)size); +} + +int enclave_svm_isolate_create(void* isolate, void* isolateThread) { + graal_isolate_t* isolate_t; + graal_isolatethread_t* thread_t; + + // Implicitly set graal_create_isolate_params_t param as NULL. + int ret = graal_create_isolate(NULL, &isolate_t, &thread_t); + *(uint64_t*)isolate = (uint64_t)isolate_t; + *(uint64_t*)isolateThread = (uint64_t)thread_t; + return ret; +} + +int enclave_svm_isolate_destroy(uint64_t isolateThread) { + return graal_detach_all_threads_and_tear_down_isolate((graal_isolatethread_t*)isolateThread); +} + +int enclave_svm_calling_entry(uint64_t isolate, void* input, size_t input_length, void* output, size_t* output_length, enclave_calling_stub stub) { + enc_data_t request; + enc_data_t response; + + request.data = (char*) input; + request.data_len = input_length; + response.data = NULL; + response.data_len = 0x0; + + callbacks_t callback_methods; + callback_methods.memcpy_char_pointer = &alloc_memory_from_host; + callback_methods.exception_handler = &tee_sdk_exception_callback; + callback_methods.get_random_number = &tee_sdk_random; + + int ret = stub(isolate, &request, &response, &callback_methods); + if(ret != 0) { return ret; } + + *(int64_t*)output = (int64_t)response.data; + *output_length = response.data_len; + + return 0x0; +} + +int load_enclave_svm_services(uint64_t isolate, void* input, size_t input_length, void* output, size_t* output_length) { + return enclave_svm_calling_entry(isolate, input, input_length, output, output_length, (enclave_calling_stub)java_loadservice_invoke); +} + +int invoke_enclave_svm_service(uint64_t isolate, void* input, size_t input_length, void* output, size_t* output_length) { + return enclave_svm_calling_entry(isolate, input, input_length, output, output_length, (enclave_calling_stub)java_enclave_invoke); +} + +int unload_enclave_svm_service(uint64_t isolate, void* input, size_t input_length, void* output, size_t* output_length) { + return enclave_svm_calling_entry(isolate, input, input_length, output, output_length, (enclave_calling_stub)java_unloadservice_invoke); +} \ No newline at end of file diff --git a/sdk/enclave/src/main/native/cpp/platform/tee_sdk_svm/wrapper/tee_sdk_wrapper.h b/sdk/enclave/src/main/native/cpp/platform/tee_sdk_svm/wrapper/tee_sdk_wrapper.h new file mode 100644 index 0000000..0558c91 --- /dev/null +++ b/sdk/enclave/src/main/native/cpp/platform/tee_sdk_svm/wrapper/tee_sdk_wrapper.h @@ -0,0 +1,17 @@ +#ifndef _TEE_SDK_WRAPPER_H_ +#define _TEE_SDK_WRAPPER_H_ + +#include <unistd.h> +#include <stdlib.h> +#include <stdio.h> + +#if defined(__cplusplus) +extern "C" +{ +#endif + void ocall_malloc(int*, int, void*); +#if defined(__cplusplus) +} +#endif + +#endif /* !_TEE_SDK_WRAPPER_H_ */ \ No newline at end of file diff --git a/sdk/enclave/src/main/resources/META-INF/native-image/serialization-config.json b/sdk/enclave/src/main/resources/META-INF/native-image/serialization-config.json index 197813b..0ad2b7e 100644 --- a/sdk/enclave/src/main/resources/META-INF/native-image/serialization-config.json +++ b/sdk/enclave/src/main/resources/META-INF/native-image/serialization-config.json @@ -44,6 +44,9 @@ { "name":"java.io.NotSerializableException" }, + { + "name":"java.io.StreamCorruptedException" + }, { "name":"java.lang.StackTraceElement" }, diff --git a/sdk/enclave/src/main/resources/native/enc_environment.h b/sdk/enclave/src/main/resources/native/enc_environment.h index f7fae35..884c098 100644 --- a/sdk/enclave/src/main/resources/native/enc_environment.h +++ b/sdk/enclave/src/main/resources/native/enc_environment.h @@ -1,25 +1,25 @@ - typedef struct enc_data_struct{ +typedef struct enc_data_struct{ //char array is used as byte array to store serialized data char* data; int data_len; - }enc_data_t; +} enc_data_t; typedef struct callback_functions_struct{ - /* - * This method is invoked inside java_enclave_invoke method's exception catch - * section, when the execution is aborted by exceptions. The caller side can - * decide what to do with the exception. - * Exception details are passed back with parameters. + /* + * This method is invoked inside java_enclave_invoke method's exception catch + * section, when the execution is aborted by exceptions. The caller side can + * decide what to do with the exception. + * Exception details are passed back with parameters. */ void (*exception_handler)(char* err_msg, char* stack_trace, char* exception_name); char* (*memcpy_char_pointer)(char* src, int len); /* - * Points to an available pseudorandom number generating function. - */ + * Points to an available pseudorandom number generating function. + */ int (*get_random_number)(void* data, long size); -}callbacks_t; +} callbacks_t; long physical_page_size(); long physical_page_number(); diff --git a/sdk/host/pom.xml b/sdk/host/pom.xml index 05b0c51..1a60a7a 100644 --- a/sdk/host/pom.xml +++ b/sdk/host/pom.xml @@ -27,7 +27,7 @@ <configuration> <executable>bash</executable> <arguments> - <argument>${project.basedir}/src/main/native/make.sh</argument> + <argument>${project.basedir}/../native/script/build_host_sdk/make.sh</argument> <argument>${project.basedir}</argument> <argument>${com.alibaba.enclave.platform}</argument> <argument>build</argument> diff --git a/sdk/host/src/main/java/com/alibaba/confidentialcomputing/host/AbstractEnclave.java b/sdk/host/src/main/java/com/alibaba/confidentialcomputing/host/AbstractEnclave.java index a5eea7d..15eefb1 100644 --- a/sdk/host/src/main/java/com/alibaba/confidentialcomputing/host/AbstractEnclave.java +++ b/sdk/host/src/main/java/com/alibaba/confidentialcomputing/host/AbstractEnclave.java @@ -180,7 +180,7 @@ abstract class AbstractEnclave implements Enclave { Class<?>[] serviceInterface = new Class[1]; serviceInterface[0] = service; - List<T> serviceProxies = new ArrayList<T>(); + List<T> serviceProxies = new ArrayList<>(); ServiceHandler[] services = loadService(service); for (ServiceHandler serviceHandler : services) { ProxyEnclaveInvocationHandler handler = new ProxyEnclaveInvocationHandler(this, serviceHandler); diff --git a/sdk/host/src/main/java/com/alibaba/confidentialcomputing/host/MockInSvmEnclave.java b/sdk/host/src/main/java/com/alibaba/confidentialcomputing/host/MockInSvmEnclave.java index 45575e8..3c4d030 100644 --- a/sdk/host/src/main/java/com/alibaba/confidentialcomputing/host/MockInSvmEnclave.java +++ b/sdk/host/src/main/java/com/alibaba/confidentialcomputing/host/MockInSvmEnclave.java @@ -96,14 +96,11 @@ class MockInSvmEnclave extends AbstractEnclave { // interrupt enclave services' recycler firstly. this.getEnclaveContext().getEnclaveServicesRecycler().interruptServiceRecycler(); // destroy svm isolate. - int ret = nativeSvmDetachIsolate( - enclaveSvmSdkHandle, - isolateThreadHandle); + int ret = nativeSvmDetachIsolate(enclaveSvmSdkHandle, isolateThreadHandle); if (ret != 0) { throw new EnclaveDestroyingException("isolate destroy native call failed."); } - ret = nativeDestroyEnclave( - enclaveSvmSdkHandle); + ret = nativeDestroyEnclave(enclaveSvmSdkHandle); if (ret != 0) { throw new EnclaveDestroyingException("enclave destroy native call failed."); } diff --git a/sdk/host/src/main/java/com/alibaba/confidentialcomputing/host/TeeSdkEnclave.java b/sdk/host/src/main/java/com/alibaba/confidentialcomputing/host/TeeSdkEnclave.java index 2cd03f7..ef71ece 100644 --- a/sdk/host/src/main/java/com/alibaba/confidentialcomputing/host/TeeSdkEnclave.java +++ b/sdk/host/src/main/java/com/alibaba/confidentialcomputing/host/TeeSdkEnclave.java @@ -10,11 +10,17 @@ import java.io.IOException; * TeeSdkEnclave is a sgx2 enclave based on Alibaba cloud's tee sdk. */ class TeeSdkEnclave extends AbstractEnclave { - private final static String JNI_EXTRACTED_PACKAGE_PATH = "jni/lib_jni_tee_sdk.so"; - private final static String TEE_SDK_SIGNED_PACKAGE_PATH = "libs/lib_enclave_tee_sdk.signed"; + private final static String JNI_EXTRACTED_PACKAGE_PATH = "jni/lib_jni_tee_sdk_svm.so"; + private final static String TEE_SDK_SIGNED_PACKAGE_PATH = "lib_tee_sdk_svm_load.signed"; private static volatile TeeSdkExtractTempPath extractTempPath; - private final EnclaveNativeContext nativeHandlerContext = new EnclaveNativeContext( - 0, 0, 0); + + // enclaveHandle stores created enclave's handle id. + private long enclaveHandle; + // isolate stores svm created isolate instance. + // In JavaEnclave only one isolateHandle instance will be created. + private long isolateHandle; + // isolateThreadHandle stores the first attached isolateThread Handle. + private long isolateThreadHandle; TeeSdkEnclave(EnclaveDebug mode) throws EnclaveCreatingException { // Set EnclaveContext for this enclave instance. @@ -33,7 +39,8 @@ class TeeSdkEnclave extends AbstractEnclave { TeeSdkEnclave.class.getClassLoader(), TEE_SDK_SIGNED_PACKAGE_PATH); extractTempPath = new TeeSdkExtractTempPath(jniTempFilePath, teeSdkSignedFilePath); - System.load(jniTempFilePath); + System.load(extractTempPath.getJniTempFilePath()); + registerNatives(); } catch (IOException e) { throw new EnclaveCreatingException("extracting tee sdk jni .so or signed .so failed.", e); } @@ -47,12 +54,14 @@ class TeeSdkEnclave extends AbstractEnclave { throw new EnclaveCreatingException("create tee sdk enclave by native calling failed."); } // Create svm attach isolate and isolateThread, and they are set in jni in nativeHandlerContext. - ret = nativeSvmAttachIsolate(nativeHandlerContext.getEnclaveHandle()); + ret = nativeSvmAttachIsolate(enclaveHandle); if (ret != 0) { throw new EnclaveCreatingException("create svm isolate by native calling failed."); } } + private static native void registerNatives(); + private native int nativeCreateEnclave(int mode, String path); private native InnerNativeInvocationResult nativeGenerateAttestationReport(byte[] userData); @@ -96,26 +105,17 @@ class TeeSdkEnclave extends AbstractEnclave { @Override InnerNativeInvocationResult loadServiceNative(byte[] payload) { - return nativeLoadService( - nativeHandlerContext.getEnclaveHandle(), - nativeHandlerContext.getIsolateHandle(), - payload); + return nativeLoadService(enclaveHandle, isolateHandle, payload); } @Override InnerNativeInvocationResult unloadServiceNative(byte[] payload) { - return nativeUnloadService( - nativeHandlerContext.getEnclaveHandle(), - nativeHandlerContext.getIsolateHandle(), - payload); + return nativeUnloadService(enclaveHandle, isolateHandle, payload); } @Override InnerNativeInvocationResult invokeMethodNative(byte[] payload) { - return nativeInvokeMethod( - nativeHandlerContext.getEnclaveHandle(), - nativeHandlerContext.getIsolateHandle(), - payload); + return nativeInvokeMethod(enclaveHandle, isolateHandle, payload); } @Override @@ -125,13 +125,12 @@ class TeeSdkEnclave extends AbstractEnclave { // interrupt enclave services' recycler firstly. this.getEnclaveContext().getEnclaveServicesRecycler().interruptServiceRecycler(); // destroy svm isolate. - int ret = nativeSvmDetachIsolate(nativeHandlerContext.getEnclaveHandle(), - nativeHandlerContext.getIsolateThreadHandle()); + int ret = nativeSvmDetachIsolate(enclaveHandle, isolateThreadHandle); if (ret != 0) { throw new EnclaveDestroyingException("isolate destroy native call failed."); } // destroy the enclave. - ret = nativeDestroyEnclave(nativeHandlerContext.getEnclaveHandle()); + ret = nativeDestroyEnclave(enclaveHandle); if (ret != 0) { throw new EnclaveDestroyingException("enclave destroy native call failed."); } @@ -155,36 +154,4 @@ class TeeSdkEnclave extends AbstractEnclave { return teeSdkSignedFilePath; } } - - /** - * JavaEnclave will create svm isolate handle and isolateThread handle by native call, - * so EnclaveNativeContextCache will cache them for usage. - */ - class EnclaveNativeContext { - // enclaveHandle stores created enclave's handle id. - private final long enclaveHandle; - // isolate stores svm created isolate instance. - // In JavaEnclave only one isolateHandle instance will be created. - private final long isolateHandle; - // isolateThreadHandle stores the first attached isolateThread Handle. - private final long isolateThreadHandle; - - EnclaveNativeContext(long enclaveHandle, long isolateHandle, long isolateThreadHandle) { - this.enclaveHandle = enclaveHandle; - this.isolateHandle = isolateHandle; - this.isolateThreadHandle = isolateThreadHandle; - } - - long getEnclaveHandle() { - return enclaveHandle; - } - - long getIsolateHandle() { - return isolateHandle; - } - - long getIsolateThreadHandle() { - return isolateThreadHandle; - } - } } diff --git a/sdk/host/src/main/native/Makefile b/sdk/host/src/main/native/Makefile deleted file mode 100644 index b498a00..0000000 --- a/sdk/host/src/main/native/Makefile +++ /dev/null @@ -1,11 +0,0 @@ -# Copyright (c) - -.PHONY: all build clean - -all: build - -build: - $(MAKE) -C cpp/platform/mock_in_svm/jni - -clean: - $(MAKE) -C cpp/platform/mock_in_svm/jni clean \ No newline at end of file diff --git a/sdk/host/src/main/native/config/config.mk b/sdk/host/src/main/native/config/config.mk deleted file mode 100644 index a34d544..0000000 --- a/sdk/host/src/main/native/config/config.mk +++ /dev/null @@ -1,13 +0,0 @@ -# parse enable MOCK_IN_SVM platform, ${MOCK_IN_SVM} is from make.sh script. -BUILD_MOCK_IN_SVM ?= $(shell echo ${MOCK_IN_SVM}) -# parse JavaEnclave SDK base dir path, ${base_dir} is from make.sh script. -BASE_DIR_PATH = $(shell echo ${base_dir}) - -# parse BIN path. -BIN = $(BASE_DIR_PATH)/src/main/native/bin -# parse CONFIG path. -CONFIG = $(BASE_DIR_PATH)/src/main/native/config -# parse CPP path. -CPP = $(BASE_DIR_PATH)/src/main/native/cpp -# parse INCLUDE path. -INCLUDE = $(BASE_DIR_PATH)/src/main/native/include \ No newline at end of file diff --git a/sdk/host/src/main/native/cpp/platform/mock_in_svm/jni/Makefile b/sdk/host/src/main/native/cpp/platform/mock_in_svm/jni/Makefile index 2e8074e..6ff8ded 100644 --- a/sdk/host/src/main/native/cpp/platform/mock_in_svm/jni/Makefile +++ b/sdk/host/src/main/native/cpp/platform/mock_in_svm/jni/Makefile @@ -1,5 +1,5 @@ -include ./../../../../config/config.mk -include ./../../../../config/platform/mock_in_svm/jni/config.mk +include $(NATIVE_BASE_DIR)/config/config.mk +include $(NATIVE_BASE_DIR)/config/platform/mock_in_svm/jni/config.mk .PHONY: all build clean diff --git a/sdk/host/src/main/native/cpp/platform/mock_in_svm/jni/jni_mock_in_svm.c b/sdk/host/src/main/native/cpp/platform/mock_in_svm/jni/jni_mock_in_svm.c index 1a8d6d8..88be3a0 100644 --- a/sdk/host/src/main/native/cpp/platform/mock_in_svm/jni/jni_mock_in_svm.c +++ b/sdk/host/src/main/native/cpp/platform/mock_in_svm/jni/jni_mock_in_svm.c @@ -13,7 +13,7 @@ #include "jni_mock_in_svm.h" -typedef int (*java_enclave_stub)(graal_isolate_t*, enc_data_t*, enc_data_t*, callbacks_t*); +typedef int (*mock_enclave_stub)(graal_isolate_t*, enc_data_t*, enc_data_t*, callbacks_t*); static JNINativeMethod mock_in_svm_methods[] = { {"nativeCreateEnclave", "(Ljava/lang/String;)I", (void *)&JavaEnclave_MockSVMNativeCreateEnclave}, @@ -41,6 +41,7 @@ void set_long_field_value(JNIEnv *env, jclass class_mirror, jobject obj, const c char* memcpy_char_pointer(char* src, int len) { char *ptr = malloc(len); + if (ptr == NULL) { return NULL; } memcpy(ptr, src, len); return (char*)ptr; } @@ -52,37 +53,42 @@ jobject build_invocation_result(JNIEnv *env, jint ret, jbyteArray array) { return (*env)->NewObject(env, invocation_result_clazz, id, (jint)ret, array); } -jobject service_operate_common(JNIEnv *env, jlong isolate_handler, jbyteArray payload, java_enclave_stub p_function) { - jbyte *service_payload_copy = (*env)->GetByteArrayElements(env, payload, NULL); - int service_payload_copy_length = (*env)->GetArrayLength(env, payload); - enc_data_t invoke_data; - invoke_data.data = (char*)service_payload_copy; - invoke_data.data_len = service_payload_copy_length; - enc_data_t result; - result.data = NULL; - result.data_len = 0x0; +jobject mock_enclave_calling_entry(JNIEnv *env, jlong isolate_handler, jbyteArray payload, mock_enclave_stub stub) { + jbyte *payload_copy = (*env)->GetByteArrayElements(env, payload, NULL); + int payload_copy_length = (*env)->GetArrayLength(env, payload); + + enc_data_t input; + input.data = (char*)payload_copy; + input.data_len = payload_copy_length; + enc_data_t output; + output.data = NULL; + output.data_len = 0x0; + callbacks_t callback_methods; callback_methods.memcpy_char_pointer = &memcpy_char_pointer; callback_methods.exception_handler = NULL; - int ret = p_function((graal_isolate_t*)isolate_handler, &invoke_data, &result, &callback_methods); - (*env)->ReleaseByteArrayElements(env, payload, service_payload_copy, 0); + callback_methods.get_random_number = NULL; + int ret = stub((graal_isolate_t*)isolate_handler, &input, &output, &callback_methods); // create a byte array. - jbyteArray invocation_result_arr = (*env)->NewByteArray(env, result.data_len); - jbyte *invocation_result_arr_point = (*env)->GetByteArrayElements(env, invocation_result_arr, NULL); - memcpy(invocation_result_arr_point, result.data, result.data_len); + jbyteArray invocation_result_array = (*env)->NewByteArray(env, output.data_len); + jbyte *invocation_result_array_ptr = (*env)->GetByteArrayElements(env, invocation_result_array, NULL); + memcpy(invocation_result_array_ptr, output.data, output.data_len); + (*env)->ReleaseByteArrayElements(env, payload, payload_copy, 0); // free buffer malloc in jni. - (*env)->ReleaseByteArrayElements(env, invocation_result_arr, invocation_result_arr_point, 0); + (*env)->ReleaseByteArrayElements(env, invocation_result_array, invocation_result_array_ptr, 0); // free buffer malloc in native image by callback mechanism. - free(result.data); + free(output.data); - return build_invocation_result(env, ret, invocation_result_arr); + return build_invocation_result(env, ret, invocation_result_array); } -JNIEXPORT jint JNICALL JavaEnclave_MockSVMNativeCreateEnclave(JNIEnv *env, jobject obj, jstring path) { +JNIEXPORT jint JNICALL +JavaEnclave_MockSVMNativeCreateEnclave(JNIEnv *env, jobject obj, jstring path) { const char *path_str = (path == 0) ? 0 : (*env)->GetStringUTFChars(env, path, 0); void *enclave_handler = dlopen(path_str , RTLD_LOCAL | RTLD_LAZY); + (*env)->ReleaseStringUTFChars(env, path, path_str); if (enclave_handler == 0x0) { fprintf(stderr, "mock in svm dlopen error:%s\n", dlerror()); return -1; @@ -114,7 +120,8 @@ JNIEXPORT jint JNICALL JavaEnclave_MockSVMNativeCreateEnclave(JNIEnv *env, jobje return 0; } -JNIEXPORT jint JNICALL JavaEnclave_MockSVMNativeSvmAttachIsolate(JNIEnv *env, jobject obj, jlong enclave_handler) { +JNIEXPORT jint JNICALL +JavaEnclave_MockSVMNativeSvmAttachIsolate(JNIEnv *env, jobject obj, jlong enclave_handler) { graal_isolate_t* isolate_t; graal_create_isolate_params_t p; graal_isolatethread_t* isolate_thread_t; @@ -140,23 +147,23 @@ JNIEXPORT jint JNICALL JavaEnclave_MockSVMNativeSvmAttachIsolate(JNIEnv *env, jo return ret; } -JNIEXPORT jobject JNICALL JavaEnclave_MockSVMNativeLoadService(JNIEnv *env, jobject obj, jlong enclave_handler, -jlong isolate_handler, jbyteArray service_payload) { - return service_operate_common(env, isolate_handler, service_payload, (java_enclave_stub) mock_in_svm_load_service_symbol); +JNIEXPORT jobject JNICALL +JavaEnclave_MockSVMNativeLoadService(JNIEnv *env, jobject obj, jlong enclave_handler, jlong isolate_handler, jbyteArray load_service_payload) { + return mock_enclave_calling_entry(env, isolate_handler, load_service_payload, (mock_enclave_stub) mock_in_svm_load_service_symbol); } -JNIEXPORT jobject JNICALL JavaEnclave_MockSVMNativeInvokeMethod(JNIEnv *env, jobject obj, jlong enclave_handler, -jlong isolate_handler, jbyteArray invoke_wrapper_payload) { - return service_operate_common(env, isolate_handler, invoke_wrapper_payload, (java_enclave_stub) mock_in_svm_invoke_service_symbol); +JNIEXPORT jobject JNICALL +JavaEnclave_MockSVMNativeInvokeMethod(JNIEnv *env, jobject obj, jlong enclave_handler, jlong isolate_handler, jbyteArray invoke_payload) { + return mock_enclave_calling_entry(env, isolate_handler, invoke_payload, (mock_enclave_stub) mock_in_svm_invoke_service_symbol); } -JNIEXPORT jobject JNICALL JavaEnclave_MockSVMNativeUnloadService(JNIEnv *env, jobject obj, jlong enclave_handler, -jlong isolate_handler, jbyteArray service_payload) { - return service_operate_common(env, isolate_handler, service_payload, (java_enclave_stub) mock_in_svm_unload_service_symbol); +JNIEXPORT jobject JNICALL +JavaEnclave_MockSVMNativeUnloadService(JNIEnv *env, jobject obj, jlong enclave_handler, jlong isolate_handler, jbyteArray unload_service_payload) { + return mock_enclave_calling_entry(env, isolate_handler, unload_service_payload, (mock_enclave_stub) mock_in_svm_unload_service_symbol); } -JNIEXPORT jint JNICALL JavaEnclave_MockSVMNativeSvmDetachIsolate(JNIEnv *env, jobject obj, jlong enclave_handler, -jlong isolate_thread_handler) { +JNIEXPORT jint JNICALL +JavaEnclave_MockSVMNativeSvmDetachIsolate(JNIEnv *env, jobject obj, jlong enclave_handler, jlong isolate_thread_handler) { int (*graal_detach_all_threads_and_tear_down_isolate)(graal_isolatethread_t* isolateThread); graal_detach_all_threads_and_tear_down_isolate = (int (*)(graal_isolatethread_t*)) dlsym((void *)enclave_handler, "graal_detach_all_threads_and_tear_down_isolate"); @@ -167,6 +174,7 @@ jlong isolate_thread_handler) { return (jint)graal_detach_all_threads_and_tear_down_isolate((graal_isolatethread_t*)isolate_thread_handler); } -JNIEXPORT jint JNICALL JavaEnclave_MockSVMNativeDestroyEnclave(JNIEnv *env, jobject obj, jlong enclave_handler) { +JNIEXPORT jint JNICALL +JavaEnclave_MockSVMNativeDestroyEnclave(JNIEnv *env, jobject obj, jlong enclave_handler) { return dlclose((void *)enclave_handler); } diff --git a/sdk/host/src/main/native/cpp/platform/tee_sdk_svm/Makefile b/sdk/host/src/main/native/cpp/platform/tee_sdk_svm/Makefile new file mode 100644 index 0000000..88dcd14 --- /dev/null +++ b/sdk/host/src/main/native/cpp/platform/tee_sdk_svm/Makefile @@ -0,0 +1,23 @@ +# Copyright (c) + +include $(NATIVE_BASE_DIR)/config/config.mk +include $(NATIVE_BASE_DIR)/config/platform/tee_sdk_svm/jni/config.mk + +.PHONY: all build clean + +all: build + +build: jni.o + $(CC) edge_routines/ocall.o edge_routines/tee_sdk_enclave_u.o jni/jni_tee_sdk_svm.o \ + $(TS_HOST_CFLAGS) $(TS_HOST_LDFLAGS) -fPIC -shared -o $(BIN)/platform/tee_sdk_svm/jni/lib_jni_tee_sdk_svm.so + rm -rf edge_routines/*.o edge_routines/tee_sdk_enclave_u.c jni/*.o + +edge_routines.o: + $(MAKE) -C edge_routines + +jni.o: edge_routines.o + $(MAKE) -C jni + +clean: + $(MAKE) -C edge_routines clean + $(MAKE) -C jni clean \ No newline at end of file diff --git a/sdk/host/src/main/native/cpp/platform/tee_sdk_svm/edge_routines/Makefile b/sdk/host/src/main/native/cpp/platform/tee_sdk_svm/edge_routines/Makefile new file mode 100644 index 0000000..b5271b9 --- /dev/null +++ b/sdk/host/src/main/native/cpp/platform/tee_sdk_svm/edge_routines/Makefile @@ -0,0 +1,18 @@ +# Copyright (c) + +include $(NATIVE_BASE_DIR)/config/config.mk +include $(NATIVE_BASE_DIR)/config/platform/tee_sdk_svm/jni/config.mk + +.PHONY: all build clean + +all: build + +build: + $(SGX_EDGER8R) $(CONFIG)/platform/tee_sdk_svm/edl/tee_sdk_enclave.edl --untrusted \ + --search-path $(TEE_SDK_PATH)/include + + $(CC) -g -c -fPIC $(TS_HOST_INCDIR) $(TS_HOST_CFLAGS) -fPIC ocall.c + $(CC) -g -c -fPIC $(TS_HOST_INCDIR) $(TS_HOST_CFLAGS) -fPIC tee_sdk_enclave_u.c + +clean: + rm -rf tee_sdk_enclave_* *.o \ No newline at end of file diff --git a/sdk/host/src/main/native/cpp/platform/tee_sdk_svm/edge_routines/ocall.c b/sdk/host/src/main/native/cpp/platform/tee_sdk_svm/edge_routines/ocall.c new file mode 100644 index 0000000..36166ee --- /dev/null +++ b/sdk/host/src/main/native/cpp/platform/tee_sdk_svm/edge_routines/ocall.c @@ -0,0 +1,14 @@ +#include "ocall.h" + +int ocall_getrlimit(int resource, void *rlim) { + return getrlimit(resource, (struct rlimit *)rlim); +} + +int ocall_malloc(size_t size, void *ptr) { + void* memptr = malloc(size); + if (memptr != NULL) { + *((char **)ptr) = (char *)memptr; + return 0; + } + return -1; +} \ No newline at end of file diff --git a/sdk/host/src/main/native/cpp/platform/tee_sdk_svm/edge_routines/ocall.h b/sdk/host/src/main/native/cpp/platform/tee_sdk_svm/edge_routines/ocall.h new file mode 100644 index 0000000..3940471 --- /dev/null +++ b/sdk/host/src/main/native/cpp/platform/tee_sdk_svm/edge_routines/ocall.h @@ -0,0 +1,20 @@ +#ifndef _OCALL_H_ +#define _OCALL_H_ + +#include <sys/resource.h> +#include <sys/mman.h> +#include <unistd.h> +#include <stdlib.h> +#include <stdio.h> + +#if defined(__cplusplus) +extern "C" +{ +#endif + int ocall_getrlimit(int resource, void *rlim); + int ocall_malloc(size_t size, void *ptr); +#if defined(__cplusplus) +} +#endif + +#endif /* !_OCALL_H_ */ \ No newline at end of file diff --git a/sdk/host/src/main/native/cpp/platform/tee_sdk_svm/jni/Makefile b/sdk/host/src/main/native/cpp/platform/tee_sdk_svm/jni/Makefile new file mode 100644 index 0000000..e635614 --- /dev/null +++ b/sdk/host/src/main/native/cpp/platform/tee_sdk_svm/jni/Makefile @@ -0,0 +1,15 @@ +# Copyright (c) + +include $(NATIVE_BASE_DIR)/config/config.mk +include $(NATIVE_BASE_DIR)/config/platform/tee_sdk_svm/jni/config.mk + +.PHONY: all build clean + +all: build + +build: + $(CC) -g -c -Wno-unused-parameter -fPIC $(TS_HOST_CFLAGS) $(TS_HOST_INCDIR) -I./../edge_routines -I$(JAVA_HOME)/lib -I$(INCLUDE) -I$(JAVA_HOME)/include \ + -I$(JAVA_HOME)/include/$(shell uname -s | tr A-Z a-z) jni_tee_sdk_svm.c + +clean: + rm -rf *.o \ No newline at end of file diff --git a/sdk/host/src/main/native/cpp/platform/tee_sdk_svm/jni/jni_tee_sdk_svm.c b/sdk/host/src/main/native/cpp/platform/tee_sdk_svm/jni/jni_tee_sdk_svm.c new file mode 100644 index 0000000..7142959 --- /dev/null +++ b/sdk/host/src/main/native/cpp/platform/tee_sdk_svm/jni/jni_tee_sdk_svm.c @@ -0,0 +1,146 @@ +#include <assert.h> +#include <limits.h> +#include <string.h> +#include <stdio.h> +#include <stdlib.h> +#include <sys/stat.h> +#include <sys/types.h> +#include <dlfcn.h> + +#include <sgx_urts.h> + +#include <graal_isolate.h> +#include <enc_environment.h> +#include <enc_exported_symbol.h> + +#include "tee_sdk_enclave_u.h" +#include "jni_tee_sdk_svm.h" + +typedef void (*enclave_calling_stub)(jlong, int*, graal_isolate_t*, void*, size_t, void*, size_t*); + +static JNINativeMethod tee_sdk_svm_methods[] = { + {"nativeCreateEnclave", "(ILjava/lang/String;)I", (void *)&JavaEnclave_TeeSDKSVMNativeCreateEnclave}, + {"nativeSvmAttachIsolate", "(J)I", (void *)&JavaEnclave_TeeSDKSVMNativeSvmAttachIsolate}, + {"nativeLoadService", TEE_SDK_SVM_NATIVE_CALL_SIGNATURE, (void *)&JavaEnclave_TeeSDKSVMNativeLoadService}, + {"nativeInvokeMethod", TEE_SDK_SVM_NATIVE_CALL_SIGNATURE, (void *)&JavaEnclave_TeeSDKSVMNativeInvokeMethod}, + {"nativeUnloadService", TEE_SDK_SVM_NATIVE_CALL_SIGNATURE, (void *)&JavaEnclave_TeeSDKSVMNativeUnloadService}, + {"nativeSvmDetachIsolate", "(JJ)I", (void *)&JavaEnclave_TeeSDKSVMNativeSvmDetachIsolate}, + {"nativeDestroyEnclave", "(J)I", (void *)&JavaEnclave_TeeSDKSVMNativeDestroyEnclave}, +}; + +JNIEXPORT void JNICALL +Java_com_alibaba_confidentialcomputing_host_TeeSdkEnclave_registerNatives(JNIEnv *env, jclass cls) { + (*env)->RegisterNatives(env, cls, tee_sdk_svm_methods, sizeof(tee_sdk_svm_methods)/sizeof(tee_sdk_svm_methods[0])); +} + +void set_long_field_value(JNIEnv *env, jclass class_mirror, jobject obj, const char *field_name, jlong value) { + jfieldID field_id = (*env)->GetFieldID(env, class_mirror, field_name, "J"); + (*env)->SetLongField(env, obj, field_id, value); +} + +jobject build_invocation_result(JNIEnv *env, jint ret, jbyteArray array) { + // build jni return object InnerNativeInvocationResult. + jclass invocation_result_clazz = (*env)->FindClass(env, TEE_SDK_SVM_RETURN_OBJECT_SIGNATURE); + jmethodID id = (*env)->GetMethodID(env, invocation_result_clazz, "<init>", "(I[B)V"); + return (*env)->NewObject(env, invocation_result_clazz, id, (jint)ret, array); +} + +jobject enclave_calling_entry(JNIEnv *env, jlong enclave_handler, jlong isolate_handler, jbyteArray payload, enclave_calling_stub stub) { + jbyte *payload_copy = (*env)->GetByteArrayElements(env, payload, NULL); + int payload_copy_length = (*env)->GetArrayLength(env, payload); + + enc_data_t input; + input.data = (char*)payload_copy; + input.data_len = payload_copy_length; + enc_data_t output; + output.data = NULL; + output.data_len = 0x0; + + int ret = 0x0; + stub(enclave_handler, &ret, (graal_isolate_t*)isolate_handler, (void*)(input.data), (size_t)(input.data_len), (void*)(&(output.data)), (size_t*)(&(output.data_len))); + + // create a byte array. + jbyteArray invocation_result_array = (*env)->NewByteArray(env, output.data_len); + jbyte *invocation_result_array_ptr = (*env)->GetByteArrayElements(env, invocation_result_array, NULL); + memcpy(invocation_result_array_ptr, output.data, (size_t)output.data_len); + + (*env)->ReleaseByteArrayElements(env, payload, payload_copy, 0); + // free buffer malloc in jni. + (*env)->ReleaseByteArrayElements(env, invocation_result_array, invocation_result_array_ptr, 0); + // free buffer malloc in native image by callback mechanism. + free(output.data); + + return build_invocation_result(env, ret, invocation_result_array); +} + +JNIEXPORT jint JNICALL +JavaEnclave_TeeSDKSVMNativeCreateEnclave(JNIEnv *env, jobject obj, jint mode, jstring path) { + // set enclave' debug mode enable_debug_mode. + // mode = 0, is SGX_RELEASE_FLAG + // mode = 1, is SGX_DEBUG_FLAG + int enable_debug_mode = 0; + if (mode == 1) { + enable_debug_mode = (int)SGX_DEBUG_FLAG; + } + + // create a tee sdk sgx enclave instance. + const char *path_str = (path == 0) ? 0 : (*env)->GetStringUTFChars(env, path, 0); + sgx_enclave_id_t enclave_id; + int ret = sgx_create_enclave(path_str, enable_debug_mode, NULL, NULL, &enclave_id, NULL); + + (*env)->ReleaseStringUTFChars(env, path, path_str); + + if (ret != SGX_SUCCESS) { + return (int)ret; + } + + // set enclave_handler back to TeeSdkEnclave.enclaveHandle field. + jclass enclave_class = (*env)->GetObjectClass(env, obj); + set_long_field_value(env, enclave_class, obj, "enclaveHandle", (jlong)enclave_id); + + return 0; +} + +JNIEXPORT jint JNICALL +JavaEnclave_TeeSDKSVMNativeSvmAttachIsolate(JNIEnv *env, jobject obj, jlong enclave_handler) { + // create an isolate in enclave. + uint64_t isolate = 0; + uint64_t isolateThread = 0; + int ret = 0; + enclave_svm_isolate_create((size_t)enclave_handler, &ret, (void *)(&isolate), (void *)(&isolateThread)); + + jclass enclave_class = (*env)->GetObjectClass(env, obj); + // set isolate back to isolateHandle field. + set_long_field_value(env, enclave_class, obj, "isolateHandle", (jlong)isolate); + // set isolateThread back to isolateThreadHandle field. + set_long_field_value(env, enclave_class, obj, "isolateThreadHandle", (jlong)isolateThread); + + return ret; +} + +JNIEXPORT jobject JNICALL +JavaEnclave_TeeSDKSVMNativeLoadService(JNIEnv *env, jobject obj, jlong enclave_handler, jlong isolate_handler, jbyteArray load_service_payload) { + return enclave_calling_entry(env, enclave_handler, isolate_handler, load_service_payload, (enclave_calling_stub) load_enclave_svm_services); +} + +JNIEXPORT jobject JNICALL +JavaEnclave_TeeSDKSVMNativeInvokeMethod(JNIEnv *env, jobject obj, jlong enclave_handler, jlong isolate_handler, jbyteArray invoke_service_payload) { + return enclave_calling_entry(env, enclave_handler, isolate_handler, invoke_service_payload, (enclave_calling_stub) invoke_enclave_svm_service); +} + +JNIEXPORT jobject JNICALL +JavaEnclave_TeeSDKSVMNativeUnloadService(JNIEnv *env, jobject obj, jlong enclave_handler, jlong isolate_handler, jbyteArray unload_service_payload) { + return enclave_calling_entry(env, enclave_handler, isolate_handler, unload_service_payload, (enclave_calling_stub) unload_enclave_svm_service); +} + +JNIEXPORT jint JNICALL +JavaEnclave_TeeSDKSVMNativeSvmDetachIsolate(JNIEnv *env, jobject obj, jlong enclave_handler, jlong isolate_thread_handler) { + int ret = 0x0; + enclave_svm_isolate_destroy((sgx_enclave_id_t)enclave_handler, &ret, (uint64_t)isolate_thread_handler); + return ret; +} + +JNIEXPORT jint JNICALL +JavaEnclave_TeeSDKSVMNativeDestroyEnclave(JNIEnv *env, jobject obj, jlong enclave_handler) { + return (jint)sgx_destroy_enclave((sgx_enclave_id_t)enclave_handler); +} \ No newline at end of file diff --git a/sdk/host/src/main/native/cpp/platform/tee_sdk_svm/jni/jni_tee_sdk_svm.h b/sdk/host/src/main/native/cpp/platform/tee_sdk_svm/jni/jni_tee_sdk_svm.h new file mode 100644 index 0000000..7ac0650 --- /dev/null +++ b/sdk/host/src/main/native/cpp/platform/tee_sdk_svm/jni/jni_tee_sdk_svm.h @@ -0,0 +1,66 @@ +#include <jni.h> + +#ifndef _Included_jni_tee_sdk_svm +#define _Included_jni_tee_sdk_svm + +#define TEE_SDK_SVM_NATIVE_CALL_SIGNATURE "(JJ[B)Lcom/alibaba/confidentialcomputing/host/InnerNativeInvocationResult;" +#define TEE_SDK_SVM_RETURN_OBJECT_SIGNATURE "com/alibaba/confidentialcomputing/host/InnerNativeInvocationResult" + +#ifdef __cplusplus +extern "C" { +#endif +JNIEXPORT void JNICALL Java_com_alibaba_confidentialcomputing_host_TeeSdkEnclave_registerNatives(JNIEnv *env, jclass cls); + +/* + * Class: com_alibaba_confidentialcomputing_host_TeeSdkEnclave + * Method: nativeCreateEnclave + * Signature: (ILjava/lang/String;)I + */ +JNIEXPORT jint JNICALL JavaEnclave_TeeSDKSVMNativeCreateEnclave(JNIEnv *, jobject, jint, jstring); + +/* + * Class: com_alibaba_confidentialcomputing_host_TeeSdkEnclave + * Method: nativeSvmAttachIsolate + * Signature: (J)I + */ +JNIEXPORT jint JNICALL JavaEnclave_TeeSDKSVMNativeSvmAttachIsolate(JNIEnv *, jobject, jlong); + +/* + * Class: com_alibaba_confidentialcomputing_host_TeeSdkEnclave + * Method: nativeLoadService + * Signature: (JJ[B)Lcom/alibaba/confidentialcomputing/host/InnerNativeInvocationResult; + */ +JNIEXPORT jobject JNICALL JavaEnclave_TeeSDKSVMNativeLoadService(JNIEnv *, jobject, jlong, jlong, jbyteArray); + +/* + * Class: com_alibaba_confidentialcomputing_host_TeeSdkEnclave + * Method: nativeInvokeMethod + * Signature: (JJ[B)Lcom/alibaba/confidentialcomputing/host/InnerNativeInvocationResult; + */ +JNIEXPORT jobject JNICALL JavaEnclave_TeeSDKSVMNativeInvokeMethod(JNIEnv *, jobject, jlong, jlong, jbyteArray); + +/* + * Class: com_alibaba_confidentialcomputing_host_TeeSdkEnclave + * Method: nativeUnloadService + * Signature: (JJ[B)Lcom/alibaba/confidentialcomputing/host/InnerNativeInvocationResult; + */ +JNIEXPORT jobject JNICALL JavaEnclave_TeeSDKSVMNativeUnloadService(JNIEnv *, jobject, jlong, jlong, jbyteArray); + +/* + * Class: com_alibaba_confidentialcomputing_host_TeeSdkEnclave + * Method: nativeSvmDetachIsolate + * Signature: (JJ)I + */ +JNIEXPORT jint JNICALL JavaEnclave_TeeSDKSVMNativeSvmDetachIsolate(JNIEnv *, jobject, jlong, jlong); + +/* + * Class: com_alibaba_confidentialcomputing_host_TeeSdkEnclave + * Method: nativeDestroyEnclave + * Signature: (J)I + */ +JNIEXPORT jint JNICALL JavaEnclave_TeeSDKSVMNativeDestroyEnclave(JNIEnv *, jobject, jlong); + +#ifdef __cplusplus +} +#endif +#endif \ No newline at end of file diff --git a/sdk/host/src/main/native/include/enc_environment.h b/sdk/host/src/main/native/include/enc_environment.h deleted file mode 100644 index 6de8c9f..0000000 --- a/sdk/host/src/main/native/include/enc_environment.h +++ /dev/null @@ -1,29 +0,0 @@ -#ifndef __ENC_ENVIRONMENT_H -#define __ENC_ENVIRONMENT_H - -#if defined(__cplusplus) -extern "C" { -#endif - -typedef struct enc_data_struct{ - //char array is used as byte array to store serialized data - char* data; - int data_len; -}enc_data_t; - -typedef struct callback_functions_struct{ - /* - * This method is invoked inside java_enclave_invoke method's exception catch - * section, when the execution is aborted by exceptions. The caller side can - * decide what to do with the exception. - * Exception details are passed back with parameters. - */ - void (*exception_handler)(char* err_msg, char* stack_trace, char* exception_name); - - char* (*memcpy_char_pointer)(char* src, int len); -}callbacks_t; - -#if defined(__cplusplus) -} -#endif -#endif diff --git a/sdk/host/src/main/native/bin/platform/mock_in_svm/jni/.gitkeep b/sdk/native/bin/platform/mock_in_svm/jni/.gitkeep similarity index 100% copy from sdk/host/src/main/native/bin/platform/mock_in_svm/jni/.gitkeep copy to sdk/native/bin/platform/mock_in_svm/jni/.gitkeep diff --git a/sdk/host/src/main/native/bin/platform/mock_in_svm/jni/.gitkeep b/sdk/native/bin/platform/tee_sdk_svm/jni/.gitkeep similarity index 100% rename from sdk/host/src/main/native/bin/platform/mock_in_svm/jni/.gitkeep rename to sdk/native/bin/platform/tee_sdk_svm/jni/.gitkeep diff --git a/sdk/native/config/config.mk b/sdk/native/config/config.mk new file mode 100644 index 0000000..8107649 --- /dev/null +++ b/sdk/native/config/config.mk @@ -0,0 +1,6 @@ +# parse BIN path. +BIN = $(NATIVE_BASE_DIR)/bin +# parse CONFIG path. +CONFIG = $(NATIVE_BASE_DIR)/config +# parse INCLUDE path. +INCLUDE = $(NATIVE_BASE_DIR)/include \ No newline at end of file diff --git a/sdk/host/src/main/native/config/platform/mock_in_svm/jni/config.mk b/sdk/native/config/platform/mock_in_svm/jni/config.mk similarity index 100% rename from sdk/host/src/main/native/config/platform/mock_in_svm/jni/config.mk rename to sdk/native/config/platform/mock_in_svm/jni/config.mk diff --git a/sdk/native/config/platform/tee_sdk_svm/edl/tee_sdk_enclave.edl b/sdk/native/config/platform/tee_sdk_svm/edl/tee_sdk_enclave.edl new file mode 100644 index 0000000..b89d8db --- /dev/null +++ b/sdk/native/config/platform/tee_sdk_svm/edl/tee_sdk_enclave.edl @@ -0,0 +1,27 @@ +enclave { + include "sgx_report.h" + + from "sgx_tstdc.edl" import *; + from "sgx_pthread.edl" import *; + from "openenclave/edl/syscall.edl" import *; + + trusted { + // create a graal isolate; + public int enclave_svm_isolate_create([out, size=8] void* isolate, [out, size=8] void* isolateThread); + // load enclave services instance; + public int load_enclave_svm_services(uint64_t isolate, [in, size=len0] void* input, size_t len0, [out, size=8] void* output, [out, size=8] size_t* len1); + // invoke enclave instance's services; + public int invoke_enclave_svm_service(uint64_t isolate, [in, size=len0] void* input, size_t len0, [out, size=8] void* output, [out, size=8] size_t* len1); + // unload enclave services instance; + public int unload_enclave_svm_service(uint64_t isolate, [in, size=len0] void* input, size_t len0, [out, size=8] void* output, [out, size=8] size_t* len1); + // destroy a graal isolate; + public int enclave_svm_isolate_destroy(uint64_t isolateThread); + }; + + untrusted { + // rlimit syscall. + int ocall_getrlimit(int resource, [out, size=16]void *rlim); + // malloc memory buffer from host side to store an returned object's serialization result. + int ocall_malloc(size_t size, [out, size=8]void *ptr); + }; +}; diff --git a/sdk/native/config/platform/tee_sdk_svm/edl/tee_sdk_enclave.lds b/sdk/native/config/platform/tee_sdk_svm/edl/tee_sdk_enclave.lds new file mode 100644 index 0000000..aa7892d --- /dev/null +++ b/sdk/native/config/platform/tee_sdk_svm/edl/tee_sdk_enclave.lds @@ -0,0 +1,11 @@ +enclave.so +{ + global: + g_global_data_sim; + g_global_data; + enclave_entry; + g_peak_heap_used; + g_peak_rsrv_mem_committed; + local: + *; +}; \ No newline at end of file diff --git a/sdk/native/config/platform/tee_sdk_svm/jni/config.mk b/sdk/native/config/platform/tee_sdk_svm/jni/config.mk new file mode 100644 index 0000000..d402596 --- /dev/null +++ b/sdk/native/config/platform/tee_sdk_svm/jni/config.mk @@ -0,0 +1,61 @@ +CC = gcc +CXX = g++ + +TEE_SDK_PATH = /opt/teesdk/sgxsdk +SGX_MODE ?= HW +SGX_ARCH ?= x64 +SGX_DEBUG ?= 1 +SGX_PCL ?= 1 + +SGX_COMMON_FLAGS := -m64 +SGX_LIBRARY_PATH := $(TEE_SDK_PATH)/lib64 +SGX_ENCLAVE_SIGNER := $(TEE_SDK_PATH)/bin/x64/sgx_sign +SGX_EDGER8R := $(TEE_SDK_PATH)/bin/x64/sgx_edger8r + +ifeq ($(SGX_DEBUG), 1) +ifeq ($(SGX_PRERELEASE), 1) + $(error Cannot set SGX_DEBUG and SGX_PRERELEASE at the same time!!) +endif +endif + +ifeq ($(SGX_DEBUG), 1) + SGX_COMMON_FLAGS += -O0 -g + Encryption_Tool_Flags := -d +else + SGX_COMMON_FLAGS += -O2 +endif + +ifneq ($(SGX_MODE), HW) + Trts_Library_Name := sgx_trts_sim + Urts_Library_Name := sgx_urts_sim + Service_Library_Name := sgx_tservice_sim +else + Trts_Library_Name := sgx_trts + Urts_Library_Name := sgx_urts + Service_Library_Name := sgx_tservice +endif + +SGX_COMMON_FLAGS += -Wall -Wextra -Winit-self -Wpointer-arith -Wreturn-type \ + -Waddress -Wsequence-point -Wformat-security \ + -Wmissing-include-dirs -Wfloat-equal -Wundef -Wshadow \ + -Wcast-align -Wcast-qual -Wconversion -Wredundant-decls + +SGX_COMMON_CFLAGS := $(SGX_COMMON_FLAGS) -Wjump-misses-init -Wstrict-prototypes -Wunsuffixed-float-constants -std=c99 +SGX_COMMON_CXXFLAGS := $(SGX_COMMON_FLAGS) -Wnon-virtual-dtor -std=c++11 + +TS_HOST_INCDIR = -I$(TEE_SDK_PATH)/include +TS_HOST_CFLAGS = $(TS_HOST_INCDIR) $(SGX_COMMON_CFLAGS) +TS_HOST_CXXFLAGS = $(SGX_COMMON_CXXFLAGS) +TS_HOST_LDFLAGS = -L$(SGX_LIBRARY_PATH) -Wl,-z,noexecstack -lc -l$(Urts_Library_Name) -lpthread -lsgx_usyscall -lsgx_urts + +Enclave_Security_Link_Flags = -Wl,-z,relro,-z,now,-z,noexecstack + +TS_ENCLAVE_INCDIR = -I$(TEE_SDK_PATH)/include -I$(TEE_SDK_PATH)/include/tlibc -I$(TEE_SDK_PATH)/include/libcxx -I$(TEE_SDK_PATH)/include/syscall +TS_ENCLAVE_CFLAGS = $(TS_ENCLAVE_INCDIR) -nostdinc -fvisibility=hidden -fpie -ffunction-sections -fdata-sections -fstack-protector-strong +TS_ENCLAVE_CXXFLAGS = $(TS_ENCLAVE_CFLAGS) -nostdinc++ +TS_ENCLAVE_LDFLAGS = -L$(SGX_LIBRARY_PATH) $(TS_ENCLAVE_CFLAGS) -Wl,--no-undefined -nostdlib -nodefaultlibs -nostartfiles $(Enclave_Security_Link_Flags) \ + -Wl,--whole-archive -l$(Trts_Library_Name) -Wl,--no-whole-archive \ + -Wl,--start-group -lsgx_tsyscall -lsgx_tstdc -lsgx_tcxx -lsgx_pthread -lsgx_tcrypto -l$(Service_Library_Name) -Wl,--end-group \ + -Wl,-Bstatic -Wl,-Bsymbolic -Wl,--no-undefined \ + -Wl,-pie,-eenclave_entry -Wl,--export-dynamic \ + -Wl,--defsym,__ImageBase=0 diff --git a/sdk/enclave/src/main/resources/native/enc_environment.h b/sdk/native/include/enc_environment.h similarity index 51% copy from sdk/enclave/src/main/resources/native/enc_environment.h copy to sdk/native/include/enc_environment.h index f7fae35..884c098 100644 --- a/sdk/enclave/src/main/resources/native/enc_environment.h +++ b/sdk/native/include/enc_environment.h @@ -1,25 +1,25 @@ - typedef struct enc_data_struct{ +typedef struct enc_data_struct{ //char array is used as byte array to store serialized data char* data; int data_len; - }enc_data_t; +} enc_data_t; typedef struct callback_functions_struct{ - /* - * This method is invoked inside java_enclave_invoke method's exception catch - * section, when the execution is aborted by exceptions. The caller side can - * decide what to do with the exception. - * Exception details are passed back with parameters. + /* + * This method is invoked inside java_enclave_invoke method's exception catch + * section, when the execution is aborted by exceptions. The caller side can + * decide what to do with the exception. + * Exception details are passed back with parameters. */ void (*exception_handler)(char* err_msg, char* stack_trace, char* exception_name); char* (*memcpy_char_pointer)(char* src, int len); /* - * Points to an available pseudorandom number generating function. - */ + * Points to an available pseudorandom number generating function. + */ int (*get_random_number)(void* data, long size); -}callbacks_t; +} callbacks_t; long physical_page_size(); long physical_page_number(); diff --git a/sdk/host/src/main/native/include/enc_exported_symbol.h b/sdk/native/include/enc_exported_symbol.h similarity index 100% rename from sdk/host/src/main/native/include/enc_exported_symbol.h rename to sdk/native/include/enc_exported_symbol.h diff --git a/sdk/native/script/build_app/Makefile b/sdk/native/script/build_app/Makefile new file mode 100644 index 0000000..4130e8b --- /dev/null +++ b/sdk/native/script/build_app/Makefile @@ -0,0 +1,36 @@ +include /opt/javaenclave/config/config.mk +include /opt/javaenclave/config/platform/tee_sdk_svm/jni/config.mk + +.PHONY: all build sign keys clean + +all: + $(MAKE) build + $(MAKE) sign + +build: +ifeq ($(TEE_SDK), TRUE) + # remove lib_tee_sdk_svm_load.so from target/svm-output, because it's + # compiled target to common execution environment. + rm -rf ${ENCLAVE_BASE_DIR}/target/svm-output/lib_tee_sdk_svm_load.so + + # copy part of dependency underlying libs to enclave_workspace + cp ${JAVA_HOME}/languages/nfi/builder/clibraries-libffi/libffi.a ${ENCLAVE_BASE_DIR}/target/enclave_workspace + cp ${JAVA_HOME}/lib/svm/builder/clibraries/linux-amd64/libjvm.a ${ENCLAVE_BASE_DIR}/target/enclave_workspace + cp ${JAVA_HOME}/lib/svm/builder/clibraries/linux-amd64/liblibchelper.a ${ENCLAVE_BASE_DIR}/target/enclave_workspace + + $(CC) -o ${ENCLAVE_BASE_DIR}/target/svm-output/lib_tee_sdk_svm_load \ + /opt/javaenclave/bin/platform/tee_sdk_svm/*.o ${ENCLAVE_BASE_DIR}/target/svm-output/lib_tee_sdk_svm_load.o \ + -L${JAVA_HOME}/lib/static/linux-amd64/glibc \ + -L${ENCLAVE_BASE_DIR}/target/enclave_workspace \ + -fpie -ljava -lzip -lnio -lnet -ljvm -lfdlibm -llibchelper \ + $(TS_ENCLAVE_LDFLAGS) -Wl,--version-script=/opt/javaenclave/config/platform/tee_sdk_svm/edl/tee_sdk_enclave.lds +endif + +sign: +ifeq ($(TEE_SDK), TRUE) + openssl genrsa -out private.pem -3 3072 + openssl rsa -in private.pem -pubout -out public.pem + $(SGX_ENCLAVE_SIGNER) sign -enclave ${ENCLAVE_BASE_DIR}/target/svm-output/lib_tee_sdk_svm_load \ + -out ${ENCLAVE_BASE_DIR}/target/svm-output/lib_tee_sdk_svm_load.signed \ + -config ${ENCLAVE_BASE_DIR}/src/main/resources/tee_sdk_svm.conf -key private.pem +endif diff --git a/sdk/native/script/build_app/make.sh b/sdk/native/script/build_app/make.sh new file mode 100644 index 0000000..3c522dd --- /dev/null +++ b/sdk/native/script/build_app/make.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +# shellcheck disable=SC2006 +this_script_dir=`dirname "$0"` + +export ENCLAVE_BASE_DIR="$1" +enclave_platform_config=$2 + +# Create a native image building workspace in application's enclave submodule. +mkdir -p "${ENCLAVE_BASE_DIR}"/target/enclave_workspace +# copy Makefile script to enclave_workspace. +cp -r "${this_script_dir}"/Makefile "${ENCLAVE_BASE_DIR}"/target/enclave_workspace + +# cd to enclave workspace. +cd "${ENCLAVE_BASE_DIR}"/target/enclave_workspace + +# process supported enclave platform set +OLD_IFS="$IFS" +IFS=":" +enclave_platform_array=($enclave_platform_config) +IFS="$OLD_IFS" + +# Setting TEE Platform for makefile build process. +# shellcheck disable=SC2068 +for enclave_platform in ${enclave_platform_array[@]} +do + echo "$enclave_platform" + # set "enclave_platform" as TRUE to indicate how + # to compile jni.so and edge routine + export "$enclave_platform"=TRUE +done + +make -f ./Makefile \ No newline at end of file diff --git a/sdk/native/script/build_enclave_sdk/Makefile b/sdk/native/script/build_enclave_sdk/Makefile new file mode 100644 index 0000000..0154540 --- /dev/null +++ b/sdk/native/script/build_enclave_sdk/Makefile @@ -0,0 +1,17 @@ +# Copyright (c) + +.PHONY: all build clean + +all: build + +build: +ifeq ($(TEE_SDK), TRUE) + $(MAKE) -C $(ENCLAVE_BASE_DIR)/src/main/native/cpp/platform/tee_sdk_svm/wrapper + $(MAKE) -C $(ENCLAVE_BASE_DIR)/src/main/native/cpp/platform/tee_sdk_svm/edge_routines +endif + +clean: +ifeq ($(TEE_SDK), TRUE) + $(MAKE) -C $(ENCLAVE_BASE_DIR)/src/main/native/cpp/platform/tee_sdk_svm/wrapper clean + $(MAKE) -C $(ENCLAVE_BASE_DIR)/src/main/native/cpp/platform/tee_sdk_svm/edge_routines clean +endif \ No newline at end of file diff --git a/sdk/host/src/main/native/make.sh b/sdk/native/script/build_enclave_sdk/make.sh similarity index 79% copy from sdk/host/src/main/native/make.sh copy to sdk/native/script/build_enclave_sdk/make.sh index 0340bb6..8746cc9 100644 --- a/sdk/host/src/main/native/make.sh +++ b/sdk/native/script/build_enclave_sdk/make.sh @@ -12,8 +12,10 @@ this_script_dir=`dirname "$0"` cd "$this_script_dir" # step two: parse parameters from pom.xml -# parse and store base dir path -export base_dir=$1 +# parse and store host base dir path +export ENCLAVE_BASE_DIR=$1 +export NATIVE_BASE_DIR="$ENCLAVE_BASE_DIR"/../native + # parse and store supported enclave platform set enclave_platform_config=$2 # process supported enclave platform set @@ -40,10 +42,6 @@ elif [ "$3" == build ]; then # make build. make -f ./Makefile build - # copy jni.so to target/classes, which will be packed into a jar file. - if [[ $MOCK_IN_SVM == TRUE ]]; then - cp -r "$base_dir"/src/main/native/bin/platform/mock_in_svm/jni "$base_dir"/target/classes - fi else echo "unsupported make command!!!" fi diff --git a/sdk/native/script/build_host_sdk/Makefile b/sdk/native/script/build_host_sdk/Makefile new file mode 100644 index 0000000..9d65c19 --- /dev/null +++ b/sdk/native/script/build_host_sdk/Makefile @@ -0,0 +1,23 @@ +# Copyright (c) + +.PHONY: all build clean + +all: build + +build: +ifeq ($(MOCK_IN_SVM), TRUE) + $(MAKE) -C $(HOST_BASE_DIR)/src/main/native/cpp/platform/mock_in_svm/jni +endif + +ifeq ($(TEE_SDK), TRUE) + $(MAKE) -C $(HOST_BASE_DIR)/src/main/native/cpp/platform/tee_sdk_svm +endif + +clean: +ifeq ($(MOCK_IN_SVM), TRUE) + $(MAKE) -C $(HOST_BASE_DIR)/src/main/native/cpp/platform/mock_in_svm/jni clean +endif + +ifeq ($(TEE_SDK), TRUE) + $(MAKE) -C $(HOST_BASE_DIR)/src/main/native/cpp/platform/tee_sdk_svm clean +endif \ No newline at end of file diff --git a/sdk/host/src/main/native/make.sh b/sdk/native/script/build_host_sdk/make.sh similarity index 72% rename from sdk/host/src/main/native/make.sh rename to sdk/native/script/build_host_sdk/make.sh index 0340bb6..c543036 100644 --- a/sdk/host/src/main/native/make.sh +++ b/sdk/native/script/build_host_sdk/make.sh @@ -12,8 +12,10 @@ this_script_dir=`dirname "$0"` cd "$this_script_dir" # step two: parse parameters from pom.xml -# parse and store base dir path -export base_dir=$1 +# parse and store host base dir path +export HOST_BASE_DIR=$1 +export NATIVE_BASE_DIR="$HOST_BASE_DIR"/../native + # parse and store supported enclave platform set enclave_platform_config=$2 # process supported enclave platform set @@ -35,14 +37,17 @@ if [ "$3" == clean ]; then # make clean. make -f ./Makefile clean - rm elif [ "$3" == build ]; then # make build. make -f ./Makefile build - # copy jni.so to target/classes, which will be packed into a jar file. + # copy MOCK_IN_SVM jni.so to target/classes, which will be packed into a jar file. if [[ $MOCK_IN_SVM == TRUE ]]; then - cp -r "$base_dir"/src/main/native/bin/platform/mock_in_svm/jni "$base_dir"/target/classes + cp -r "$NATIVE_BASE_DIR"/bin/platform/mock_in_svm/jni "$HOST_BASE_DIR"/target/classes + fi + # copy jni.so to target/classes, which will be packed into a jar file. + if [[ $TEE_SDK == TRUE ]]; then + cp -r "$NATIVE_BASE_DIR"/bin/platform/tee_sdk_svm/jni "$HOST_BASE_DIR"/target/classes fi else echo "unsupported make command!!!" diff --git a/sdk/pom.xml b/sdk/pom.xml index c5401b1..653eadc 100644 --- a/sdk/pom.xml +++ b/sdk/pom.xml @@ -11,7 +11,7 @@ <maven.compiler.source>11</maven.compiler.source> <maven.compiler.target>11</maven.compiler.target> <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> - <com.alibaba.enclave.platform>MOCK_IN_SVM</com.alibaba.enclave.platform> + <com.alibaba.enclave.platform>MOCK_IN_SVM:TEE_SDK</com.alibaba.enclave.platform> </properties> <dependencyManagement> <dependencies> diff --git a/test/common/src/main/java/com/alibaba/confidentialcomputing/test/common/ReflectionCallService.java b/test/common/src/main/java/com/alibaba/confidentialcomputing/test/common/ReflectionCallService.java index f4dbf44..d466dea 100644 --- a/test/common/src/main/java/com/alibaba/confidentialcomputing/test/common/ReflectionCallService.java +++ b/test/common/src/main/java/com/alibaba/confidentialcomputing/test/common/ReflectionCallService.java @@ -5,6 +5,5 @@ import com.alibaba.confidentialcomputing.common.annotations.EnclaveService; @EnclaveService public interface ReflectionCallService { int add(int a, int b); - int sub(int a, int b); } diff --git a/test/enclave/pom.xml b/test/enclave/pom.xml index 0cee904..2d99e31 100644 --- a/test/enclave/pom.xml +++ b/test/enclave/pom.xml @@ -57,6 +57,24 @@ </configuration> <phase>package</phase> </execution> + <execution> + <id>build-native-tee</id> + <goals> + <goal>build</goal> + </goals> + <configuration> + <imageName>lib_tee_sdk_svm_load</imageName> + <buildArgs> + <buildArg>--shared</buildArg> + <buildArg>--no-fallback</buildArg> + <buildArg>--allow-incomplete-classpath</buildArg> + <buildArg>-H:Path=svm-output</buildArg> + <buildArg>-H:DisableFeatures=com.oracle.svm.core.posix.NativeSecureRandomFilesCloser,com.oracle.svm.core.posix.linux.LinuxPhysicalMemory$PhysicalMemoryFeature</buildArg> + <buildArg>-H:ReflectionConfigurationFiles=${project.basedir}/target/native/agent-output/test/reflect-config.json</buildArg> + </buildArgs> + </configuration> + <phase>package</phase> + </execution> </executions> <configuration> <agent> @@ -64,6 +82,27 @@ </agent> </configuration> </plugin> + <plugin> + <artifactId>exec-maven-plugin</artifactId> + <groupId>org.codehaus.mojo</groupId> + <executions> + <execution> + <id>ApplicationNativeCompile</id> + <phase>package</phase> + <goals> + <goal>exec</goal> + </goals> + <configuration> + <executable>bash</executable> + <arguments> + <argument>/opt/javaenclave/build_app/make.sh</argument> + <argument>${project.basedir}</argument> + <argument>${com.alibaba.enclave.platform}</argument> + </arguments> + </configuration> + </execution> + </executions> + </plugin> </plugins> </build> </profile> diff --git a/test/enclave/src/main/resources/tee_sdk_svm.conf b/test/enclave/src/main/resources/tee_sdk_svm.conf new file mode 100644 index 0000000..59ff1d8 --- /dev/null +++ b/test/enclave/src/main/resources/tee_sdk_svm.conf @@ -0,0 +1,12 @@ +<!-- Please refer to User's Guide for the explanation of each field --> +<EnclaveConfiguration> + <ProdID>0</ProdID> + <ISVSVN>0</ISVSVN> + <StackMaxSize>0x101000</StackMaxSize> + <HeapMaxSize>0x1000000</HeapMaxSize> + <TCSNum>10</TCSNum> + <TCSPolicy>1</TCSPolicy> + <DisableDebug>0</DisableDebug> + <MiscSelect>0</MiscSelect> + <MiscMask>0xFFFFFFFF</MiscMask> +</EnclaveConfiguration> \ No newline at end of file diff --git a/test/host/pom.xml b/test/host/pom.xml index 6565a58..ceacc29 100644 --- a/test/host/pom.xml +++ b/test/host/pom.xml @@ -18,6 +18,7 @@ <directory>${project.basedir}/../enclave/target/svm-output</directory> <includes> <include>**/*.so</include> + <include>**/*.signed</include> </includes> </resource> </resources> diff --git a/test/host/src/test/java/com/alibaba/confidentialcomputing/test/host/TestJavaEnclaveService.java b/test/host/src/test/java/com/alibaba/confidentialcomputing/test/host/TestJavaEnclaveService.java index 3640a12..4304ac9 100644 --- a/test/host/src/test/java/com/alibaba/confidentialcomputing/test/host/TestJavaEnclaveService.java +++ b/test/host/src/test/java/com/alibaba/confidentialcomputing/test/host/TestJavaEnclaveService.java @@ -56,17 +56,20 @@ public class TestJavaEnclaveService { EnclaveCreatingException, EnclaveDestroyingException, ServicesLoadingException { assertEquals("Hello World", sayHelloService(EnclaveType.MOCK_IN_JVM, "Hello World")); assertEquals("Hello World", sayHelloService(EnclaveType.MOCK_IN_SVM, "Hello World")); + assertEquals("Hello World", sayHelloService(EnclaveType.TEE_SDK, "Hello World")); } @Test public void testReflectionCallService() throws ServicesLoadingException, EnclaveCreatingException, EnclaveDestroyingException { reflectionCallService(EnclaveType.MOCK_IN_JVM); reflectionCallService(EnclaveType.MOCK_IN_SVM); + reflectionCallService(EnclaveType.TEE_SDK); } @Test public void testJavaEnclaveException() throws ServicesLoadingException, EnclaveCreatingException, EnclaveDestroyingException { javaEnclaveException(EnclaveType.MOCK_IN_JVM); javaEnclaveException(EnclaveType.MOCK_IN_SVM); + javaEnclaveException(EnclaveType.TEE_SDK); } } diff --git a/test/pom.xml b/test/pom.xml index 8e8df10..7c2f566 100644 --- a/test/pom.xml +++ b/test/pom.xml @@ -12,7 +12,7 @@ <maven.compiler.source>11</maven.compiler.source> <maven.compiler.target>11</maven.compiler.target> <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> - <com.alibaba.enclave.platform>MOCK_IN_SVM</com.alibaba.enclave.platform> + <com.alibaba.enclave.platform>MOCK_IN_SVM:TEE_SDK</com.alibaba.enclave.platform> </properties> <dependencyManagement> <dependencies> diff --git a/tools/cicd/Dockerfile b/tools/cicd/Dockerfile index b430e9b..1dd55bd 100644 --- a/tools/cicd/Dockerfile +++ b/tools/cicd/Dockerfile @@ -9,13 +9,19 @@ ADD ["graalvm-enclave-22.0.0.tar", "/root/tools/"] ADD ["x86_64-linux-musl-native.tgz", "/root/tools/"] ADD ["zlib-1.2.12.tar.gz", "/root/tools/"] ADD ["settings.xml", "/root/tools/"] +ADD ["sgx_linux_x64_sdk_2.15.100.0.bin", "/root/tools/"] ENV GRAALVM_HOME "/root/tools/graalvm-enclave-22.0.0" ENV JAVA_HOME "/root/tools/graalvm-enclave-22.0.0" ENV CC "/root/tools/x86_64-linux-musl-native/bin/gcc" ENV PATH $PATH:"/root/tools/x86_64-linux-musl-native/bin" + # install necessary tools. -RUN apt-get update && \ +RUN apt-get update && apt-get install -y gnupg wget && \ + echo 'deb [arch=amd64] https://download.01.org/intel-sgx/sgx_repo/ubuntu bionic main' > /etc/apt/sources.list.d/intel-sgx.list && \ + wget -qO - https://download.01.org/intel-sgx/sgx_repo/ubuntu/intel-sgx-deb.key | apt-key add - && \ + apt-get update && apt-get install -y --no-install-recommends libsgx-launch libsgx-urts && \ echo -e 'yes\n' | apt-get install -y maven && \ echo -e 'yes\n' | apt-get install -y build-essential libz-dev zlib1g-dev && \ - cd /root/tools/zlib-1.2.12 && ./configure --prefix=/root/tools/x86_64-linux-musl-native --static && make && make install + cd /root/tools/zlib-1.2.12 && ./configure --prefix=/root/tools/x86_64-linux-musl-native --static && make && make install && \ + cd /root/tools && chmod 777 sgx_linux_x64_sdk_2.15.100.0.bin && echo -e 'no\n/opt/teesdk\n' | ./sgx_linux_x64_sdk_2.15.100.0.bin diff --git a/tools/cicd/make.sh b/tools/cicd/make.sh index 41535ef..fd768b7 100755 --- a/tools/cicd/make.sh +++ b/tools/cicd/make.sh @@ -17,15 +17,19 @@ if [[ "$(docker images -q ${BUILD_IMAGE}:${BUILD_TAG} 2> /dev/null)" == "" ]]; t wget http://graal.oss-cn-beijing.aliyuncs.com/graal-enclave/x86_64-linux-musl-native.tgz wget http://graal.oss-cn-beijing.aliyuncs.com/graal-enclave/zlib-1.2.12.tar.gz wget http://graal.oss-cn-beijing.aliyuncs.com/graal-enclave/settings_taobao.xml -O settings.xml + wget https://dragonwell.oss-cn-shanghai.aliyuncs.com/11/tee_java/dependency/sgx_linux_x64_sdk_2.15.100.0.bin docker build -t ${BUILD_IMAGE}:${BUILD_TAG} . rm -f graalvm-enclave-22.0.0.tar rm -f x86_64-linux-musl-native.tgz rm -f zlib-1.2.12.tar.gz + rm -f sgx_linux_x64_sdk_2.15.100.0.bin fi # test JavaEnclave's unit test cases and samples docker run -i --rm --privileged --network host \ -w "${WORKDIR}" \ -v "${HOME}"/.m2:/root/.m2 -v "${WORKDIR}":"${WORKDIR}" \ +-v /dev/sgx_enclave:/dev/sgx/enclave \ +-v /dev/sgx_provision:/dev/sgx/provision \ ${BUILD_IMAGE}:${BUILD_TAG} /bin/bash build.sh --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
