diff --git a/drivers/tee/optee/call.c b/drivers/tee/optee/call.c index 24b5153db47ea6..499ebbed3387c6 100644 --- a/drivers/tee/optee/call.c +++ b/drivers/tee/optee/call.c @@ -28,28 +28,393 @@ static struct optee_session *find_session(struct optee_context_data *ctxdata, return NULL; } +static void param_clear_ocall(struct tee_param *ocall) +{ + if (ocall) + memset(&ocall->u, 0, sizeof(ocall->u)); +} + +static u64 param_get_ocall_func(struct tee_param *param) +{ + return TEE_IOCTL_OCALL_GET_FUNC(param->u.value.a); +} + +/* Requires @sem in the parent struct optee_session to be held */ +static int verify_ocall_request(u32 num_params, struct optee_call_ctx *call_ctx) +{ + size_t n; + size_t sz; + struct optee_msg_arg *arg = call_ctx->rpc_arg; + + switch (arg->cmd) { + case OPTEE_MSG_RPC_CMD_SHM_ALLOC: + case OPTEE_MSG_RPC_CMD_SHM_FREE: + if (!num_params) + return -EINVAL; + + /* + * This parameter either carries the requested allocation size + * or a pointer to the SHM to be freed. + */ + if (!arg->num_params || + arg->params[0].attr != OPTEE_MSG_ATTR_TYPE_VALUE_INPUT) + return -EINVAL; + + /* + * Ensure that we won't read past the end of the SHM underlying + * arg->params. + */ + sz = sizeof(*arg) + sizeof(*arg->params) * arg->num_params; + if (sz > call_ctx->rpc_shm->size) + return -EINVAL; + + /* The remaining parameters are unused */ + for (n = 1; n < arg->num_params; n++) + if (arg->params[n].attr != OPTEE_MSG_ATTR_TYPE_NONE) + return -EINVAL; + break; + case OPTEE_MSG_RPC_CMD_OCALL: + /* 'num_params' is checked later */ + + /* These parameters carry the OCALL descriptors */ + if (arg->num_params < 2 || + arg->params[0].attr != OPTEE_MSG_ATTR_TYPE_VALUE_INOUT || + arg->params[1].attr != OPTEE_MSG_ATTR_TYPE_VALUE_INPUT || + arg->params[0].u.value.a > U32_MAX || /* OCALL Cmd Id */ + arg->params[1].u.value.c != 0) /* TA UUID (128 bytes) */ + return -EINVAL; + break; + default: + return -EINVAL; + } + + return 0; +} + +/* Requires @sem in the parent struct optee_session to be held */ +static int verify_ocall_reply(u64 func, struct tee_param *params, + u32 num_params, struct optee_call_ctx *call_ctx) +{ + size_t n; + + switch (func) { + case TEE_IOCTL_OCALL_CMD_SHM_ALLOC: + if (call_ctx->rpc_arg->cmd != OPTEE_MSG_RPC_CMD_SHM_ALLOC || + !num_params) + return -EINVAL; + + /* This parameter carries the allocated SHM ID */ + if (params[0].attr != TEE_IOCTL_PARAM_ATTR_TYPE_VALUE_INOUT || + params[0].u.value.c > INT_MAX) + return -EINVAL; + break; + case TEE_IOCTL_OCALL_CMD_SHM_FREE: + if (call_ctx->rpc_arg->cmd != OPTEE_MSG_RPC_CMD_SHM_FREE || + !num_params) + return -EINVAL; + + /* Sanity check, not used while processing the reply */ + if (params[0].attr != TEE_IOCTL_PARAM_ATTR_TYPE_VALUE_INPUT) + return -EINVAL; + break; + case TEE_IOCTL_OCALL_CMD_INVOKE: + if (call_ctx->rpc_arg->cmd != OPTEE_MSG_RPC_CMD_OCALL) + return -EINVAL; + + /* Skip the loop below */ + return 0; + default: + return -EINVAL; + } + + /* The remaining parameters are unused */ + for (n = 1; n < num_params; n++) + if (params[n].attr != TEE_IOCTL_PARAM_ATTR_TYPE_NONE) + return -EINVAL; + + return 0; +} + +/* Requires @sem in the parent struct optee_session to be held */ +static void process_ocall_memrefs(struct optee_msg_param *params, + u32 num_params, bool increment) +{ + size_t n; + + for (n = 0; n < num_params; n++) { + struct tee_shm *shm; + const struct optee_msg_param *mp = params + n; + u32 attr = mp->attr & OPTEE_MSG_ATTR_TYPE_MASK; + + switch (attr) { + case OPTEE_MSG_ATTR_TYPE_RMEM_INPUT: + case OPTEE_MSG_ATTR_TYPE_RMEM_OUTPUT: + case OPTEE_MSG_ATTR_TYPE_RMEM_INOUT: + shm = (struct tee_shm *)(uintptr_t)mp->u.rmem.shm_ref; + break; + default: + shm = NULL; + break; + } + + if (!shm) + continue; + + if (increment) + tee_shm_get(shm); + else + tee_shm_put(shm); + } +} + +/* + * Requires @sem in the parent struct optee_session to be held (if OCALLs are + * expected) + */ +static void call_prologue(struct optee_call_ctx *call_ctx) +{ + struct optee *optee = tee_get_drvdata(call_ctx->ctx->teedev); + + /* Initialize waiter */ + optee_cq_wait_init(&optee->call_queue, &call_ctx->waiter); +} + +/* + * Requires @sem in the parent struct optee_session to be held (if OCALLs are + * expected) + */ +static void call_epilogue(struct optee_call_ctx *call_ctx) +{ + struct optee *optee = tee_get_drvdata(call_ctx->ctx->teedev); + + optee_rpc_finalize_call(call_ctx); + + /* + * We're done with our thread in secure world, if there's any + * thread waiters wake up one. + */ + optee_cq_wait_final(&optee->call_queue, &call_ctx->waiter); +} + +/* Requires @sem in the parent struct optee_session to be held */ +static int process_ocall_request(struct tee_param *params, u32 num_params, + struct tee_param *ocall, + struct optee_call_ctx *call_ctx) +{ + u32 cmd_id; + struct tee_shm *shm; + size_t shm_sz; + struct optee_msg_param *msg_param; + u32 msg_num_params; + int rc = 0; + + /* + * Points to the octets of the UUID corresponding to the TA requesting + * the OCALL, if applicable for this call. + */ + void *clnt_id; + + rc = verify_ocall_request(num_params, call_ctx); + if (rc) + goto exit_set_ret; + + /* + * Clear out the parameters of the original function invocation. The + * original contents are backed up in call_ctx->msg_arg and will be + * restored elsewhere once the OCALL is over. + */ + memset(params, 0, num_params * sizeof(*params)); + + /* Set up the OCALL request */ + switch (call_ctx->rpc_arg->cmd) { + case OPTEE_MSG_RPC_CMD_SHM_ALLOC: + ocall->u.value.a = + TEE_IOCTL_OCALL_MAKE_PAIR(TEE_IOCTL_OCALL_CMD_SHM_ALLOC, + 0); + + shm_sz = call_ctx->rpc_arg->params[0].u.value.b; + params[0].attr = TEE_IOCTL_PARAM_ATTR_TYPE_VALUE_INOUT; + params[0].u.value.a = 0; + params[0].u.value.b = shm_sz; + params[0].u.value.c = 0; + break; + case OPTEE_MSG_RPC_CMD_SHM_FREE: + ocall->u.value.a = + TEE_IOCTL_OCALL_MAKE_PAIR(TEE_IOCTL_OCALL_CMD_SHM_FREE, + 0); + + shm = (struct tee_shm *)(uintptr_t) + call_ctx->rpc_arg->params[0].u.value.b; + params[0].attr = TEE_IOCTL_PARAM_ATTR_TYPE_VALUE_INPUT; + params[0].u.value.a = 0; + params[0].u.value.b = 0; + params[0].u.value.c = tee_shm_get_id(shm); + break; + case OPTEE_MSG_RPC_CMD_OCALL: + /* -2 here and +2 below to skip the OCALL descriptors */ + msg_num_params = call_ctx->rpc_arg->num_params - 2; + if (num_params < msg_num_params) { + rc = -EINVAL; + goto exit_set_ret; + } + + msg_param = call_ctx->rpc_arg->params + 2; + rc = optee_from_msg_param(params, msg_num_params, msg_param); + if (rc) + goto exit_set_ret; + + process_ocall_memrefs(msg_param, msg_num_params, true); + call_ctx->rpc_must_release = true; + + cmd_id = (u32)call_ctx->rpc_arg->params[0].u.value.a; + ocall->u.value.a = + TEE_IOCTL_OCALL_MAKE_PAIR(TEE_IOCTL_OCALL_CMD_INVOKE, + cmd_id); + + clnt_id = &call_ctx->rpc_arg->params[1].u.value; + memcpy(&ocall->u.value.b, clnt_id, TEE_IOCTL_UUID_LEN); + break; + default: + /* NOT REACHED */ + rc = -EINVAL; + goto exit_set_ret; + } + + return rc; + +exit_set_ret: + call_ctx->rpc_arg->ret = TEEC_ERROR_BAD_PARAMETERS; + call_ctx->rpc_arg->ret_origin = TEEC_ORIGIN_COMMS; + return rc; +} + +/* Requires @sem in the parent struct optee_session to be held */ +static int process_ocall_reply(u32 ret, u32 ret_origin, + struct tee_param *params, u32 num_params, + struct tee_param *ocall, + struct optee_call_ctx *call_ctx) +{ + const u64 func = param_get_ocall_func(ocall); + struct tee_shm *shm; + void *shm_pages_list; + struct optee_msg_param *msg_param; + u32 msg_num_params; + int rc = 0; + + rc = verify_ocall_reply(func, params, num_params, call_ctx); + if (rc) + goto exit_set_ret; + + switch (func) { + case TEE_IOCTL_OCALL_CMD_SHM_ALLOC: + if (ret != TEEC_SUCCESS) + goto exit_propagate_ret; + + if (call_ctx->ocall_pages_list) + goto exit_set_ret; + + shm = tee_shm_get_from_id(call_ctx->ctx, + (int)params[0].u.value.c); + if (IS_ERR(shm)) { + rc = PTR_ERR(shm); + goto exit_set_ret; + } + + rc = optee_rpc_process_shm_alloc(shm, call_ctx->rpc_arg->params, + &shm_pages_list); + + /* The CA holds a reference */ + tee_shm_put(shm); + + if (rc) + goto exit_set_ret; + + /* Could be NULL and zero, respectively */ + call_ctx->ocall_pages_list = shm_pages_list; + call_ctx->ocall_num_entries = shm->num_pages; + break; + case TEE_IOCTL_OCALL_CMD_SHM_FREE: + /* OP-TEE ignores ret and ret_origin for an SHM_FREE RPC */ + if (ret != TEEC_SUCCESS) + goto exit_propagate_ret; + + if (call_ctx->ocall_pages_list) { + optee_free_pages_list(call_ctx->ocall_pages_list, + call_ctx->ocall_num_entries); + call_ctx->ocall_pages_list = NULL; + call_ctx->ocall_num_entries = 0; + } + break; + case TEE_IOCTL_OCALL_CMD_INVOKE: + /* -2 here and +2 below to skip the OCALL descriptors */ + msg_num_params = call_ctx->rpc_arg->num_params - 2; + if (num_params < msg_num_params) { + rc = -EINVAL; + goto exit_set_ret; + } + + msg_param = call_ctx->rpc_arg->params + 2; + rc = optee_to_msg_param(msg_param, msg_num_params, params); + if (rc) + goto exit_set_ret; + + process_ocall_memrefs(msg_param, msg_num_params, false); + call_ctx->rpc_must_release = false; + + call_ctx->rpc_arg->params[0].u.value.b = ret; + call_ctx->rpc_arg->params[0].u.value.c = ret_origin; + break; + default: + rc = -EINVAL; + goto exit_set_ret; + } + + call_ctx->rpc_arg->ret = TEEC_SUCCESS; + call_ctx->rpc_arg->ret_origin = TEEC_ORIGIN_COMMS; + + return rc; + +exit_propagate_ret: + call_ctx->rpc_arg->ret = ret; + call_ctx->rpc_arg->ret_origin = ret_origin; + return -EINVAL; +exit_set_ret: + call_ctx->rpc_arg->ret = TEEC_ERROR_BAD_PARAMETERS; + call_ctx->rpc_arg->ret_origin = TEEC_ORIGIN_COMMS; + return rc; +} + +static void clear_call_ctx(struct optee_call_ctx *call_ctx) +{ + memset(call_ctx, 0, sizeof(*call_ctx)); +} + /** - * optee_do_call_with_arg() - Do an SMC to OP-TEE in secure world + * optee_do_call_with_ctx() - Invoke OP-TEE in secure world * @ctx: calling context - * @parg: physical address of message to pass to secure world * * Does and SMC to OP-TEE in secure world and handles eventual resulting * Remote Procedure Calls (RPC) from OP-TEE. * - * Returns return code from secure world, 0 is OK + * Returns return code from secure world, 0 is OK, -EAGAIN means an OCALL + * request was received. */ -u32 optee_do_call_with_arg(struct tee_context *ctx, phys_addr_t parg) +u32 optee_do_call_with_ctx(struct optee_call_ctx *call_ctx) { - struct optee *optee = tee_get_drvdata(ctx->teedev); - struct optee_call_waiter w; + struct optee *optee = tee_get_drvdata(call_ctx->ctx->teedev); struct optee_rpc_param param = { }; - struct optee_call_ctx call_ctx = { }; u32 ret; - param.a0 = OPTEE_SMC_CALL_WITH_ARG; - reg_pair_from_64(¶m.a1, ¶m.a2, parg); - /* Initialize waiter */ - optee_cq_wait_init(&optee->call_queue, &w); + if (call_ctx->rpc_shm) { + param.a0 = OPTEE_SMC_CALL_RETURN_FROM_RPC; + reg_pair_from_64(¶m.a1, ¶m.a2, + (uintptr_t)call_ctx->rpc_shm); + param.a3 = call_ctx->thread_id; + } else { + param.a0 = OPTEE_SMC_CALL_WITH_ARG; + reg_pair_from_64(¶m.a1, ¶m.a2, call_ctx->msg_parg); + } + while (true) { struct arm_smccc_res res; @@ -63,33 +428,64 @@ u32 optee_do_call_with_arg(struct tee_context *ctx, phys_addr_t parg) if (res.a0 == OPTEE_SMC_RETURN_ETHREAD_LIMIT) { /* - * Out of threads in secure world, wait for a thread + * Out of threads in secure world, wait for a thread to * become available. */ - optee_cq_wait_for_completion(&optee->call_queue, &w); + optee_cq_wait_for_completion(&optee->call_queue, + &call_ctx->waiter); } else if (OPTEE_SMC_RETURN_IS_RPC(res.a0)) { - might_sleep(); param.a0 = res.a0; param.a1 = res.a1; param.a2 = res.a2; param.a3 = res.a3; - optee_handle_rpc(ctx, ¶m, &call_ctx); + + if (optee_rpc_is_ocall(¶m, call_ctx)) + return -EAGAIN; + + might_sleep(); + optee_handle_rpc(call_ctx->ctx, ¶m, call_ctx); } else { ret = res.a0; break; } } - optee_rpc_finalize_call(&call_ctx); - /* - * We're done with our thread in secure world, if there's any - * thread waiters wake up one. - */ - optee_cq_wait_final(&optee->call_queue, &w); - return ret; } +/** + * optee_do_call_with_arg() - Invoke OP-TEE in secure world + * @ctx: calling context + * @parg: physical address of message to pass to secure world + * + * Wraps a call to optee_do_call_with_ctx that sets up the calling context on + * behalf of a caller that does not expect OCALLs. + * + * Returns return code from secure world, 0 is OK + */ +u32 optee_do_call_with_arg(struct tee_context *ctx, phys_addr_t parg) +{ + struct optee_call_ctx call_ctx = { }; + int rc; + + call_ctx.ctx = ctx; + call_ctx.msg_parg = parg; + + call_prologue(&call_ctx); + + rc = optee_do_call_with_ctx(&call_ctx); + if (rc == -EAGAIN) { + pr_warn("received an unexpected OCALL, cancelling it now"); + call_ctx.rpc_arg->ret = TEEC_ERROR_NOT_SUPPORTED; + call_ctx.rpc_arg->ret_origin = TEEC_ORIGIN_COMMS; + optee_do_call_with_ctx(&call_ctx); + } + + call_epilogue(&call_ctx); + + return rc; +} + static struct tee_shm *get_msg_arg(struct tee_context *ctx, size_t num_params, struct optee_msg_arg **msg_arg, phys_addr_t *msg_parg) @@ -125,97 +521,256 @@ static struct tee_shm *get_msg_arg(struct tee_context *ctx, size_t num_params, return shm; } -int optee_open_session(struct tee_context *ctx, - struct tee_ioctl_open_session_arg *arg, - struct tee_param *normal_param, - u32 num_normal_params, - struct tee_param *ocall_param) +/* + * Requires @sem in the parent struct optee_session to be held; the caller is + * expected to have filled in the ret and ret_origin elements of rpc_arg. + */ +static int cancel_ocall(struct optee_call_ctx *call_ctx) { - struct optee_context_data *ctxdata = ctx->data; int rc; + + /* +2 and -2 to skip the OCALL descriptors */ + if (call_ctx->rpc_must_release) { + process_ocall_memrefs(call_ctx->rpc_arg->params + 2, + call_ctx->rpc_arg->num_params - 2, false); + call_ctx->rpc_must_release = false; + } + + if (call_ctx->ocall_pages_list) { + optee_free_pages_list(call_ctx->ocall_pages_list, + call_ctx->ocall_num_entries); + call_ctx->ocall_pages_list = NULL; + call_ctx->ocall_num_entries = 0; + } + + rc = optee_do_call_with_ctx(call_ctx); + if (rc == -EAGAIN) + pr_warn("received an OCALL while cancelling an OCALL"); + + call_epilogue(call_ctx); + + return rc; +} + +static int close_session(struct tee_context *ctx, u32 session) +{ struct tee_shm *shm; struct optee_msg_arg *msg_arg; phys_addr_t msg_parg; - struct optee_session *sess = NULL; - - if (ocall_param) { - pr_err("OCALLs not supported\n"); - return -EOPNOTSUPP; - } - /* +2 for the meta parameters added below */ - shm = get_msg_arg(ctx, arg->num_params + 2, &msg_arg, &msg_parg); + shm = get_msg_arg(ctx, 0, &msg_arg, &msg_parg); if (IS_ERR(shm)) return PTR_ERR(shm); - msg_arg->cmd = OPTEE_MSG_CMD_OPEN_SESSION; - msg_arg->cancel_id = arg->cancel_id; + msg_arg->cmd = OPTEE_MSG_CMD_CLOSE_SESSION; + msg_arg->session = session; + optee_do_call_with_arg(ctx, msg_parg); - /* - * Initialize and add the meta parameters needed when opening a - * session. - */ - msg_arg->params[0].attr = OPTEE_MSG_ATTR_TYPE_VALUE_INPUT | - OPTEE_MSG_ATTR_META; - msg_arg->params[1].attr = OPTEE_MSG_ATTR_TYPE_VALUE_INPUT | - OPTEE_MSG_ATTR_META; - memcpy(&msg_arg->params[0].u.value, arg->uuid, sizeof(arg->uuid)); - msg_arg->params[1].u.value.c = arg->clnt_login; - - rc = tee_session_calc_client_uuid((uuid_t *)&msg_arg->params[1].u.value, - arg->clnt_login, arg->clnt_uuid); - if (rc) - goto out; + tee_shm_free(shm); + return 0; +} - rc = optee_to_msg_param(msg_arg->params + 2, arg->num_params, - normal_param); - if (rc) - goto out; +int optee_open_session(struct tee_context *ctx, + struct tee_ioctl_open_session_arg *arg, + struct tee_param *normal_param, u32 num_normal_params, + struct tee_param *ocall_param) +{ + struct optee_context_data *ctxdata = ctx->data; + struct optee_session *sess = NULL; + struct optee_call_ctx *call_ctx = NULL; + int sess_tmp_id; + u64 ocall_func; + int rc = 0; - sess = kzalloc(sizeof(*sess), GFP_KERNEL); - if (!sess) { - rc = -ENOMEM; - goto out; - } + if (ocall_param && !ctx->cap_ocall) + return -EOPNOTSUPP; - if (optee_do_call_with_arg(ctx, msg_parg)) { - msg_arg->ret = TEEC_ERROR_COMMUNICATION; - msg_arg->ret_origin = TEEC_ORIGIN_COMMS; - } + ocall_func = ocall_param ? param_get_ocall_func(ocall_param) : 0; + if (ocall_func) { + if (arg->session > INT_MAX) + return -EINVAL; - if (msg_arg->ret == TEEC_SUCCESS) { - /* A new session has been created, add it to the list. */ - sess->session_id = msg_arg->session; + sess_tmp_id = (int)arg->session; mutex_lock(&ctxdata->mutex); - list_add(&sess->list_node, &ctxdata->sess_list); + sess = idr_remove(&ctxdata->tmp_sess_list, sess_tmp_id); mutex_unlock(&ctxdata->mutex); + if (!sess) + return -EINVAL; + + call_ctx = &sess->call_ctx; + if (!call_ctx->rpc_shm) { + rc = -EINVAL; + call_ctx->rpc_arg->ret = TEEC_ERROR_BAD_PARAMETERS; + call_ctx->rpc_arg->ret_origin = TEEC_ORIGIN_COMMS; + goto exit_cancel; + } + + rc = process_ocall_reply(arg->ret, arg->ret_origin, + normal_param, num_normal_params, + ocall_param, call_ctx); + if (rc) + goto exit_cancel; } else { - kfree(sess); + sess = kzalloc(sizeof(*sess), GFP_KERNEL); + if (!sess) + return -ENOMEM; + + call_ctx = &sess->call_ctx; + + /* +2 for the meta parameters added below */ + call_ctx->msg_shm = get_msg_arg(ctx, num_normal_params + 2, + &call_ctx->msg_arg, + &call_ctx->msg_parg); + if (IS_ERR(call_ctx->msg_shm)) { + rc = PTR_ERR(call_ctx->msg_shm); + goto exit_free; + } + + call_ctx->ctx = ctx; + call_ctx->msg_arg->cmd = OPTEE_MSG_CMD_OPEN_SESSION; + call_ctx->msg_arg->cancel_id = arg->cancel_id; + + /* + * Initialize and add the meta parameters needed when opening a + * session. + */ + call_ctx->msg_arg->params[0].attr = + OPTEE_MSG_ATTR_TYPE_VALUE_INPUT | OPTEE_MSG_ATTR_META; + call_ctx->msg_arg->params[1].attr = + OPTEE_MSG_ATTR_TYPE_VALUE_INPUT | OPTEE_MSG_ATTR_META; + memcpy(&call_ctx->msg_arg->params[0].u.value, arg->uuid, + sizeof(arg->uuid)); + call_ctx->msg_arg->params[1].u.value.c = arg->clnt_login; + rc = tee_session_calc_client_uuid((uuid_t *) + &call_ctx->msg_arg->params[1].u.value, + arg->clnt_login, arg->clnt_uuid); + if (rc) + goto exit_free_shm; + + rc = optee_to_msg_param(call_ctx->msg_arg->params + 2, + num_normal_params, normal_param); + if (rc) + goto exit_free_shm; + + call_prologue(call_ctx); } - if (optee_from_msg_param(normal_param, arg->num_params, - msg_arg->params + 2)) { - arg->ret = TEEC_ERROR_COMMUNICATION; - arg->ret_origin = TEEC_ORIGIN_COMMS; - /* Close session again to avoid leakage */ - optee_close_session(ctx, msg_arg->session); + rc = optee_do_call_with_ctx(call_ctx); + if (rc == -EAGAIN) { + rc = process_ocall_request(normal_param, num_normal_params, + ocall_param, call_ctx); + if (rc) + goto exit_cancel; + + /* + * 'sess' becomes globally visible after adding it to the IDR, + * so do not touch it once the mutex is unlocked. + */ + mutex_lock(&ctxdata->mutex); + sess_tmp_id = idr_alloc(&ctxdata->tmp_sess_list, sess, 1, 0, + GFP_KERNEL); + if (sess_tmp_id >= 1) + sess->session_id = sess_tmp_id; + mutex_unlock(&ctxdata->mutex); + if (sess_tmp_id < 0) { + rc = sess_tmp_id; + call_ctx->rpc_arg->ret = TEEC_ERROR_OUT_OF_MEMORY; + call_ctx->rpc_arg->ret_origin = TEEC_ORIGIN_COMMS; + goto exit_cancel; + } + + arg->session = sess_tmp_id; } else { - arg->session = msg_arg->session; - arg->ret = msg_arg->ret; - arg->ret_origin = msg_arg->ret_origin; + call_epilogue(call_ctx); + + if (rc) { + arg->ret = TEEC_ERROR_COMMUNICATION; + arg->ret_origin = TEEC_ORIGIN_COMMS; + } else { + arg->ret = call_ctx->msg_arg->ret; + arg->ret_origin = call_ctx->msg_arg->ret_origin; + } + + if (optee_from_msg_param(normal_param, num_normal_params, + call_ctx->msg_arg->params + 2)) { + if (arg->ret == TEEC_SUCCESS) + close_session(ctx, call_ctx->msg_arg->session); + + arg->ret = TEEC_ERROR_COMMUNICATION; + arg->ret_origin = TEEC_ORIGIN_COMMS; + } + + if (arg->ret) + goto exit_clear_free_all; + + /* + * A new session has been created, initialize it and add it to + * the list. + */ + sema_init(&sess->sem, 1); + arg->session = call_ctx->msg_arg->session; + sess->session_id = call_ctx->msg_arg->session; + + tee_shm_free(call_ctx->msg_shm); + clear_call_ctx(call_ctx); + + mutex_lock(&ctxdata->mutex); + list_add(&sess->list_node, &ctxdata->sess_list); + mutex_unlock(&ctxdata->mutex); + + param_clear_ocall(ocall_param); } -out: - tee_shm_free(shm); return rc; + +exit_cancel: + /* See comment in optee_cancel_open_session_ocall */ + if (cancel_ocall(call_ctx) == 0 && + call_ctx->msg_arg->ret == TEEC_SUCCESS) + close_session(ctx, call_ctx->msg_arg->session); + optee_from_msg_param(normal_param, num_normal_params, + call_ctx->msg_arg->params); +exit_clear_free_all: + param_clear_ocall(ocall_param); +exit_free_shm: + tee_shm_free(call_ctx->msg_shm); +exit_free: + kfree(sess); + return rc; +} + +void optee_cancel_open_session_ocall(struct optee_session *sess) +{ + struct optee_call_ctx *call_ctx = &sess->call_ctx; + + call_ctx->rpc_arg->ret = TEEC_ERROR_TARGET_DEAD; + call_ctx->rpc_arg->ret_origin = TEEC_ORIGIN_COMMS; + + /* + * Reaching this function means an OCALL is pending during session open + * but the CA has terminated abnormally. As such, the OCALL is + * cancelled. However, there is a chance that the TA's session open + * handler ignores the cancellation and lets the session open anyway. If + * that happens, close it. + */ + if (cancel_ocall(&sess->call_ctx) == 0 && + call_ctx->msg_arg->ret == TEEC_SUCCESS) + close_session(call_ctx->ctx, call_ctx->msg_arg->session); + + /* + * Decrease the ref count on all shared memory pointers passed into the + * original function invocation. + */ + process_ocall_memrefs(call_ctx->msg_arg->params, + call_ctx->msg_arg->num_params, false); + + tee_shm_free(call_ctx->msg_shm); + kfree(sess); } int optee_close_session(struct tee_context *ctx, u32 session) { struct optee_context_data *ctxdata = ctx->data; - struct tee_shm *shm; - struct optee_msg_arg *msg_arg; - phys_addr_t msg_parg; struct optee_session *sess; /* Check that the session is valid and remove it from the list */ @@ -226,17 +781,20 @@ int optee_close_session(struct tee_context *ctx, u32 session) mutex_unlock(&ctxdata->mutex); if (!sess) return -EINVAL; - kfree(sess); - shm = get_msg_arg(ctx, 0, &msg_arg, &msg_parg); - if (IS_ERR(shm)) - return PTR_ERR(shm); + /* + * If another thread found the session before we removed it from the + * list and that thread is operating on the session object itself, wait + * until it is done before we destroy it. + */ + down(&sess->sem); - msg_arg->cmd = OPTEE_MSG_CMD_CLOSE_SESSION; - msg_arg->session = session; - optee_do_call_with_arg(ctx, msg_parg); + if (sess->call_ctx.rpc_shm) + optee_cancel_invoke_function_ocall(&sess->call_ctx); + + kfree(sess); + close_session(ctx, session); - tee_shm_free(shm); return 0; } @@ -245,54 +803,140 @@ int optee_invoke_func(struct tee_context *ctx, struct tee_ioctl_invoke_arg *arg, struct tee_param *ocall_param) { struct optee_context_data *ctxdata = ctx->data; - struct tee_shm *shm; - struct optee_msg_arg *msg_arg; - phys_addr_t msg_parg; + struct optee_call_ctx *call_ctx; struct optee_session *sess; - int rc; + u64 ocall_func; + int rc = 0; - if (ocall_param) { - pr_err("OCALLs not supported\n"); - return -EOPNOTSUPP; + if (ocall_param && !ctx->cap_ocall) { + rc = -EOPNOTSUPP; + goto exit; } /* Check that the session is valid */ mutex_lock(&ctxdata->mutex); sess = find_session(ctxdata, arg->session); + if (sess) + down(&sess->sem); mutex_unlock(&ctxdata->mutex); if (!sess) return -EINVAL; - shm = get_msg_arg(ctx, arg->num_params, &msg_arg, &msg_parg); - if (IS_ERR(shm)) - return PTR_ERR(shm); - msg_arg->cmd = OPTEE_MSG_CMD_INVOKE_COMMAND; - msg_arg->func = arg->func; - msg_arg->session = arg->session; - msg_arg->cancel_id = arg->cancel_id; + call_ctx = &sess->call_ctx; + ocall_func = ocall_param ? param_get_ocall_func(ocall_param) : 0; + if (ocall_func) { + /* The current call is a reply to an OCALL request */ - rc = optee_to_msg_param(msg_arg->params, arg->num_params, normal_param); - if (rc) - goto out; + if (!call_ctx->rpc_shm) { + rc = -EINVAL; + goto exit; + } - if (optee_do_call_with_arg(ctx, msg_parg)) { - msg_arg->ret = TEEC_ERROR_COMMUNICATION; - msg_arg->ret_origin = TEEC_ORIGIN_COMMS; + rc = process_ocall_reply(arg->ret, arg->ret_origin, + normal_param, num_normal_params, + ocall_param, call_ctx); + if (rc) + goto exit_cancel; + } else { + /* + * The current call is an invocation that may result in an OCALL + * request. + */ + + if (call_ctx->rpc_shm) { + rc = -EINVAL; + call_ctx->rpc_arg->ret = TEEC_ERROR_BAD_PARAMETERS; + call_ctx->rpc_arg->ret_origin = TEEC_ORIGIN_COMMS; + goto exit_cancel; + } + + call_ctx->msg_shm = get_msg_arg(ctx, num_normal_params, + &call_ctx->msg_arg, + &call_ctx->msg_parg); + if (IS_ERR(call_ctx->msg_shm)) { + rc = PTR_ERR(call_ctx->msg_shm); + goto exit_clear; + } + + call_ctx->ctx = ctx; + call_ctx->msg_arg->cmd = OPTEE_MSG_CMD_INVOKE_COMMAND; + call_ctx->msg_arg->func = arg->func; + call_ctx->msg_arg->session = arg->session; + call_ctx->msg_arg->cancel_id = arg->cancel_id; + + rc = optee_to_msg_param(call_ctx->msg_arg->params, + num_normal_params, normal_param); + if (rc) { + tee_shm_free(call_ctx->msg_shm); + goto exit_clear; + } + + call_prologue(call_ctx); } - if (optee_from_msg_param(normal_param, arg->num_params, - msg_arg->params)) { - msg_arg->ret = TEEC_ERROR_COMMUNICATION; - msg_arg->ret_origin = TEEC_ORIGIN_COMMS; + rc = optee_do_call_with_ctx(call_ctx); + if (rc == -EAGAIN) { + rc = process_ocall_request(normal_param, num_normal_params, + ocall_param, call_ctx); + if (rc) + goto exit_cancel; + } else { + call_epilogue(call_ctx); + + arg->ret = call_ctx->msg_arg->ret; + arg->ret_origin = call_ctx->msg_arg->ret_origin; + + if (rc) { + arg->ret = TEEC_ERROR_COMMUNICATION; + arg->ret_origin = TEEC_ORIGIN_COMMS; + } + + if (optee_from_msg_param(normal_param, num_normal_params, + call_ctx->msg_arg->params)) { + arg->ret = TEEC_ERROR_COMMUNICATION; + arg->ret_origin = TEEC_ORIGIN_COMMS; + } + + tee_shm_free(call_ctx->msg_shm); + clear_call_ctx(call_ctx); + param_clear_ocall(ocall_param); } - arg->ret = msg_arg->ret; - arg->ret_origin = msg_arg->ret_origin; -out: - tee_shm_free(shm); + up(&sess->sem); + return rc; + +exit_cancel: + cancel_ocall(call_ctx); + optee_from_msg_param(normal_param, num_normal_params, + call_ctx->msg_arg->params); + tee_shm_free(call_ctx->msg_shm); + param_clear_ocall(ocall_param); +exit_clear: + clear_call_ctx(call_ctx); +exit: + up(&sess->sem); return rc; } +/* Requires @sem in the parent struct optee_session to be held */ +void optee_cancel_invoke_function_ocall(struct optee_call_ctx *call_ctx) +{ + call_ctx->rpc_arg->ret = TEEC_ERROR_TARGET_DEAD; + call_ctx->rpc_arg->ret_origin = TEEC_ORIGIN_COMMS; + + cancel_ocall(call_ctx); + + /* + * Decrease the ref count on all shared memory pointers passed into the + * original function invocation. + */ + process_ocall_memrefs(call_ctx->msg_arg->params, + call_ctx->msg_arg->num_params, false); + + tee_shm_free(call_ctx->msg_shm); + clear_call_ctx(call_ctx); +} + int optee_cancel_req(struct tee_context *ctx, u32 cancel_id, u32 session) { struct optee_context_data *ctxdata = ctx->data; diff --git a/drivers/tee/optee/core.c b/drivers/tee/optee/core.c index c2fae61401ecb9..21f9d570fe8eed 100644 --- a/drivers/tee/optee/core.c +++ b/drivers/tee/optee/core.c @@ -18,6 +18,7 @@ #include #include #include +#include #include "optee_bench.h" #include "optee_private.h" #include "optee_smc.h" @@ -266,11 +267,10 @@ static int optee_open(struct tee_context *ctx) } mutex_init(&ctxdata->mutex); INIT_LIST_HEAD(&ctxdata->sess_list); + idr_init(&ctxdata->tmp_sess_list); - if (optee->sec_caps & OPTEE_SMC_SEC_CAP_MEMREF_NULL) - ctx->cap_memref_null = true; - else - ctx->cap_memref_null = false; + ctx->cap_memref_null = optee->sec_caps & OPTEE_SMC_SEC_CAP_MEMREF_NULL; + ctx->cap_ocall = optee->sec_caps & OPTEE_SMC_SEC_CAP_OCALL; ctx->data = ctxdata; return 0; @@ -315,6 +315,7 @@ static void optee_release(struct tee_context *ctx) } kfree(sess); } + idr_destroy(&ctxdata->tmp_sess_list); kfree(ctxdata); if (!IS_ERR(shm)) @@ -331,10 +332,68 @@ static void optee_release(struct tee_context *ctx) } } +static void optee_pre_release(struct tee_context *ctx) +{ + struct optee_context_data *ctxdata = ctx->data; + struct optee_session *sess; + bool have_xa = false; + unsigned long i = 0; + struct xarray xa; + int id; + + if (!ctxdata) + return; + + /* + * Only if necessary, add into 'xa' sessions that have to have an OCALL + * cancelled instead of doing so in the loops to avoid calling into + * secure world with @mutex held. + */ + mutex_lock(&ctxdata->mutex); + idr_for_each_entry(&ctxdata->tmp_sess_list, sess, id) { + if (!have_xa) { + xa_init(&xa); + have_xa = true; + } + idr_remove(&ctxdata->tmp_sess_list, id); + xa_store(&xa, i++, xa_tag_pointer(sess, 1), GFP_KERNEL); + } + list_for_each_entry(sess, &ctxdata->sess_list, list_node) { + if (!sess->call_ctx.rpc_shm) + continue; + if (!have_xa) { + xa_init(&xa); + have_xa = true; + } + xa_store(&xa, i++, sess, GFP_KERNEL); + } + mutex_unlock(&ctxdata->mutex); + + if (!have_xa) + return; + + xa_for_each(&xa, i, sess) { + if (xa_pointer_tag(sess)) { + optee_cancel_open_session_ocall(xa_untag_pointer(sess)); + } else { + /* + * Holding @sem here while calling into secure world is + * fine seeing as there is no code path that would + * recursively acquire it. + */ + down(&sess->sem); + optee_cancel_invoke_function_ocall(&sess->call_ctx); + up(&sess->sem); + } + } + + xa_destroy(&xa); +} + static const struct tee_driver_ops optee_ops = { .get_version = optee_get_version, .open = optee_open, - .pre_release = NULL, + .pre_release = optee_pre_release, .release = optee_release, .open_session = optee_open_session, .close_session = optee_close_session, diff --git a/drivers/tee/optee/optee_msg.h b/drivers/tee/optee/optee_msg.h index 8839fa1b2ea767..4c2aae8f932fe7 100644 --- a/drivers/tee/optee/optee_msg.h +++ b/drivers/tee/optee/optee_msg.h @@ -404,12 +404,13 @@ struct optee_msg_arg { */ #define OPTEE_MSG_RPC_CMD_SHM_ALLOC 6 /* Memory that can be shared with a non-secure user space application */ -#define OPTEE_MSG_RPC_SHM_TYPE_APPL 0 +#define OPTEE_MSG_RPC_SHM_TYPE_APPL 0 /* Memory only shared with non-secure kernel */ -#define OPTEE_MSG_RPC_SHM_TYPE_KERNEL 1 +#define OPTEE_MSG_RPC_SHM_TYPE_KERNEL 1 /* Memory shared with non-secure kernel, but exported to userspace */ -#define OPTEE_MSG_RPC_SHM_TYPE_GLOBAL 2 - +#define OPTEE_MSG_RPC_SHM_TYPE_GLOBAL 2 +/* Memory shared with the requesting TA's Client Application */ +#define OPTEE_MSG_RPC_SHM_TYPE_CLIENT_APPL 3 /* * Free shared memory previously allocated with OPTEE_MSG_RPC_CMD_SHM_ALLOC * @@ -429,4 +430,18 @@ struct optee_msg_arg { * [in] param[0].u.value.c Size of buffer */ #define OPTEE_MSG_RPC_CMD_BENCH_REG 20 + +/* + * Send a command to the Client Application. + * + * [in] param[0].u.value[0].a command Id + * [out] param[0].u.value[0].b OCALL return value + * [out] param[0].u.value[0].c OCALL return value origin + * [in] param[0].u.value[1].a UUID of TA whence OCALL originated (Hi) + * [out] param[0].u.value[1].b UUID of TA whence OCALL originated (Lo) + * + * [in/out] any[2..5].* OCALL parameters as specified by the TA, if any + */ +#define OPTEE_MSG_RPC_CMD_OCALL 22 + #endif /* _OPTEE_MSG_H */ diff --git a/drivers/tee/optee/optee_private.h b/drivers/tee/optee/optee_private.h index 53ccc89c3e7663..7e5679c74edbe5 100644 --- a/drivers/tee/optee/optee_private.h +++ b/drivers/tee/optee/optee_private.h @@ -16,10 +16,13 @@ /* Some Global Platform error codes used in this driver */ #define TEEC_SUCCESS 0x00000000 +#define TEEC_ERROR_CANCEL 0xFFFF0002 #define TEEC_ERROR_BAD_PARAMETERS 0xFFFF0006 +#define TEEC_ERROR_NOT_SUPPORTED 0xFFFF000A #define TEEC_ERROR_COMMUNICATION 0xFFFF000E #define TEEC_ERROR_OUT_OF_MEMORY 0xFFFF000C #define TEEC_ERROR_SHORT_BUFFER 0xFFFF0010 +#define TEEC_ERROR_TARGET_DEAD 0xFFFF3024 #define TEEC_ORIGIN_COMMS 0x00000002 @@ -97,15 +100,69 @@ struct optee { struct work_struct scan_bus_work; }; +struct optee_call_waiter { + struct list_head list_node; + struct completion c; +}; + +/** + * struct optee_call_ctx - holds context that is preserved during one STD call + * @pages_list: list of pages allocated for RPC requests + * @num_entries: number of pages in 'pages_list' + * @ctx: TEE context whence the OCALL originated, if any + * @msg_shm: shared memory object used for calling into OP-TEE + * @msg_arg: arguments used for calling into OP-TEE, namely the data + * behind 'msg_shm' + * @msg_parg: physical pointer underlying 'msg_shm' + * @rpc_must_release: indicates that OCALL parameters have had their refcount + * increased and must be decreased on cancellation + * @rpc_shm: shared memory object used for responding to RPCs + * @rpc_arg: arguments used for responding to RPCs, namely the data + * behind 'rpc_shm' + * @thread_id: secure thread Id whence the OCALL originated and which + * must be resumed when replying to the OCALL + * @waiter: object used to wait until a secure thread becomes + * available is the previous call into OP-TEE failed + * because all secure threads are in use + * @ocall_pages_list: list of pages allocated for OCALL requests + * @ocall_num_entries: number of pages in 'ocall_pages_list' + */ +struct optee_call_ctx { + /* Information about pages list used in last allocation */ + void *pages_list; + size_t num_entries; + + /* OCALL support */ + struct tee_context *ctx; + + struct tee_shm *msg_shm; + struct optee_msg_arg *msg_arg; + phys_addr_t msg_parg; + + bool rpc_must_release; + struct tee_shm *rpc_shm; + struct optee_msg_arg *rpc_arg; + + u32 thread_id; + struct optee_call_waiter waiter; + + void *ocall_pages_list; + size_t ocall_num_entries; +}; + struct optee_session { + /* Serializes access to this struct */ + struct semaphore sem; struct list_head list_node; u32 session_id; + struct optee_call_ctx call_ctx; }; struct optee_context_data { /* Serializes access to this struct */ struct mutex mutex; struct list_head sess_list; + struct idr tmp_sess_list; }; struct optee_rpc_param { @@ -119,25 +176,41 @@ struct optee_rpc_param { u32 a7; }; -struct optee_call_waiter { - struct list_head list_node; - struct completion c; -}; - -/* Holds context that is preserved during one STD call */ -struct optee_call_ctx { - /* information about pages list used in last allocation */ - void *pages_list; - size_t num_entries; -}; +/* + * RPC support + */ void optee_handle_rpc(struct tee_context *ctx, struct optee_rpc_param *param, struct optee_call_ctx *call_ctx); +bool optee_rpc_is_ocall(struct optee_rpc_param *param, + struct optee_call_ctx *call_ctx); +int optee_rpc_process_shm_alloc(struct tee_shm *shm, + struct optee_msg_param *msg_param, void **list); void optee_rpc_finalize_call(struct optee_call_ctx *call_ctx); +/* + * Wait queue + */ + void optee_wait_queue_init(struct optee_wait_queue *wq); void optee_wait_queue_exit(struct optee_wait_queue *wq); +/* + * Call queue + */ + +void optee_cq_wait_init(struct optee_call_queue *cq, + struct optee_call_waiter *w); +void optee_cq_wait_for_completion(struct optee_call_queue *cq, + struct optee_call_waiter *w); +void optee_cq_complete_one(struct optee_call_queue *cq); +void optee_cq_wait_final(struct optee_call_queue *cq, + struct optee_call_waiter *w); + +/* + * Supplicant + */ + u32 optee_supp_thrd_req(struct tee_context *ctx, u32 func, size_t num_params, struct tee_param *param); @@ -152,17 +225,40 @@ int optee_supp_recv(struct tee_context *ctx, u32 *func, u32 *num_params, int optee_supp_send(struct tee_context *ctx, u32 ret, u32 num_params, struct tee_param *param); +/* + * Calls into OP-TEE + */ + u32 optee_do_call_with_arg(struct tee_context *ctx, phys_addr_t parg); + +/* + * Sessions + */ + int optee_open_session(struct tee_context *ctx, struct tee_ioctl_open_session_arg *arg, struct tee_param *normal_param, u32 num_normal_params, struct tee_param *ocall_param); int optee_close_session(struct tee_context *ctx, u32 session); + +/* + * Function invocations + */ + int optee_invoke_func(struct tee_context *ctx, struct tee_ioctl_invoke_arg *arg, struct tee_param *normal_param, u32 num_normal_params, struct tee_param *ocall_param); + +/* + * Cancellations + */ + int optee_cancel_req(struct tee_context *ctx, u32 cancel_id, u32 session); +/* + * Shared memory + */ + void optee_enable_shm_cache(struct optee *optee); void optee_disable_shm_cache(struct optee *optee); @@ -176,27 +272,38 @@ int optee_shm_register_supp(struct tee_context *ctx, struct tee_shm *shm, unsigned long start); int optee_shm_unregister_supp(struct tee_context *ctx, struct tee_shm *shm); +/* + * Paremeters + */ + int optee_from_msg_param(struct tee_param *params, size_t num_params, const struct optee_msg_param *msg_params); int optee_to_msg_param(struct optee_msg_param *msg_params, size_t num_params, const struct tee_param *params); +/* + * RPC memory + */ + u64 *optee_allocate_pages_list(size_t num_entries); void optee_free_pages_list(void *array, size_t num_entries); void optee_fill_pages_list(u64 *dst, struct page **pages, int num_pages, size_t page_offset); +/* + * Devices + */ + #define PTA_CMD_GET_DEVICES 0x0 #define PTA_CMD_GET_DEVICES_SUPP 0x1 int optee_enumerate_devices(u32 func); -void optee_cq_wait_init(struct optee_call_queue *cq, - struct optee_call_waiter *w); -void optee_cq_wait_for_completion(struct optee_call_queue *cq, - struct optee_call_waiter *w); -void optee_cq_complete_one(struct optee_call_queue *cq); -void optee_cq_wait_final(struct optee_call_queue *cq, - struct optee_call_waiter *w); +/* + * OCALLs + */ + +void optee_cancel_open_session_ocall(struct optee_session *sess); +void optee_cancel_invoke_function_ocall(struct optee_call_ctx *call_ctx); /* * Small helpers diff --git a/drivers/tee/optee/rpc.c b/drivers/tee/optee/rpc.c index 6ac3058c7d0957..37c5a74607e099 100644 --- a/drivers/tee/optee/rpc.c +++ b/drivers/tee/optee/rpc.c @@ -513,3 +513,38 @@ void optee_handle_rpc(struct tee_context *ctx, struct optee_rpc_param *param, param->a0 = OPTEE_SMC_CALL_RETURN_FROM_RPC; } + +bool optee_rpc_is_ocall(struct optee_rpc_param *param, + struct optee_call_ctx *call_ctx) +{ + u32 func; + u64 shm_type; + + struct tee_shm *shm; + struct optee_msg_arg *arg; + + func = OPTEE_SMC_RETURN_GET_RPC_FUNC(param->a0); + if (func != OPTEE_SMC_RPC_FUNC_CMD) + return false; + + shm = reg_pair_to_ptr(param->a1, param->a2); + arg = tee_shm_get_va(shm, 0); + + switch (arg->cmd) { + case OPTEE_MSG_RPC_CMD_SHM_ALLOC: + case OPTEE_MSG_RPC_CMD_SHM_FREE: + shm_type = arg->params[0].u.value.a; + if (shm_type != OPTEE_MSG_RPC_SHM_TYPE_CLIENT_APPL) + break; + fallthrough; + case OPTEE_MSG_RPC_CMD_OCALL: + call_ctx->rpc_shm = shm; + call_ctx->rpc_arg = arg; + call_ctx->thread_id = param->a3; + return true; + default: + break; + } + + return false; +}