Linux内核模块加载深度剖析(中篇)

模块加载

load_module()函数调用的功能函数如下：

• module_sig_check()函数用于检查模块的签名验证
• elf_header_check()函数用于检查模块的elf头和区段有效性
• layout_and_allocate()函数用于分配内核内存空间，把模块相关的节区复制过来
• audit_log_kern_module()函数用于检查是否开启了安全审计
• add_unformed_module()函数用于判断模块是否已经加载，若没有则将模块添加到内核模块链表中
• percpu_modalloc()函数用于申请percpu变量内存，这个变量每个CPU都有一份，无相独立
• module_unload_init()函数用于初始化化模块依赖和引用链表，并对模块引用计数加1
• init_param_lock()函数用于初始化互斥锁
• find_module_sections()函数用于遍历模块中的其它分段
• check_module_license_and_versions()函数用于检测模块的CRC是否正确，license授权是否正确
• setup_modinfo()函数根据.modinfo段设置模块信息
• simplify_symbols()函数用于将模块内存中的静态链接重定位表中的符号，全部指向其真实的内存地址
• apply_relocations()函数用于遍历目标文件中的所有内存节区的重定位节区，并遍历每个节区中的每个静态链接重定位表项，对其做静态链接
• post_relocation()函数用于对重定位后的percpu变量重新赋值，并将即将加载完成的模块的符号加入到内核模块的符号链表中，如果成功加载此模块且内核配置了CONFIG_KALLSYMS，那么在/proc/kallsyms下可以看到此模块的符号
• flush_module_icache()函数用于刷新模块的init_layout和core_layout的cache。
• strndup_user()函数用于复制用户空间的参数到内核空间
• dynamic_debug_setup()函数用于处理debug节区，需要开启内核CONFIG_DYNAMIC_DEBUG才会启用
• ftrace_module_init()函数需要开启相关的ftrace配置
• complete_formation()函数用于遍历模块中的所有导出函数，并检查在内核中是否有同名的导出符号，为模块的init_layout/core_layout做RONX保护
• prepare_coming_module()函数用于发送模块加载通知链
• parse_args()函数用于参数解析与sysfs、livepatch的设置
• mod_sysfs_setup()函数用于在sysfs中创建模块相应的项
• is_livepatch_module()函数用于检查模块是否是热补丁模块
• copy_module_elf()函数用于拷贝模块的elf头
• free_copy()函数用于释放最初内核申请的用于保存模块原文件信息的内存
• trace_module_load()函数用于加载一些和trace相关的内容，便于后期跟踪调试
• do_init_module()函数用于调用模块的初始化函数

static int load_module(struct load_info *info, const char __user *uargs,
         int flags)
{
 struct module *mod;
 long err;
 char *after_dashes;

 err = module_sig_check(info, flags);
 if (err)
  goto free_copy;

 err = elf_header_check(info);
 if (err)
  goto free_copy;

 /* Figure out module layout, and allocate all the memory. */
 mod = layout_and_allocate(info, flags);
 if (IS_ERR(mod)) {
  err = PTR_ERR(mod);
  goto free_copy;
 }

 audit_log_kern_module(mod->name);

 /* Reserve our place in the list. */
 err = add_unformed_module(mod);
 if (err)
  goto free_module;

#ifdef CONFIG_MODULE_SIG
 mod->sig_ok = info->sig_ok;
 if (!mod->sig_ok) {
  pr_notice_once("%s: module verification failed: signature "
          "and/or required key missing - tainting "
          "kernel\n", mod->name);
  add_taint_module(mod, TAINT_UNSIGNED_MODULE, LOCKDEP_STILL_OK);
 }
#endif

 /* To avoid stressing percpu allocator, do this once we're unique. */
 err = percpu_modalloc(mod, info);
 if (err)
  goto unlink_mod;

 /* Now module is in final location, initialize linked lists, etc. */
 err = module_unload_init(mod);
 if (err)
  goto unlink_mod;

 init_param_lock(mod);

 /* Now we've got everything in the final locations, we can
  * find optional sections. */
 err = find_module_sections(mod, info);
 if (err)
  goto free_unload;

 err = check_module_license_and_versions(mod);
 if (err)
  goto free_unload;

 /* Set up MODINFO_ATTR fields */
 setup_modinfo(mod, info);

 /* Fix up syms, so that st_value is a pointer to location. */
 err = simplify_symbols(mod, info);
 if (err < 0)
  goto free_modinfo;

 err = apply_relocations(mod, info);
 if (err < 0)
  goto free_modinfo;

 err = post_relocation(mod, info);
 if (err < 0)
  goto free_modinfo;

 flush_module_icache(mod);

 /* Now copy in args */
 mod->args = strndup_user(uargs, ~0UL >> 1);
 if (IS_ERR(mod->args)) {
  err = PTR_ERR(mod->args);
  goto free_arch_cleanup;
 }

 dynamic_debug_setup(mod, info->debug, info->num_debug);

 /* Ftrace init must be called in the MODULE_STATE_UNFORMED state */
 ftrace_module_init(mod);

 /* Finally it's fully formed, ready to start executing. */
 err = complete_formation(mod, info);
 if (err)
  goto ddebug_cleanup;

 err = prepare_coming_module(mod);
 if (err)
  goto bug_cleanup;

 /* Module is ready to execute: parsing args may do that. */
 after_dashes = parse_args(mod->name, mod->args, mod->kp, mod->num_kp,
      -32768, 32767, mod,
      unknown_module_param_cb);
 if (IS_ERR(after_dashes)) {
  err = PTR_ERR(after_dashes);
  goto coming_cleanup;
 } else if (after_dashes) {
  pr_warn("%s: parameters '%s' after `--' ignored\n",
         mod->name, after_dashes);
 }

 /* Link in to sysfs. */
 err = mod_sysfs_setup(mod, info, mod->kp, mod->num_kp);
 if (err < 0)
  goto coming_cleanup;

 if (is_livepatch_module(mod)) {
  err = copy_module_elf(mod, info);
  if (err < 0)
   goto sysfs_cleanup;
 }

 /* Get rid of temporary copy. */
 free_copy(info);

 /* Done! */
 trace_module_load(mod);

 return do_init_module(mod);

 sysfs_cleanup:
 mod_sysfs_teardown(mod);
 coming_cleanup:
 mod->state = MODULE_STATE_GOING;
 destroy_params(mod->kp, mod->num_kp);
 blocking_notifier_call_chain(&module_notify_list,
         MODULE_STATE_GOING, mod);
 klp_module_going(mod);
 bug_cleanup:
 mod->state = MODULE_STATE_GOING;
 /* module_bug_cleanup needs module_mutex protection */
 mutex_lock(&module_mutex);
 module_bug_cleanup(mod);
 mutex_unlock(&module_mutex);

 /* we can't deallocate the module until we clear memory protection */
 module_disable_ro(mod);
 module_disable_nx(mod);

 ddebug_cleanup:
 dynamic_debug_remove(mod, info->debug);
 synchronize_sched();
 kfree(mod->args);
 free_arch_cleanup:
 module_arch_cleanup(mod);
 free_modinfo:
 free_modinfo(mod);
 free_unload:
 module_unload_free(mod);
 unlink_mod:
 mutex_lock(&module_mutex);
 /* Unlink carefully: kallsyms could be walking list. */
 list_del_rcu(&mod->list);
 mod_tree_remove(mod);
 wake_up_all(&module_wq);
 /* Wait for RCU-sched synchronizing before releasing mod->list. */
 synchronize_sched();
 mutex_unlock(&module_mutex);
 free_module:
 /*
  * Ftrace needs to clean up what it initialized.
  * This does nothing if ftrace_module_init() wasn't called,
  * but it must be called outside of module_mutex.
  */
 ftrace_release_mod(mod);
 /* Free lock-classes; relies on the preceding sync_rcu() */
 lockdep_free_key_range(mod->core_layout.base, mod->core_layout.size);

 module_deallocate(mod, info);
 free_copy:
 free_copy(info);
 return err;
}

module_sig_check()函数首先判断elf模块文件长度是否大于签名字符串长度。然后比较该elf格式的模块文件是否在文件末尾有signature string，若有的话则将该文件长度减去该字符串长度。最后调用mod_verify_sig()函数验证模块签名。

static int module_sig_check(struct load_info *info, int flags)
{
 int err = -ENOKEY;
 const unsigned long markerlen = sizeof(MODULE_SIG_STRING) - 1;
 const void *mod = info->hdr;

 /*
  * Require flags == 0, as a module with version information
  * removed is no longer the module that was signed
  */
 if (flags == 0 &&
     info->len > markerlen &&
     memcmp(mod + info->len - markerlen, MODULE_SIG_STRING, markerlen) == 0) {
  /* We truncate the module to discard the signature */
  info->len -= markerlen;
  err = mod_verify_sig(mod, &info->len);
 }

 if (!err) {
  info->sig_ok = true;
  return 0;
 }

 /* Not having a signature is only an error if we're strict. */
 if (err == -ENOKEY && !sig_enforce)
  err = 0;

 return err;
}

mod_verify_sig()函数首先提取模块签名字符串中的module_signature结构并保存在变量ms中，然后修改模块的文件长度，然后获取签名数据长度保存在变量sig_len变量中，最后调用verify_pkcs7_signature()函数对内核模块中的PKCS#7格式的消息进行验证。

int mod_verify_sig(const void *mod, unsigned long *_modlen)
{
 struct module_signature ms;
 size_t modlen = *_modlen, sig_len;

 pr_devel("==>%s(,%zu)\n", __func__, modlen);

 if (modlen <= sizeof(ms))
  return -EBADMSG;

 memcpy(&ms, mod + (modlen - sizeof(ms)), sizeof(ms));
 modlen -= sizeof(ms);

 sig_len = be32_to_cpu(ms.sig_len);
 if (sig_len >= modlen)
  return -EBADMSG;
 modlen -= sig_len;
 *_modlen = modlen;

 if (ms.id_type != PKEY_ID_PKCS7) {
  pr_err("Module is not signed with expected PKCS#7 message\n");
  return -ENOPKG;
 }

 if (ms.algo != 0 ||
     ms.hash != 0 ||
     ms.signer_len != 0 ||
     ms.key_id_len != 0 ||
     ms.__pad[0] != 0 ||
     ms.__pad[1] != 0 ||
     ms.__pad[2] != 0) {
  pr_err("PKCS#7 signature info has unexpected non-zero params\n");
  return -EBADMSG;
 }

 return verify_pkcs7_signature(mod, modlen, mod + modlen, sig_len,
          NULL, VERIFYING_MODULE_SIGNATURE,
          NULL, NULL);
}

elf_header_check()函数检查elf文件头，以及文件类型,架构以及节区大小是否设置正确，检查节区的偏移地址

static int elf_header_check(struct load_info *info)
{
 if (info->len < sizeof(*(info->hdr)))
  return -ENOEXEC;

 if (memcmp(info->hdr->e_ident, ELFMAG, SELFMAG) != 0
     || info->hdr->e_type != ET_REL
     || !elf_check_arch(info->hdr)
     || info->hdr->e_shentsize != sizeof(Elf_Shdr))
  return -ENOEXEC;

 if (info->hdr->e_shoff >= info->len
     || (info->hdr->e_shnum * sizeof(Elf_Shdr) >
  info->len - info->hdr->e_shoff))
  return -ENOEXEC;

 return 0;
}

layout_and_allocate()函数调用setup_load_info()函数初始化load_info结构体变量info，调用blacklisted()函数检查模块名字字符串是否在模块加载黑名单中，调用check_modinfo()函数检查modinfo节区信息，module_frob_arch_sections()函数在arm32架构下为空，调用find_sec()函数为.data..ro_after_init节区段加上ro_after_init属性，调用layout_sections()函数计算模块内存ELF节区最终需要的内存大小和总大小，并记录到core_layout/init_layout中，调用layout_symtab()函数在init_layout为静态链接符号表,core_layout为核心符号表预留空间，调用move_module()函数分配模块内存布局，复制模块二进制代码到真正运行时内存，修复所有模块内存ELF对应节区头部表指向内存布局，切换模块的struct module结构体变量mod指向最终模块内存布局中的位置，调用kmemleak_load_module()函数扫描检查各个节区的内存分布。

static struct module *layout_and_allocate(struct load_info *info, int flags)
{
 /* Module within temporary copy. */
 struct module *mod;
 unsigned int ndx;
 int err;

 mod = setup_load_info(info, flags);
 if (IS_ERR(mod))
  return mod;

 if (blacklisted(info->name))
  return ERR_PTR(-EPERM);

 err = check_modinfo(mod, info, flags);
 if (err)
  return ERR_PTR(err);

 /* Allow arches to frob section contents and sizes.  */
 err = module_frob_arch_sections(info->hdr, info->sechdrs,
     info->secstrings, mod);
 if (err < 0)
  return ERR_PTR(err);

 /* We will do a special allocation for per-cpu sections later. */
 info->sechdrs[info->index.pcpu].sh_flags &= ~(unsigned long)SHF_ALLOC;

 /*
  * Mark ro_after_init section with SHF_RO_AFTER_INIT so that
  * layout_sections() can put it in the right place.
  * Note: ro_after_init sections also have SHF_{WRITE,ALLOC} set.
  */
 ndx = find_sec(info, ".data..ro_after_init");
 if (ndx)
  info->sechdrs[ndx].sh_flags |= SHF_RO_AFTER_INIT;

 /* Determine total sizes, and put offsets in sh_entsize.  For now
    this is done generically; there doesn't appear to be any
    special cases for the architectures. */
 layout_sections(mod, info);
 layout_symtab(mod, info);

 /* Allocate and move to the final place */
 err = move_module(mod, info);
 if (err)
  return ERR_PTR(err);

 /* Module has been copied to its final place now: return it. */
 mod = (void *)info->sechdrs[info->index.mod].sh_addr;
 kmemleak_load_module(mod, info);
 return mod;
}

add_unformed_module()函数调用find_module_all()函数遍历内核中的所有模块检查是否有同名模块已经存在或正在加载，最后调用mod_update_bounds()函数更新内核已有模块的地址范围边界。

static int add_unformed_module(struct module *mod)
{
 int err;
 struct module *old;

 mod->state = MODULE_STATE_UNFORMED;

again:
 mutex_lock(&module_mutex);
 old = find_module_all(mod->name, strlen(mod->name), true);
 if (old != NULL) {
  if (old->state != MODULE_STATE_LIVE) {
   /* Wait in case it fails to load. */
   mutex_unlock(&module_mutex);
   err = wait_event_interruptible(module_wq,
            finished_loading(mod->name));
   if (err)
    goto out_unlocked;
   goto again;
  }
  err = -EEXIST;
  goto out;
 }
 mod_update_bounds(mod);
 list_add_rcu(&mod->list, &modules);
 mod_tree_insert(mod);
 err = 0;

out:
 mutex_unlock(&module_mutex);
out_unlocked:
 return err;
}

本篇主要介绍了load_module()函数的前半部分实现，主要是模块的签名验签和拷贝模块到真正运行时内存。