openwrt procd启动流程和脚本分析

Linux内核执行start_kernel函数时会调用kernel_init来启动init进程，流程如下图：

kernel_init部分代码如下：

994     if (execute_command) {
 995         ret = run_init_process(execute_command);
 996         if (!ret)
 997             return 0;
 998         panic("Requested init %s failed (error %d).",999               execute_command,ret);
1000     }
1001     if (!try_to_run_init_process("/sbin/init") ||
1002         !try_to_run_init_process("/etc/init") ||
1003         !try_to_run_init_process("/bin/init") ||
1004         !try_to_run_init_process("/bin/sh"))
1005         return 0;
1006
1007     panic("No working init found.  Try passing init= option to kernel. "
1008           "See Linux Documentation/init.txt for guidance.");

接着分析openwrt中package/system/procd/Makefile，这里将procd源码编译生成的可执行文件安装到文件系统的*/sbin*目录中。

define Package/procd/install
    $(INSTALL_DIR) $(1)/sbin $(1)/etc $(1)/lib/functions

    $(INSTALL_BIN) $(PKG_INSTALL_DIR)/usr/sbin/{init,procd,askfirst,udevtrigger} $(1)/sbin/
    $(INSTALL_DATA) $(PKG_INSTALL_DIR)/usr/lib/libsetlbf.so $(1)/lib
    $(INSTALL_BIN) ./files/reload_config $(1)/sbin/
    $(INSTALL_DATA) ./files/hotplug*.json $(1)/etc/
    $(INSTALL_DATA) ./files/procd.sh $(1)/lib/functions/
endef

查看procd源码目录的CMakeList.txt，以init为例，对应源码编译文件如下

56 IF(DISABLE_INIT)
 57 ADD_DEFINITIONS(-DDISABLE_INIT)
 58 ELSE()
 59 ADD_EXECUTABLE(init initd/init.c initd/early.c initd/preinit.c initd/mkdev.c watchdog.c
 60     utils/utils.c ${SOURCES_ZRAM})
 61 TARGET_LINK_LIBRARIES(init ${LIBS})
 62 INSTALL(TARGETS init
 63     RUNTIME DESTINATION ${CMAKE_INSTALL_SBINDIR}
 64 )
 65
 66 ADD_EXECUTABLE(udevtrigger plug/udevtrigger.c)
 67 INSTALL(TARGETS udevtrigger
 68     RUNTIME DESTINATION ${CMAKE_INSTALL_SBINDIR}
 69 )
 70 ENDIF()

main函数入口位于initd/init.c

int
main(int argc,char **argv)
{
    pid_t pid;

    ulog_open(ULOG_KMSG,LOG_DAEMON,"init");

    sigaction(SIGTERM,&sa_shutdown,NULL);
    sigaction(SIGUSR1,NULL);
    sigaction(SIGUSR2,NULL);

	early();
    
	cmdline();
    
	watchdog_init(1);

	pid = fork();
    if (!pid) {
        char *kmod[] = { "/sbin/kmodloader","/etc/modules-boot.d/",NULL };

        if (debug < 3)
            patch_stdio("/dev/null");

        execvp(kmod[0],kmod);
        ERROR("Failed to start kmodloadern");
        exit(-1);
    }
    if (pid <= 0) {
        ERROR("Failed to start kmodloader instancen");
    } else {
        int i;

        for (i = 0; i < 1200; i++) {
            if (waitpid(pid,NULL,WNOHANG) > 0)
                break;
            usleep(10 * 1000);
            watchdog_ping();
        }
    }
    
	uloop_init();
	preinit();
    uloop_run();

    return 0;
}

uloop_init实现位于libubox源码uloop.c

int uloop_init(void)
{
	if (uloop_init_pollfd() < 0)
        return -1;

    if (waker_init() < 0) {
		uloop_done();
		return -1;
    }

    return 0;
}

static int uloop_init_pollfd(void)
{
    if (poll_fd >= 0)
        return 0;

    poll_fd = epoll_create(32);	
    if (poll_fd < 0)
        return -1;

    fcntl(poll_fd,F_SETFD,fcntl(poll_fd,F_GETFD) | FD_CLOEXEC);
    return 0;
}

preinit实现位于procd源码文件initd/preinit.c

static struct uloop_process preinit_proc;
static struct uloop_process plugd_proc;

void
preinit(void)
{
    char *init[] = { "/bin/sh","/etc/preinit",NULL };
    char *plug[] = { "/sbin/procd","-h","/etc/hotplug-preinit.json",NULL };
    int fd;

    LOG("- preinit -n");

    plugd_proc.cb = plugd_proc_cb;
    plugd_proc.pid = fork();
    if (!plugd_proc.pid) {
        execvp(plug[0],plug);
        ERROR("Failed to start plugdn");
        exit(-1);
    }
    if (plugd_proc.pid <= 0) {
        ERROR("Failed to start new plugd instancen");
        return;
    }
    uloop_process_add(&plugd_proc);	

    setenv("PREINIT","1",1);

    fd = creat("/tmp/.preinit",0600);

    if (fd < 0)
        ERROR("Failed to create sentinel filen");
    else
        close(fd);

    preinit_proc.cb = spawn_procd;
    preinit_proc.pid = fork();
    if (!preinit_proc.pid) {
        execvp(init[0],init);
        ERROR("Failed to start preinitn");
        exit(-1);
    }
    if (preinit_proc.pid <= 0) {
        ERROR("Failed to start new preinit instancen");
        return;
    }
    uloop_process_add(&preinit_proc);

    DEBUG(4,"Launched preinit instance,pid=%dn",(int) preinit_proc.pid);
}

这里fork出2个子进程，执行procd和*/etc/preinit*，

• 首先看procd，因为带有参数*“-h /etc/hotplug-preinit.json”，所以会执行hotplug_run*函数。

int main(int argc,char **argv)
{
    int ch;
    char *dbglvl = getenv("DBGLVL");
    int ulog_channels = ULOG_KMSG;

    if (dbglvl) {
        debug = atoi(dbglvl);
        unsetenv("DBGLVL");
    }

    while ((ch = getopt(argc,argv,"d:s:h:S")) != -1) {
        switch (ch) {
        case ‘h‘:
            return hotplug_run(optarg);
        case ‘s‘:
            ubus_socket = optarg;
            break;
        case ‘d‘:
            debug = atoi(optarg);
            break;
        case ‘S‘:
            ulog_channels = ULOG_STDIO;
            break;
        default:
            return usage(argv[0]);
        }
    }

    ulog_open(ulog_channels,"procd");

    setsid();
    uloop_init();
    procd_signal();
    if (getpid() != 1)
        procd_connect_ubus();
    else
        procd_state_next();
    uloop_run();
    uloop_done();

    return 0;
}

hotplug实现如下，这里是建立netlink通信机制，完成用户层和内核的交互，监听内核的uevent事件。

void hotplug(char *rules)
{
    struct sockaddr_nl nls;
    int nlbufsize = 512 * 1024;

    rule_file = strdup(rules);
    memset(&nls,sizeof(struct sockaddr_nl));
    nls.nl_family = AF_NETLINK;
    nls.nl_pid = getpid();
    nls.nl_groups = -1;

    if ((hotplug_fd.fd = socket(PF_NETLINK,SOCK_DGRAM | SOCK_CLOEXEC,NETLINK_KOBJECT_UEVENT)) == -1) {
        ERROR("Failed to open hotplug socket: %sn",strerror(errno));
        exit(1);
    }
    if (bind(hotplug_fd.fd,(void *)&nls,sizeof(struct sockaddr_nl))) {
        ERROR("Failed to bind hotplug socket: %sn",strerror(errno));
        exit(1);
    }

    if (setsockopt(hotplug_fd.fd,SOL_SOCKET,SO_RCVBUFFORCE,&nlbufsize,sizeof(nlbufsize)))
        ERROR("Failed to resize receive buffer: %sn",strerror(errno));

    json_script_init(&jctx);
    queue_proc.cb = queue_proc_cb;
    uloop_fd_add(&hotplug_fd,ULOOP_READ);
}

int hotplug_run(char *rules)
{
    uloop_init();
    hotplug(rules);
    uloop_run();

    return 0;
}

• /etc/preinit脚本大致内容如下，先调用另外的shell脚本，获取函数定义

. /lib/functions.sh
. /lib/functions/preinit.sh
. /lib/functions/system.sh

# 初始化hook链
boot_hook_init preinit_essential
boot_hook_init preinit_main
boot_hook_init failsafe
boot_hook_init initramfs

# 依次执行/lib/preinit目录中的脚本，将函数调用添加到hook链中
for pi_source_file in /lib/preinit/*; do
    . $pi_source_file
done

# 执行preinit_essential注册的hook链的所有函数
boot_run_hook preinit_essential
 
# 执行preinit_main注册的hook链的所有函数
boot_run_hook preinit_main

lib/functions/preinit.sh中定义

boot_hook_init() {
    local hook="${1}_hook"
    export -n "PI_STACK_LIST=${PI_STACK_LIST:+$PI_STACK_LIST }$hook"
    export -n "$hook="
}

/lib/preinit/10_sysinfo中添加hook函数

boot_hook_add preinit_main do_sysinfo_generic

/etc/preinit脚本执行完成后，调用spawn_procd

static void
spawn_procd(struct uloop_process *proc,int ret)
{
    char *wdt_fd = watchdog_fd();
    char *argv[] = { "/sbin/procd",NULL};
    struct stat s;
    char dbg[2];

    if (plugd_proc.pid > 0)
        kill(plugd_proc.pid,SIGKILL);

    if (!stat("/tmp/sysupgrade",&s))
        while (true)
            sleep(1);

    unsetenv("INITRAMFS");
    unsetenv("PREINIT");
    unlink("/tmp/.preinit");
    DEBUG(2,"Exec to real procd nown");
    if (wdt_fd)
        setenv("WDTFD",wdt_fd,1);
    check_dbglvl();
    if (debug > 0) {
        snprintf(dbg,2,"%d",debug);
        setenv("DBGLVL",dbg,1);
    }

	//调用procd
    execvp(argv[0],argv);
}

此时getpid()等于1，所以调用procd_state_next，进入到状态机处理中。

对应的procd log如下，procd state不断迁移，包括STATE_EARLY，STATE_UBUS，STATE_INIT等。

[    [email?protected]] init: Console is alive		
[    [email?protected]] init: Ping		
[    [email?protected]] init: Ping
[    [email?protected]] kmodloader: loading kernel modules from /etc/modules-boot.d/*	
[    [email?protected]] init: Ping
[    [email?protected]] kmodloader: done loading kernel modules from /etc/modules-boot.d/*
[    [email?protected]] init: Ping
[    [email?protected]] init: - preinit -
[    [email?protected]] init: Launched preinit instance,pid=1308
[    [email?protected]] init: Exec to real procd now
[    [email?protected]] procd: - early -
[    [email?protected]] procd: Finished udevtrigger
[    [email?protected]] procd: Coldplug complete
[    [email?protected]] procd: - ubus -
[    [email?protected]] procd: Create service ubus
[    [email?protected]] procd: Create instance ubus::instance1
[    [email?protected]] procd: Started instance ubus::instance1[1554]
[    [email?protected]] procd: Connected to ubus,id=459ede6c
[    [email?protected]] procd: - init -
[    [email?protected]] procd: Launched new askconsole action,pid=1555
[    [email?protected]] procd: Launched new askfirst action,pid=1556

以STATE_INIT为例，执行procd_inittab_run("xxx")会调用对应handlers的callback，对应所有的init_action是在*procd_inittab()*中添加的。

case STATE_INIT:
        LOG("- init -n");
        procd_inittab();
        procd_inittab_run("respawn");
        procd_inittab_run("askconsole");
        procd_inittab_run("askfirst");
        procd_inittab_run("sysinit");

static struct init_handler handlers[] = {
    {
        .name = "sysinit",.cb = runrc,},{
        .name = "shutdown",{
        .name = "askfirst",.cb = askfirst,.multi = 1,{
        .name = "askconsole",.cb = askconsole,{
        .name = "respawn",.cb = rcrespawn,}
};

void procd_inittab_run(const char *handler)
{
    struct init_action *a;

    list_for_each_entry(a,&actions,list) {
        if (!strcmp(a->handler->name,handler)) {
            if (a->handler->multi) {
                a->handler->cb(a);
                continue;
            }
            a->handler->cb(a);
            break;
        }
    }
}

这里来看runrc的实现，代码位于inittab.c

static void runrc(struct init_action *a)
{
    if (!a->argv[1] || !a->argv[2]) {
        ERROR("valid format is rcS <S|K> <param>n");
        return;
    }

    /* proceed even if no init or shutdown scripts run */
    if (rcS(a->argv[1],a->argv[2],rcdone)) {
        printf("---rcdone---n");
        rcdone(NULL);
    } else {
        printf("----rcdone errorn");
    }
}

rcS.c

int rcS(char *pattern,char *param,void (*q_empty)(struct runqueue *))
{
    runqueue_init(&q);
    q.empty_cb = q_empty;
    q.max_running_tasks = 1;

    return _rc(&q,"/etc/rc.d",pattern,"*",param);
}

执行*/etc/rc.d*目录下S/K开头的脚本