lxcfs: 根据 cgroup 信息生成容器内的 /proc（上）

// lxcfs.c: 717
const struct fuse_operations lxcfs_ops = {
     // ... 省略 ...
     .open = lxcfs_open,
     .read = lxcfs_read,
     // ... 省略 ...
}

// lxcfs.c: 550
static int lxcfs_open(const char *path, struct fuse_file_info *fi)
{
    int ret;
    if (strncmp(path, "/cgroup", 7) == 0) {
        up_users();                    //用户计数器加1
        ret = do_cg_open(path, fi);
        down_users();                  //用户计数器减1
        return ret;
    }
    if (strncmp(path, "/proc", 5) == 0) {
        up_users();                    //用户计数器加1
        ret = do_proc_open(path, fi);
        down_users();                  //用户计数器减1
        return ret;
    }

    return -EACCES;
}

// /usr/include/fuse/fuse_common.h: 45
struct fuse_file_info {
    /** Open flags.  Available in open() and release() */
    int flags;

    /** Old file handle, don't use */
    unsigned long fh_old;

    /** In case of a write operation indicates if this was caused by a
        writepage */
    int writepage;

    /** Can be filled in by open, to use direct I/O on this file.
        Introduced in version 2.4 */
    unsigned int direct_io : 1;

    /** Can be filled in by open, to indicate, that cached file data
        need not be invalidated.  Introduced in version 2.4 */
    unsigned int keep_cache : 1;

    /** Indicates a flush operation.  Set in flush operation, also
        maybe set in highlevel lock operation and lowlevel release
        operation.  Introduced in version 2.6 */
    unsigned int flush : 1;

    /** Can be filled in by open, to indicate that the file is not
        seekable.  Introduced in version 2.8 */
    unsigned int nonseekable : 1;

    /* Indicates that flock locks for this file should be
       released.  If set, lock_owner shall contain a valid value.
       May only be set in ->release().  Introduced in version
       2.9 */
    unsigned int flock_release : 1;

    /** Padding.  Do not use*/
    unsigned int padding : 27;

    /** File handle.  May be filled in by filesystem in open().
        Available in all other file operations */
    uint64_t fh;

    /** Lock owner id.  Available in locking operations and flush */
    uint64_t lock_owner;
};

// lxcfs/bindings.c：4174
int proc_open(const char *path, struct fuse_file_info *fi)
{
    int type = -1;
    struct file_info *info;

    if (strcmp(path, "/proc/meminfo") == 0)
        type = LXC_TYPE_PROC_MEMINFO;
    else if (strcmp(path, "/proc/cpuinfo") == 0)
        type = LXC_TYPE_PROC_CPUINFO;
    else if (strcmp(path, "/proc/uptime") == 0)
        type = LXC_TYPE_PROC_UPTIME;
    else if (strcmp(path, "/proc/stat") == 0)
        type = LXC_TYPE_PROC_STAT;
    else if (strcmp(path, "/proc/diskstats") == 0)
        type = LXC_TYPE_PROC_DISKSTATS;
    else if (strcmp(path, "/proc/swaps") == 0)
        type = LXC_TYPE_PROC_SWAPS;
    if (type == -1)
        return -ENOENT;

    info = malloc(sizeof(*info));
    if (!info)
        return -ENOMEM;

    memset(info, 0, sizeof(*info));
    info->type = type;

    info->buflen = get_procfile_size(path) + BUF_RESERVE_SIZE;
    do {
        info->buf = malloc(info->buflen);
    } while (!info->buf);
    memset(info->buf, 0, info->buflen);
    /* set actual size to buffer size */
    info->size = info->buflen;

    fi->fh = (unsigned long)info;
    return 0;
}

// lxcfs/lxcfs.c: 569
static int lxcfs_read(const char *path, char *buf, size_t size, off_t offset,
        struct fuse_file_info *fi)
{
    int ret;
    if (strncmp(path, "/cgroup", 7) == 0) {
        up_users();
        ret = do_cg_read(path, buf, size, offset, fi);
        down_users();
        return ret;
    }
    if (strncmp(path, "/proc", 5) == 0) {
        up_users();
        ret = do_proc_read(path, buf, size, offset, fi);
        down_users();
        return ret;
    }

    return -EINVAL;
}

// lxcfs/bindings.c：4230
int proc_read(const char *path, char *buf, size_t size, off_t offset,
        struct fuse_file_info *fi)
{
    struct file_info *f = (struct file_info *) fi->fh;

    switch (f->type) {
    case LXC_TYPE_PROC_MEMINFO:
        return proc_meminfo_read(buf, size, offset, fi);
    case LXC_TYPE_PROC_CPUINFO:
        return proc_cpuinfo_read(buf, size, offset, fi);
    case LXC_TYPE_PROC_UPTIME:
        return proc_uptime_read(buf, size, offset, fi);
    case LXC_TYPE_PROC_STAT:
        return proc_stat_read(buf, size, offset, fi);
    case LXC_TYPE_PROC_DISKSTATS:
        return proc_diskstats_read(buf, size, offset, fi);
    case LXC_TYPE_PROC_SWAPS:
        return proc_swaps_read(buf, size, offset, fi);
    default:
        return -EINVAL;
    }
}

//fc->pid就是发起读请求的进程的进程号
struct fuse_context *fc = fuse_get_context();

pid_t initpid = lookup_initpid_in_store(fc->pid);

// lxcfs/bindings.c: 1080
static pid_t lookup_initpid_in_store(pid_t qpid)
{
    pid_t answer = 0;
    struct stat sb;
    struct pidns_init_store *e;
    char fnam[100];

    snprintf(fnam, 100, "/proc/%d/ns/pid", qpid);
    store_lock();
    if (stat(fnam, &sb) < 0)
        goto out;
    e = lookup_verify_initpid(&sb);
    if (e) {
        answer = e->initpid;
        goto out;
    }
    answer = get_init_pid_for_task(qpid);
    if (answer > 0)
        save_initpid(&sb, answer);

out:
    /* we prune at end in case we are returning
     * the value we were about to return */
    prune_initpid_store();
    store_unlock();
    return answer;
}

// lxcfs/bindings.c: 1046
static pid_t get_init_pid_for_task(pid_t task)
{
    int sock[2];
    pid_t pid;
    pid_t ret = -1;
    char v = '0';
    struct ucred cred;

    if (socketpair(AF_UNIX, SOCK_DGRAM, 0, sock) < 0) {
        perror("socketpair");
        return -1;
    }

    pid = fork();
    if (pid < 0)
        goto out;
    if (!pid) {
        close(sock[1]);
        // 子进程调用函数，写入sock[0]
        write_task_init_pid_exit(sock[0], task);
        _exit(0);
    }

    if (!recv_creds(sock[1], &cred, &v))
        goto out;
    ret = cred.pid;

out:
    close(sock[0]);
    close(sock[1]);
    if (pid > 0)
        wait_for_pid(pid);
    return ret;
}

// lxcfs/bindings.c: 999
static void write_task_init_pid_exit(int sock, pid_t target)
{
    char fnam[100];
    pid_t pid;
    int fd, ret;
    size_t stack_size = sysconf(_SC_PAGESIZE);
    void *stack = alloca(stack_size);
    
    //获取发起读请求的进程的namespace，也就是容器的namespace
    ret = snprintf(fnam, sizeof(fnam), "/proc/%d/ns/pid", (int)target);
    if (ret < 0 || ret >= sizeof(fnam))
        _exit(1);

    fd = open(fnam, O_RDONLY);
    if (fd < 0) {
        perror("write_task_init_pid_exit open of ns/pid");
        _exit(1);
    }
    // 绑定到容器的namespace
    if (setns(fd, 0)) {
        perror("write_task_init_pid_exit setns 1");
        close(fd);
        _exit(1);
    }
    // 传给孙进程执行
    pid = clone(send_creds_clone_wrapper, stack + stack_size, SIGCHLD, &sock);
    if (pid < 0)
        _exit(1);
    if (pid != 0) {
        if (!wait_for_pid(pid))
            _exit(1);
        _exit(0);
    }
}

// lxcfs/bindings.c：1031
static int send_creds_clone_wrapper(void *arg) {
    struct ucred cred;
    char v;
    int sock = *(int *)arg;

    /* we are the child */
    cred.uid = 0;
    cred.gid = 0;
    cred.pid = 1;      //传入的进程号是1
    v = '1';
    if (send_creds(sock, &cred, v, true) != SEND_CREDS_OK)
        return 1;
    return 0;
}

// lxcfs/bindings.c：2053
static int send_creds(int sock, struct ucred *cred, char v, bool pingfirst)
{
    struct msghdr msg = { 0 };
    struct iovec iov;
    struct cmsghdr *cmsg;
    char cmsgbuf[CMSG_SPACE(sizeof(*cred))];
    char buf[1];
    buf[0] = 'p';

    if (pingfirst) {
        if (msgrecv(sock, buf, 1) != 1) {
            lxcfs_error("%s\n", "Error getting reply from server over socketpair.");
            return SEND_CREDS_FAIL;
        }
    }

    msg.msg_control = cmsgbuf;
    msg.msg_controllen = sizeof(cmsgbuf);

    cmsg = CMSG_FIRSTHDR(&msg);
    cmsg->cmsg_len = CMSG_LEN(sizeof(struct ucred));
    cmsg->cmsg_level = SOL_SOCKET;
    cmsg->cmsg_type = SCM_CREDENTIALS;
    memcpy(CMSG_DATA(cmsg), cred, sizeof(*cred));

    msg.msg_name = NULL;
    msg.msg_namelen = 0;

    buf[0] = v;
    iov.iov_base = buf;
    iov.iov_len = sizeof(buf);
    msg.msg_iov = &iov;
    msg.msg_iovlen = 1;

    if (sendmsg(sock, &msg, 0) < 0) {
        lxcfs_error("Failed at sendmsg: %s.\n",strerror(errno));
        if (errno == 3)
            return SEND_CREDS_NOTSK;
        return SEND_CREDS_FAIL;
    }

    return SEND_CREDS_OK;
}

// lxcfs/bindings.c: 3347
cg = get_pid_cgroup(initpid, "cpuset");    // cg就是/proc/XX/cgroup中"cpuset"对应的路径
if (!cg)
    return read_file("proc/cpuinfo", buf, size, d);
prune_init_slice(cg);