在线代码运行

代码运行流程：

1. 用户输入代码
2. 将代码写入宿主机的临时文件中
3. 启动一个Docker容器
4. 将临时文件复制到容器中
5. 利用容器执行编译运行代码
6. 返回执行结果
7. 清理临时文件和Docker容器

镜像构建

Dockerfile:

# Create from ubuntu image
FROM ubuntu:20.04

# Change default shell to bash
SHELL ["/bin/bash", "-c"]

# 设置为中国国内源
RUN sed -i s@/ports.ubuntu.com/@/mirrors.aliyun.com/@g /etc/apt/sources.list
RUN sed -i s@/archive.ubuntu.com/@/mirrors.aliyun.com/@g /etc/apt/sources.list
RUN sed -i s@/security.ubuntu.com/@/mirrors.aliyun.com/@g /etc/apt/sources.list

RUN apt-get clean

# Update the image to the latest packages
RUN apt-get update
RUN apt-get upgrade -y

# Install required packages
RUN apt-get install software-properties-common -y
RUN apt-get install zip unzip curl wget tar -y


# Install python
RUN apt-get install python python3-pip -y

# Install C
RUN apt-get install gcc -y

# Install C++
RUN apt-get install g++ -y

# Install java
RUN apt-get install default-jre -y
RUN apt-get install default-jdk -y

# Install nodejs
RUN curl -fsSL https://deb.nodesource.com/setup_16.x | bash -
RUN apt-get install nodejs -y

# Install golang
RUN apt-get install golang -y
ENV GOCACHE /box
ENV GOTMPDIR /box


# Update packages
RUN apt-get clean -y
RUN apt-get autoclean -y
RUN apt-get autoremove -y

# Set default workdir
WORKDIR /box

构建镜像：

docker build . -t codesandbox:latest

运行后查看占用情况：

docker stats

Java操作Docker

<!-- https://mvnrepository.com/artifact/com.github.docker-java/docker-java -->
<dependency>
    <groupId>com.github.docker-java</groupId>
    <artifactId>docker-java</artifactId>
    <version>3.3.0</version>
</dependency>
<!-- https://mvnrepository.com/artifact/com.github.docker-java/docker-java-transport-httpclient5 -->
<dependency>
    <groupId>com.github.docker-java</groupId>
    <artifactId>docker-java-transport-httpclient5</artifactId>
    <version>3.3.0</version>
</dependency>

请求实体

@Data
public class ExecuteRequest {
    private String language;
    private String code;
}

执行结果实体

@Data
@AllArgsConstructor
@NoArgsConstructor
@Builder
public class ExecuteMessage {
    private boolean success;
    private String message;
    private String errorMessage;
}

可运行语言枚举

/**
 * 编程语言 cmd 枚举
 * 不需要编译的语言编译的 cmd 设置为空即可
 */
@Getter
public enum LanguageCmdEnum {
    JAVA("java", "Main.java", new String[]{"javac", "-encoding", "utf-8", "Main.java"}, new String[]{"java", "-Dfile.encoding=UTF-8", "Main"}),
    CPP("cpp", "main.cpp", new String[]{"g++", "-finput-charset=UTF-8", "-fexec-charset=UTF-8", "-o", "main", "main.cpp"}, new String[]{"./main"}),
    C("c", "main.c", new String[]{"gcc", "-finput-charset=UTF-8", "-fexec-charset=UTF-8", "-o", "main", "main.c"}, new String[]{"./main"}),
    PYTHON3("python", "main.py", null, new String[]{"python3", "main.py"}),
    JAVASCRIPT("javascript", "main.js", null, new String[]{"node", "main.js"}),
    TYPESCRIPT("typescript", "main.ts", null, new String[]{"node", "main.ts"}),
    GO("go", "main.go", null, new String[]{"go", "run", "main.go"}),
    ;

    private final String language;

    /**
     * 保存的文件名
     */
    private final String saveFileName;

    private final String[] compileCmd;

    private final String[] runCmd;


    LanguageCmdEnum(String language, String saveFileName, String[] compileCmd, String[] runCmd) {
        this.language = language;
        this.saveFileName = saveFileName;
        this.compileCmd = compileCmd;
        this.runCmd = runCmd;
    }

    /**
     * 根据 language 获取枚举
     *
     * @param language 值
     * @return {@link LanguageCmdEnum}
     */
    public static LanguageCmdEnum getEnumByValue(String language) {
        if (StringUtils.isBlank(language)) {
            return null;
        }
        for (LanguageCmdEnum languageCmdEnum : LanguageCmdEnum.values()) {
            if (languageCmdEnum.language.equals(language)) {
                return languageCmdEnum;
            }
        }
        return null;
    }
}

Docker沙箱执行

@Slf4j
@Data
@ConfigurationProperties(prefix = "codesandbox.config")
@Configuration
public class DockerSandbox {
    private static final DockerClient DOCKER_CLIENT = DockerClientBuilder.getInstance().build();

    /**
     * 代码沙箱的镜像，Dockerfile 构建的镜像名，默认为 codesandbox:latest
     */
    private String image = "codesandbox:latest";

    /**
     * 内存限制，单位为字节，默认为 1024 * 1024 * 60 MB
     */
    private long memoryLimit = 1024 * 1024 * 60;

    private long memorySwap = 0;

    /**
     * 最大可消耗的 cpu 数
     */
    private long cpuCount = 1;

    private long timeoutLimit = 1;

    private TimeUnit timeUnit = TimeUnit.SECONDS;

    /**
     * 执行代码
     *
     * @param languageCmdEnum 编程语言枚举
     * @param code            代码
     * @return {@link ExecuteMessage}
     */
    public ExecuteMessage execute(LanguageCmdEnum languageCmdEnum, String code) {

        // 写入文件
        String userDir = System.getProperty("user.dir");
        String language = languageCmdEnum.getLanguage();
        String globalCodePathName = userDir + File.separator + "tempCode" + File.separator + language;
        // 判断全局代码目录是否存在，没有则新建
        File globalCodePath = new File(globalCodePathName);
        if (!globalCodePath.exists()) {
            boolean mkdir = globalCodePath.mkdirs();
            if (!mkdir) {
                log.info("创建全局代码目录失败");
            }
        }

        // 把用户的代码隔离存放
        String userCodeParentPath = globalCodePathName + File.separator + UUID.randomUUID();
        String userCodePath = userCodeParentPath + File.separator + languageCmdEnum.getSaveFileName();
        FileUtil.writeString(code, userCodePath, StandardCharsets.UTF_8);
        String containerId = createContainer(userCodePath);

        // 编译代码
        String[] compileCmd = languageCmdEnum.getCompileCmd();
        ExecuteMessage executeMessage;

        // 不为空则代表需要编译
        if (compileCmd != null) {
            executeMessage = execCmd(containerId, compileCmd);

            log.info("编译完成...");
            // 编译错误
            if (!executeMessage.isSuccess()) {
                // 清理文件和容器
                cleanFileAndContainer(userCodeParentPath, containerId);
                return executeMessage;
            }
        }
        executeMessage = execCmd(containerId, languageCmdEnum.getRunCmd());
        log.info("运行完成...");

        // 清理文件和容器
        cleanFileAndContainer(userCodeParentPath, containerId);
        return executeMessage;
    }

    /**
     * 清理文件和容器
     *
     * @param userCodePath 用户代码路径
     * @param containerId  容器 ID
     */
    private static void cleanFileAndContainer(String userCodePath, String containerId) {
        // 清理临时目录
        FileUtil.del(userCodePath);

        // 关闭并删除容器
        DOCKER_CLIENT.stopContainerCmd(containerId).exec();
        DOCKER_CLIENT.removeContainerCmd(containerId).exec();
    }

    /**
     * 执行命令
     *
     * @param containerId 容器 ID
     * @param cmd         CMD
     * @return {@link ExecuteMessage}
     */
    private ExecuteMessage execCmd(String containerId, String[] cmd) {
        // 正常返回信息
        ByteArrayOutputStream resultStream = new ByteArrayOutputStream();
        // 错误信息
        ByteArrayOutputStream errorResultStream = new ByteArrayOutputStream();

        // 结果
        final boolean[] result = {true};
        try (ResultCallback.Adapter<Frame> frameAdapter = new ResultCallback.Adapter<Frame>() {
            @Override
            public void onNext(Frame frame) {
                StreamType streamType = frame.getStreamType();
                byte[] payload = frame.getPayload();
                if (StreamType.STDERR.equals(streamType)) {
                    try {
                        result[0] = false;
                        errorResultStream.write(payload);
                    } catch (IOException e) {
                        throw new RuntimeException(e);
                    }
                } else {
                    try {
                        result[0] = true;
                        resultStream.write(payload);
                    } catch (IOException e) {
                        throw new RuntimeException(e);
                    }
                }
                super.onNext(frame);
            }
        }) {
            ExecCreateCmdResponse execCompileCmdResponse = DOCKER_CLIENT.execCreateCmd(containerId)
                    .withCmd(cmd)
                    .withAttachStderr(true)
                    .withAttachStdin(true)
                    .withAttachStdout(true)
                    .exec();
            String execId = execCompileCmdResponse.getId();
            DOCKER_CLIENT.execStartCmd(execId).exec(frameAdapter).awaitCompletion(timeoutLimit, timeUnit);
            return ExecuteMessage
                    .builder()
                    .success(result[0])
                    .message(resultStream.toString())
                    .errorMessage(errorResultStream.toString())
                    .build();

        } catch (IOException | InterruptedException e) {
            log.info(e.getMessage());
            return ExecuteMessage
                    .builder()
                    .success(false)
                    .errorMessage(e.getMessage())
                    .build();
        }
    }

    /**
     * 创建容器
     *
     * @return {@link String}
     */
    private String createContainer(String codeFile) {
        CreateContainerCmd containerCmd = DOCKER_CLIENT.createContainerCmd(image);
        HostConfig hostConfig = new HostConfig();
        hostConfig.withMemory(memoryLimit);
        hostConfig.withMemorySwap(memorySwap);
        hostConfig.withCpuCount(cpuCount);

        CreateContainerResponse createContainerResponse = containerCmd
                .withHostConfig(hostConfig)
                .withNetworkDisabled(true)
                .withAttachStdin(true)
                .withAttachStderr(true)
                .withAttachStdout(true)
                .withTty(true)
                .exec();
        // 启动容器
        String containerId = createContainerResponse.getId();
        DOCKER_CLIENT.startContainerCmd(containerId).exec();

        // 将代码复制到容器中
        DOCKER_CLIENT.copyArchiveToContainerCmd(containerId)
                .withHostResource(codeFile)
                .withRemotePath("/box")
                .exec();
        return containerId;
    }
}

Spring中使用

在resources/META-INF/spring目录下，新建一个org.springframework.boot.autoconfigure.AutoConfiguation.imports文件

里面加载Bean：com.yunfei.codesandbox.executor.DockerSandbox

其他项目使用时添加配置：

codesandbox:
  config:
    cpu-count: 1
    image: codesandbox:latest
    memory-limit: 1024 * 1024 * 60
    memory-swap: 0
    time-unit: seconds
    timeout-limit: 1

容器池设计

创建一个容器池，保留一下可用的Docker容器，如果没有可用的容器，就需要等待，如果等待的过多，就需要扩容。

DockerDao

@Slf4j
@Data
@Configuration
@ConfigurationProperties(prefix = "codesandbox.config")
public class DockerDao {

    /**
     * 代码沙箱的镜像，Dockerfile 构建的镜像名，默认为 codesandbox:latest
     */
    private String image = "codesandbox:latest";

    /**
     * 内存限制，单位为字节，默认为 1024 * 1024 * 60 MB
     */
    private long memoryLimit = 1024 * 1024 * 60;

    private long memorySwap = 0;

    /**
     * 最大可消耗的 cpu 数
     */
    private long cpuCount = 1;

    private long timeoutLimit = 1;

    private TimeUnit timeUnit = TimeUnit.SECONDS;

    private static final DockerClient DOCKER_CLIENT = DockerClientBuilder.getInstance().build();

    /**
     * 执行命令
     *
     * @param containerId 容器 ID
     * @param cmd         CMD
     * @return {@link ExecuteMessage}
     */
    public ExecuteMessage execCmd(String containerId, String[] cmd) {
        // 正常返回信息
        ByteArrayOutputStream resultStream = new ByteArrayOutputStream();
        // 错误信息
        ByteArrayOutputStream errorResultStream = new ByteArrayOutputStream();

        // 结果
        final boolean[] result = {true};
        final boolean[] timeout = {true};
        try (ResultCallback.Adapter<Frame> frameAdapter = new ResultCallback.Adapter<Frame>() {

            @Override
            public void onComplete() {
                // 是否超时
                timeout[0] = false;
                super.onComplete();
            }

            @Override
            public void onNext(Frame frame) {
                StreamType streamType = frame.getStreamType();
                byte[] payload = frame.getPayload();
                if (StreamType.STDERR.equals(streamType)) {
                    try {
                        result[0] = false;
                        errorResultStream.write(payload);
                    } catch (IOException e) {
                        throw new RuntimeException(e);
                    }
                } else {
                    try {
                        resultStream.write(payload);
                    } catch (IOException e) {
                        throw new RuntimeException(e);
                    }
                }
                super.onNext(frame);
            }
        }) {
            ExecCreateCmdResponse execCompileCmdResponse = DOCKER_CLIENT.execCreateCmd(containerId)
                    .withCmd(cmd)
                    .withAttachStderr(true)
                    .withAttachStdin(true)
                    .withAttachStdout(true)
                    .exec();
            String execId = execCompileCmdResponse.getId();
            DOCKER_CLIENT.execStartCmd(execId).exec(frameAdapter).awaitCompletion(timeoutLimit, timeUnit);

            // 超时
            if (timeout[0]) {
                return ExecuteMessage
                        .builder()
                        .success(false)
                        .errorMessage("执行超时")
                        .build();
            }

            return ExecuteMessage
                    .builder()
                    .success(result[0])
                    .message(resultStream.toString())
                    .errorMessage(errorResultStream.toString())
                    .build();

        } catch (IOException | InterruptedException e) {
            return ExecuteMessage
                    .builder()
                    .success(false)
                    .errorMessage(e.getMessage())
                    .build();
        }
    }

    public ContainerInfo startContainer(String codePath) {
        CreateContainerCmd containerCmd = DOCKER_CLIENT.createContainerCmd(image);
        HostConfig hostConfig = new HostConfig();
        hostConfig.withMemory(memoryLimit);
        hostConfig.withMemorySwap(memorySwap);
        hostConfig.withCpuCount(cpuCount);
//        hostConfig.withReadonlyRootfs(true);
//        hostConfig.setBinds(new Bind(codePath, new Volume("/box")));
        CreateContainerResponse createContainerResponse = containerCmd
                .withHostConfig(hostConfig)
                .withNetworkDisabled(true)
                .withAttachStdin(true)
                .withAttachStderr(true)
                .withAttachStdout(true)
                .withTty(true)
                .exec();
        String containerId = createContainerResponse.getId();
        log.info("containerId: {}", containerId);
        // 启动容器
        DOCKER_CLIENT.startContainerCmd(containerId).exec();
        return ContainerInfo
                .builder()
                .containerId(containerId)
                .codePathName(codePath)
                .lastActivityTime(System.currentTimeMillis())
                .build();
    }

    /**
     * 复制文件到容器
     *
     * @param codeFile    代码文件
     * @param containerId 容器 ID
     */
    public void copyFileToContainer(String containerId, String codeFile) {
        DOCKER_CLIENT.copyArchiveToContainerCmd(containerId)
                .withHostResource(codeFile)
                .withRemotePath("/box")
                .exec();
    }

    public void cleanContainer(String containerId) {
        DOCKER_CLIENT.stopContainerCmd(containerId).exec();
        DOCKER_CLIENT.removeContainerCmd(containerId).exec();
    }
}

CotainerInfo

@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class ContainerInfo {
    private String containerId;

    private String codePathName;

    private long lastActivityTime;

    /**
     * 错误计数，默认为 0
     */
    private int errorCount = 0;
}

ContainerPoolExecutor

@Slf4j
@Configuration
@Data
@ConfigurationProperties(prefix = "codesandbox.pool")
public class ContainerPoolExecutor {

    private Integer corePoolSize = Runtime.getRuntime().availableProcessors() * 10;

    private Integer maximumPoolSize = Runtime.getRuntime().availableProcessors() * 20;

    private Integer waitQueueSize = 200;

    private Integer keepAliveTime = 5;

    private TimeUnit timeUnit = TimeUnit.SECONDS;


    /**
     * 容器池
     * key: 容器 id
     * value：上次活跃时间
     */
    private BlockingQueue<ContainerInfo> containerPool;

    /**
     * 容器使用排队计数
     */
    private AtomicInteger blockingThreadCount;

    /**
     * 可扩展的数量
     */
    private AtomicInteger expandCount;

    @Resource
    private DockerDao dockerDao;


    @PostConstruct
    public void initPool() {

        // 初始化容器池
        this.containerPool = new LinkedBlockingQueue<>(maximumPoolSize);
        this.blockingThreadCount = new AtomicInteger(0);
        this.expandCount = new AtomicInteger(maximumPoolSize - corePoolSize);

        // 初始化池中的数据
        for (int i = 0; i < corePoolSize; i++) {
            createNewPool();
        }

        // 定时清理过期的容器
        ScheduledExecutorService scheduledExecutorService = Executors.newScheduledThreadPool(1);
        scheduleExpirationCleanup(scheduledExecutorService);
    }

    private void createNewPool() {
        // 写入文件
        String userDir = System.getProperty("user.dir");
        String codePathName = userDir + File.separator + "tempCode";

        // 把用户的代码隔离存放
        UUID uuid = UUID.randomUUID();
        codePathName += File.separator + uuid;

        // 判断代码目录是否存在，没有则新建
        File codePath = new File(codePathName);
        if (!codePath.exists()) {
            boolean mkdir = codePath.mkdirs();
            if (!mkdir) {
                log.info("创建代码目录失败");
            }
        }
        ContainerInfo containerInfo = dockerDao.startContainer(codePathName);
        boolean result = containerPool.offer(containerInfo);
        if (!result) {
            log.error("current capacity: {}, the capacity limit is exceeded...", containerPool.size());
        }
    }

    private boolean expandPool() {
        log.info("超过指定数量，触发扩容");
        if (expandCount.decrementAndGet() < 0) {
            log.error("不能再扩容了");
            return false;
        }
        log.info("扩容了");
        createNewPool();
        return true;
    }


    private ContainerInfo getContainer() throws InterruptedException {
        if (containerPool.isEmpty()) {
            // 增加阻塞线程计数
            try {
                if (blockingThreadCount.incrementAndGet() >= waitQueueSize && !expandPool()) {
                    log.error("扩容失败");
                    return null;
                }
                log.info("没有数据，等待数据，当前等待长度：{}", blockingThreadCount.get());
                // 阻塞等待可用的数据
                return containerPool.take();
            } finally {
                // 减少阻塞线程计数
                log.info("减少阻塞线程计数");
                blockingThreadCount.decrementAndGet();
            }
        }
        return containerPool.take();
    }

    /**
     * 清理过期容器
     */
    private void cleanExpiredContainers() {
        long currentTime = System.currentTimeMillis();
        int needCleanCount = containerPool.size() - corePoolSize;
        if (needCleanCount <= 0) {
            return;
        }
        // 处理过期的容器
        containerPool.stream().filter(containerInfo -> {
            long lastActivityTime = containerInfo.getLastActivityTime();
            lastActivityTime += timeUnit.toMillis(keepAliveTime);
            return lastActivityTime < currentTime;
        }).forEach(containerInfo -> {
            boolean remove = containerPool.remove(containerInfo);
            if (remove) {
                String containerId = containerInfo.getContainerId();
                expandCount.incrementAndGet();
                if (StringUtils.isNotBlank(containerId)) {
                    dockerDao.cleanContainer(containerId);
                }
            }
        });
        log.info("当前容器大小: " + containerPool.size());
    }

    private void scheduleExpirationCleanup(ScheduledExecutorService scheduledExecutorService) {
        scheduledExecutorService.scheduleAtFixedRate(() -> {
            log.info("定时清理过期容器...");
            cleanExpiredContainers();
            // 每隔 20 秒执行一次清理操作
        }, 0, 20, TimeUnit.SECONDS);
    }


    private void recordError(ContainerInfo containerInfo) {
        if (containerInfo != null) {
            containerInfo.setErrorCount(containerInfo.getErrorCount() + 1);
        }
    }


    public ExecuteMessage run(Function<ContainerInfo, ExecuteMessage> function) {
        ContainerInfo containerInfo = null;
        try {
            containerInfo = getContainer();
            if (containerInfo == null) {
                return ExecuteMessage.builder().success(false).message("不能处理了").build();
            }

            log.info("有数据，拿到了: {}", containerInfo);
            ExecuteMessage executeMessage = function.apply(containerInfo);
            if (!executeMessage.isSuccess()) {
                recordError(containerInfo);
            }
            return executeMessage;
        } catch (InterruptedException e) {
            throw new RuntimeException(e);
        } finally {
            if (containerInfo != null) {
                ContainerInfo finalContainerInfo = containerInfo;
                dockerDao.execCmd(containerInfo.getContainerId(), new String[]{"rm", "-rf", "/box"});
                CompletableFuture.runAsync(() -> {
                    try {
                        // 更新时间
                        log.info("操作完了，还回去");
                        String codePathName = finalContainerInfo.getCodePathName();
                        FileUtil.del(codePathName);
                        // 错误超过 3 次就不放回，重新运行一个
                        if (finalContainerInfo.getErrorCount() > 3) {
                            CompletableFuture.runAsync(() -> {
                                dockerDao.cleanContainer(finalContainerInfo.getContainerId());
                                this.createNewPool();
                            });
                            return;
                        }
                        finalContainerInfo.setLastActivityTime(System.currentTimeMillis());
                        containerPool.put(finalContainerInfo);
                        log.info("容器池还剩: {}", containerPool.size());
                    } catch (InterruptedException e) {
                        log.error("无法放入");
                    }
                });
            }
        }
    }
}

DockerSandbox

@Service
@Slf4j
public class DockerSandbox {
    @Resource
    private DockerDao dockerDao;

    @Resource
    private ContainerPoolExecutor containerPoolExecutor;

    public ExecuteMessage execute(LanguageCmdEnum languageCmdEnum, String code) {
        return containerPoolExecutor.run(containerInfo -> {
            try {
                String containerId = containerInfo.getContainerId();

                String codePathName = containerInfo.getCodePathName();

                String codeFileName = codePathName + File.separator + languageCmdEnum.getSaveFileName();

                FileUtil.writeString(code, codeFileName, StandardCharsets.UTF_8);

                dockerDao.copyFileToContainer(containerId, codeFileName);

                // 编译代码
                String[] compileCmd = languageCmdEnum.getCompileCmd();
                ExecuteMessage executeMessage;

                // 不为空则代表需要编译
                if (compileCmd != null) {
                    executeMessage = dockerDao.execCmd(containerId, compileCmd);
                    log.info("compile complete...");
                    // 编译错误
                    if (!executeMessage.isSuccess()) {
                        return executeMessage;
                    }
                }
                String[] runCmd = languageCmdEnum.getRunCmd();
                executeMessage = dockerDao.execCmd(containerId, runCmd);
                log.info("run complete...");
                return executeMessage;
            } catch (Exception e) {
                return ExecuteMessage.builder()
                        .success(false)
                        .errorMessage(e.getMessage())
                        .build();
            }
        });
    }
}

优雅关闭

@Component
public class CleanContainerListener implements ApplicationListener<ContextClosedEvent> {

    @Resource
    private DockerDao dockerDao;

    @Resource
    private ContainerPoolExecutor containerPoolExecutor;

    @Override
    public void onApplicationEvent(ContextClosedEvent event) {
        // 清理所有的容器以及残余文件
        containerPoolExecutor
                .getContainerPool()
                .forEach(containerInfo -> {
                    FileUtil.del(containerInfo.getCodePathName());
                    dockerDao.cleanContainer(containerInfo.getContainerId());
                });
        System.out.println("container clean end...");
    }
}