rublon-ssh/build_all.py

import os
import subprocess
from concurrent.futures import ThreadPoolExecutor, as_completed
from pydoc import doc

# Configuration
ROOT_DIR = "./os"
MAX_WORKERS = 8  # Maximum parallel jobs for build docker image and configure project
MAX_WORKERS_COMPILE = 2 # number of compilation jobs t run

def find_dockerfiles(root_dir):
    """Recursively find all Dockerfiles in the given directory."""
    dockerfiles = []
    for dirpath, _, filenames in os.walk(root_dir):
        for filename in filenames:
            if filename == "Dockerfile":
                dockerfiles.append(os.path.join(dirpath, filename))
    return dockerfiles


def tags_from_path(dockerfile):
    dir_path = os.path.dirname(dockerfile)
    tag = dir_path.replace(ROOT_DIR + "/", "").replace(os.path.sep, "_")
    return dir_path, tag


def build_docker_image(dockerfile):
    """Build a Docker image for the given Dockerfile."""
    dir_path, tag = tags_from_path(dockerfile)
    print(f"Starting build image step for {tag} (directory: {dir_path})...")
    try:
        log_file = f"_tmp/logs/{tag}/build_docker_image.log"
        os.makedirs(os.path.dirname(log_file), exist_ok=True)
        with open(log_file, "w") as log:
            result = subprocess.run(
                ["docker", "build", "-t", tag, dir_path],
                stdout=log,
                stderr=subprocess.STDOUT,
                check=True
            )
        return f"Successfully built {tag}"
    except subprocess.CalledProcessError:
        print(f"Failed to build {tag}. Check {log_file} for details.")
        return f"Failed to build {tag}"


in_image_source_dir = "/home/rublon-ssh"
in_image_build_dir = "/home/build"

def configure_rublon(dockerfile):
    """Build a Docker image for the given Dockerfile."""
    dir_path, tag = tags_from_path(dockerfile)
    print(f"Starting configure step for {tag} (directory: {dir_path})...")
    try:
        log_file = f"_tmp/logs/{tag}/configure_rublon_image.log"
        build_dir = f"_tmp/builds/{tag}"
        os.makedirs(os.path.dirname(log_file), exist_ok=True)
        os.makedirs(os.path.dirname(build_dir), exist_ok=True)
        with open(log_file, "w") as log:
            result = subprocess.run(
                # $tag /bin/bash /home/rublon-ssh/configure.sh $tag
                ["docker", "run",
                        "-v", f"./:{in_image_source_dir}",
                        "-v", f"./{build_dir}:{in_image_build_dir}",
                        tag,
                        f"{in_image_source_dir}/configure.sh", in_image_source_dir, in_image_build_dir
                ],
                stdout=log,
                stderr=subprocess.STDOUT,
                check=True
            )
        return f"Successfully built {tag}"
    except subprocess.CalledProcessError:
        print(f"Failed to build {tag}. Check {log_file} for details.")
        return f"Failed to build {tag}"


def build_rublon(dockerfile):
    """Build a Docker image for the given Dockerfile."""
    dir_path, tag = tags_from_path(dockerfile)
    print(f"Starting build for {tag} (directory: {dir_path})...")
    try:
        log_file = f"_tmp/logs/{tag}/compile_rublon_image.log"
        build_dir = f"_tmp/builds/{tag}"
        packages_dir = "_tmp/packages/"
        os.makedirs(os.path.dirname(log_file), exist_ok=True)
        os.makedirs(os.path.dirname(build_dir), exist_ok=True)
        os.makedirs(os.path.dirname(packages_dir), exist_ok=True)
        with open(log_file, "w") as log:
            result = subprocess.run(
                # $tag /bin/bash /home/rublon-ssh/configure.sh $tag
                ["docker", "run",
                        "-v", f"./:{in_image_source_dir}",
                        "-v", f"./{build_dir}:{in_image_build_dir}",
                        tag,
                        f"{in_image_source_dir}/build.sh", in_image_source_dir, in_image_build_dir, tag
                ],
                stdout=log,
                stderr=subprocess.STDOUT,
                check=True
            )
        return f"Successfully built {tag}"
    except subprocess.CalledProcessError:
        print(f"Failed to build {tag}. Check {log_file} for details.")
        return f"Failed to build {tag}"


def run_parralel(num, job, name, dockerfiles):
    # Use ThreadPoolExecutor for parallel builds
    with ThreadPoolExecutor(max_workers=num) as executor:
        future_to_dockerfile = {executor.submit(job, df): df for df in dockerfiles}
        for future in as_completed(future_to_dockerfile):
            dockerfile = future_to_dockerfile[future]
            try:
                result = future.result()
                print(result)
            except Exception as e:
                print(f"Job {name} failed for {dockerfile}: {e}")


def main():
    # Find all Dockerfiles
    dockerfiles = find_dockerfiles(ROOT_DIR)
    if not dockerfiles:
        print("No Dockerfiles found.")
        return

    run_parralel(MAX_WORKERS, build_docker_image, "Docker Image Build", dockerfiles)
    run_parralel(MAX_WORKERS, configure_rublon, "Configuring project", dockerfiles)
    run_parralel(MAX_WORKERS_COMPILE, build_rublon, "Building project", dockerfiles)


    print("All Docker builds completed.")

if __name__ == "__main__":
    main()