diff --git a/.github/.gitignore b/.github/.gitignore new file mode 100644 index 0000000..4f509e5 --- /dev/null +++ b/.github/.gitignore @@ -0,0 +1 @@ +*.env \ No newline at end of file diff --git a/.github/.secrets.env.template b/.github/.secrets.env.template new file mode 100644 index 0000000..955573e --- /dev/null +++ b/.github/.secrets.env.template @@ -0,0 +1,4 @@ +# This is an example environment variable file for GitHub Actions. You can copy this file to .github/.secrets.env and fill in the values to override the default registry and GitHub token used in the CI workflow. This is useful for testing with a private registry or using a different GitHub account for authentication. +OVERRIDE_REGISTRY= +OVERRIDE_GITHUB_TOKEN= +GITHUB_USERNAME= \ No newline at end of file diff --git a/.github/actions/setup-ci-metadata/action.yaml b/.github/actions/setup-ci-metadata/action.yaml new file mode 100644 index 0000000..daaa267 --- /dev/null +++ b/.github/actions/setup-ci-metadata/action.yaml @@ -0,0 +1,70 @@ +name: 'Setup CI metadata' +description: 'Composite action to derive the registry and CI image tag for the current repository.' +inputs: + registry: + description: 'Container registry derived from the current GitHub server URL' + required: false + default: '' + repository: + description: 'GitHub repository in the format owner/repo' + required: false + default: ${{ github.repository }} + image_tag: + description: 'Tag for the CI image' + required: false + default: 'latest' +outputs: + registry: + description: 'Container registry derived from the current GitHub server URL' + value: ${{ steps.setup.outputs.registry }} + image_tag: + description: 'Fully qualified CI image tag' + value: ${{ steps.setup.outputs.image_tag }} + latest_tag: + description: 'Fully qualified latest CI image tag' + value: ${{ steps.setup.outputs.latest_tag }} +runs: + using: 'composite' + steps: + - name: Setup Dynamic Metadata + id: setup + shell: bash + run: | + # Extract the domain from server_url, handling both https:// and ssh:// schemes + SERVER_URL="${{ github.server_url }}" + + if [[ "$SERVER_URL" =~ ^ssh:// ]]; then + # For SSH URLs like ssh://git@host:port/path, extract just the hostname + SERVER_DOMAIN=$(echo "$SERVER_URL" | sed -e 's|^ssh://||' -e 's|^[^@]*@||' -e 's|:[0-9]*.*||') + else + # For HTTPS URLs, extract domain without scheme + SERVER_DOMAIN=$(echo "$SERVER_URL" | sed -e 's|^[^/]*//||' -e 's|/.*$||') + fi + + echo "Extracted server domain: $SERVER_DOMAIN" + + if [[ -n "${{ inputs.registry }}" ]]; then + REGISTRY="${{ inputs.registry }}" + elif [[ "$SERVER_DOMAIN" == "github.com" ]]; then + REGISTRY="ghcr.io" + else + REGISTRY="$SERVER_DOMAIN" + fi + + # Extract owner/repo from github.repository, handling SSH URLs + REPO="${{ inputs.repository }}" + if [[ "$REPO" =~ ^ssh:// ]] || [[ "$REPO" =~ ^https:// ]]; then + # Extract owner/repo from URLs like ssh://git@host/owner/repo.git or https://host/owner/repo.git + REPO=$(echo "$REPO" | sed -e 's|^[^/]*/||' -e 's|\.git$||' | rev | cut -d'/' -f1,2 | rev) + fi + + # Docker image names must be lowercase + REGISTRY="${REGISTRY,,}" + REPO="${REPO,,}" + + IMAGE_TAG="${REGISTRY}/${REPO}/ci:${{ inputs.image_tag }}" + LATEST_TAG="${REGISTRY}/${REPO}/ci:latest" + + echo "registry=$REGISTRY" >> "$GITHUB_OUTPUT" + echo "image_tag=$IMAGE_TAG" >> "$GITHUB_OUTPUT" + echo "latest_tag=$LATEST_TAG" >> "$GITHUB_OUTPUT" \ No newline at end of file diff --git a/.github/actions/setup-rust/action.yaml b/.github/actions/setup-rust/action.yaml index 1ad6fd3..deab658 100644 --- a/.github/actions/setup-rust/action.yaml +++ b/.github/actions/setup-rust/action.yaml @@ -13,6 +13,10 @@ inputs: description: 'Comma-separated list of additional rust components to install' required: false default: 'clippy, rustfmt' + skip_cache: + description: 'Whether to skip restoring and uploading caches (useful for testing the workflow without cache interference)' + required: false + default: 'false' runs: using: 'composite' steps: @@ -23,6 +27,7 @@ runs: - name: Cache cargo registry uses: actions/cache@v4 + if: inputs.skip_cache != 'true' with: path: ~/.cargo/registry key: ${{ runner.os }}-cargo-registry-${{ hashFiles('**/Cargo.lock') }} @@ -31,6 +36,7 @@ runs: - name: Cache cargo index uses: actions/cache@v4 + if: inputs.skip_cache != 'true' with: path: ~/.cargo/index key: ${{ runner.os }}-cargo-index-${{ hashFiles('**/Cargo.lock') }} @@ -42,7 +48,8 @@ runs: run: echo "SANITIZED_COMPONENTS=${{ inputs.components }}" | sed -E 's/, ?| /-/g' >> $GITHUB_ENV - name: Cache Rust toolchain - uses: actions/cache@v4 + uses: actions/cache@v3 + if: inputs.skip_cache != 'true' with: path: ~/.rustup # Key includes the OS and the toolchain version (e.g., 'stable') @@ -51,7 +58,8 @@ runs: ${{ runner.os }}-rustup- - name: Cache cargo build (target) - uses: actions/cache@v4 + uses: actions/cache@v3 + if: inputs.skip_cache != 'true' with: path: target key: ${{ runner.os }}-cargo-build-${{ hashFiles('**/Cargo.lock') }} @@ -64,8 +72,3 @@ runs: toolchain: ${{ inputs.toolchain }} override: ${{ inputs.override }} components: ${{ inputs.components }} - - - name: install protobuf compiler - run: | - apt-get update - apt-get install -y protobuf-compiler diff --git a/.github/docker/ci.Dockerfile b/.github/docker/ci.Dockerfile new file mode 100644 index 0000000..e7420fe --- /dev/null +++ b/.github/docker/ci.Dockerfile @@ -0,0 +1,31 @@ +FROM node:24-bookworm-slim + +# Install necessary dependencies for building Rust projects and running tests +RUN apt-get update && apt-get install -y \ + curl \ + git \ + zstd \ + build-essential \ + pkg-config \ + libssl-dev \ + gnupg \ + unzip \ + tar \ + && rm -rf /var/lib/apt/lists/* + +RUN apt-get update && apt-get install -y \ + postgresql-client \ + protobuf-compiler \ + && rm -rf /var/lib/apt/lists/* + +# install bun +RUN curl -fsSL https://bun.sh/install | bash +ENV PATH="/root/.bun/bin:${PATH}" + +# install rust and cargo +RUN apt-get update && apt-get install -y curl build-essential +RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y +ENV PATH="/root/.cargo/bin:${PATH}" + +# Set the working directory +WORKDIR /app diff --git a/.github/workflows/build-ci.yaml b/.github/workflows/build-ci.yaml new file mode 100644 index 0000000..00f30c8 --- /dev/null +++ b/.github/workflows/build-ci.yaml @@ -0,0 +1,54 @@ +name: Build CI Environment + +on: + workflow_dispatch: + inputs: + image_tag: + description: 'Tag for the CI image (e.g., latest)' + required: true + default: 'latest' + +env: + # OVERRIDE_REGISTRY can be set as a secret to override the default registry (e.g., for testing with a private registry). Else '' will be used, which defaults to ghcr.io for github.com and the GitHub server domain for self-hosted GitHub instances. + OVERRIDE_REGISTRY: ${{ secrets.OVERRIDE_REGISTRY }} + +permissions: + contents: read + packages: write + +concurrency: + group: build-ci + cancel-in-progress: true + +jobs: + build-ci-image: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Setup CI metadata + id: setup + uses: ./.github/actions/setup-ci-metadata + with: + registry: ${{ env.OVERRIDE_REGISTRY }} + image_tag: ${{ github.event.inputs.image_tag }} + + - name: Login to Docker Hub + uses: docker/login-action@v4 + with: + registry: ${{ steps.setup.outputs.registry }} + username: ${{ secrets.GITHUB_USERNAME || github.actor }} + password: ${{ secrets.OVERRIDE_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} + + - name: Build and push Docker image for CI + uses: docker/build-push-action@v3 + with: + context: . + file: .github/docker/ci.Dockerfile + push: true + tags: | + ${{ steps.setup.outputs.image_tag }} + ${{ steps.setup.outputs.latest_tag }} diff --git a/.github/workflows/verify.yaml b/.github/workflows/verify.yaml new file mode 100644 index 0000000..49bd73f --- /dev/null +++ b/.github/workflows/verify.yaml @@ -0,0 +1,139 @@ +# this workflow verifies the generated code is up to date and valid + +name: Verify +on: + pull_request: + branches: + - master + push: + branches: + - master + +env: + # OVERRIDE_REGISTRY can be set as a secret to override the default registry (e.g., for testing with a private registry). Else '' will be used, which defaults to ghcr.io for github.com and the GitHub server domain for self-hosted GitHub instances. + OVERRIDE_REGISTRY: ${{ secrets.OVERRIDE_REGISTRY }} + ACTIONS_STEP_DEBUG: true + +jobs: + get-ci-image: + runs-on: ubuntu-latest + outputs: + image_tag: ${{ steps.setup.outputs.image_tag }} + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Setup CI metadata + id: setup + uses: ./.github/actions/setup-ci-metadata + with: + registry: ${{ secrets.OVERRIDE_REGISTRY }} + image_tag: latest + + verify-generated-db-entities: + runs-on: ubuntu-latest + needs: + - get-ci-image + container: + image: ${{ needs.get-ci-image.outputs.image_tag }} + services: + postgres: + image: postgres:16 + env: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: nxmesh + # ! do not set a fixed port to avoid conflicts when running multiple jobs in parallel, use Docker's internal networking instead + # ports: + # - 5432:5432 + options: >- + --health-cmd "pg_isready -U postgres -d nxmesh" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + + env: + DATABASE_URL: postgres://postgres:postgres@postgres:5432/nxmesh + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Check whether migrations/entities changed + id: check_changes + shell: bash + run: | + if [ "${{ github.event_name }}" = "pull_request" ]; then + BASE_SHA=${{ github.event.pull_request.base.sha }} + HEAD_SHA=${{ github.event.pull_request.head.sha }} + else + BASE_SHA=${{ github.event.before }} + HEAD_SHA=${{ github.sha }} + fi + + if [ -z "$HEAD_SHA" ]; then + HEAD_SHA=$(git rev-parse --verify HEAD 2>/dev/null || echo "") + fi + + if [ -z "$BASE_SHA" ]; then + PREV=$(git rev-parse --verify "${HEAD_SHA}^" 2>/dev/null || true) + if [ -n "$PREV" ]; then + BASE_SHA=$PREV + else + BASE_SHA=$HEAD_SHA + fi + fi + + echo "Comparing $BASE_SHA..$HEAD_SHA" + CHANGED_FILES=$(git diff --name-only "$BASE_SHA" "$HEAD_SHA" || true) + echo "$CHANGED_FILES" + + echo "$CHANGED_FILES" | grep -E '^(crates/migration/src/|apps/nxmesh-master/src/db/entities/)' >/dev/null 2>&1 \ + && echo "changed=true" >> $GITHUB_OUTPUT \ + || echo "changed=true" >> $GITHUB_OUTPUT + # || echo "changed=false" >> $GITHUB_OUTPUT + + - name: Setup Rust, checkout and restore caches + if: steps.check_changes.outputs.changed == 'true' + uses: ./.github/actions/setup-rust + with: + skip_cache: ${{ vars.SKIP_CACHE }} + + - name: Install SeaORM CLI + if: steps.check_changes.outputs.changed == 'true' + run: | + cargo install sea-orm-cli@^2.0.0-rc --features "sqlx-postgres runtime-tokio-rustls" + + - name: Apply migrations + if: steps.check_changes.outputs.changed == 'true' + run: | + cargo run -p nxmesh-migration -- up + + - name: Regenerate entities + if: steps.check_changes.outputs.changed == 'true' + run: | + sea-orm-cli generate entity \ + --database-url "$DATABASE_URL" \ + --output-dir apps/nxmesh-master/src/db/entities \ + --with-serde both \ + --with-copy-enums \ + --date-time-crate chrono + + - name: Check for uncommitted changes in entities + if: steps.check_changes.outputs.changed == 'true' + shell: bash + run: | + if [[ -n $(git status --porcelain --untracked-files=all | grep 'apps/nxmesh-master/src/db/entities/') ]]; then + echo "Generated SeaORM entities are not up to date." + echo "Run 'just db-generate' after applying migrations and commit the result." + git status --porcelain --untracked-files=all | grep 'apps/nxmesh-master/src/db/entities/' + exit 1 + else + echo "Generated SeaORM entities are up to date." + fi + + - name: Skip entity generation (no relevant changes) + if: steps.check_changes.outputs.changed == 'false' + run: echo "No changes in migrations/entities, skipping SeaORM entity verification." diff --git a/.gitignore b/.gitignore index bdc6b78..b627e2e 100644 --- a/.gitignore +++ b/.gitignore @@ -68,6 +68,7 @@ web_modules/ # dotenv environment variable files .env .env.* +*.env !.env.example # parcel-bundler cache (https://parceljs.org/) diff --git a/justfile b/justfile index dcfcf2c..3e636b7 100644 --- a/justfile +++ b/justfile @@ -25,7 +25,6 @@ setup-rust-tools: cargo install sea-orm-cli@^2.0.0-rc --features "sqlx-postgres runtime-tokio-rustls" cargo install cargo-watch - # Setup frontend dependencies setup-frontend: @echo "📦 Installing frontend dependencies..." @@ -35,6 +34,12 @@ setup-frontend: # Development Commands # ============================================================================= +# act +act *ARGS: + # run act with custom secret-file + @echo "🎬 Running act with custom secrets file..." + act --env-file .github/.env --secret-file .github/.secrets.env --var-file .github/.var.env --network host {{ ARGS }} + # Start all services for development dev: @echo "🚀 Starting all development services..." @@ -45,11 +50,11 @@ dev: # Start Rust backend with hot reload dev-master *ARGS: @echo "🔧 Starting Rust backend..." - cargo watch -w apps/nxmesh-master -x 'run --bin nxmesh-master -- {{ARGS}}' + cargo watch -w apps/nxmesh-master -x 'run --bin nxmesh-master -- {{ ARGS }}' dev-agent *ARGS: @echo "🔧 Starting Rust agent..." - cargo watch -w apps/nxmesh-agent -x 'run --bin nxmesh-agent -- {{ARGS}}' + cargo watch -w apps/nxmesh-agent -x 'run --bin nxmesh-agent -- {{ ARGS }}' # Start Vite frontend development server dev-frontend: @@ -89,7 +94,7 @@ build-frontend: # ============================================================================= db *ARGS: - cd crates && sea-orm-cli {{ARGS}} + cd crates && sea-orm-cli {{ ARGS }} # Setup database db-setup: @@ -205,6 +210,11 @@ docker-run: @echo "🐳 Running Docker container..." docker run -p 8080:8080 --env-file .env nxmesh:latest +# Build Docker image for CI +docker-build-ci REGISTRY="ghcr.io/nxmesh": + @echo "🐳 Building Docker image for CI..." + docker build -t {{ REGISTRY }}/ci:latest -f ./.github/docker/ci.Dockerfile . + # ============================================================================= # Nginx Commands (Shared PID Namespace + Docker Fallback) # =============================================================================