Feat(flow): Add thread-flow crate, integrates ReCoco (CocoIndex) capabilities into Thread, providing dataflow driven ETL pipeline management

.github/workflows/release.yml

@@ -0,0 +1,368 @@
+# SPDX-FileCopyrightText: 2025 Knitli Inc. <knitli@knit.li>
+# SPDX-FileContributor: Adam Poulemanos <adam@knit.li>
+#
+# SPDX-License-Identifier: MIT OR Apache-2.0
+# ! GitHub Action for automated releases
+# ! Builds and publishes releases for multiple platforms
+name: Release
+
+on:
+    push:
+        tags:
+            - "v*.*.*"
+    workflow_dispatch:
+        inputs:
+            version:
+                description: "Version to release (e.g., 0.1.0)"
+                required: true
+                type: string
+
+env:
+    CARGO_TERM_COLOR: always
+    CARGO_INCREMENTAL: 0
+
+permissions:
+    contents: write
+    packages: write
+
+jobs:
+    # Create GitHub release
+    create-release:
+        name: Create Release
+        runs-on: ubuntu-latest
+        outputs:
+            upload_url: ${{ steps.create_release.outputs.upload_url }}
+            version: ${{ steps.get_version.outputs.version }}
+        steps:
+            - uses: actions/checkout@v4
+              with:
+                  fetch-depth: 0
+
+            - name: Get version
+              id: get_version
+              env:
+                  INPUT_VERSION: ${{ github.event.inputs.version }}
+                  REF_NAME: ${{ github.ref }}
+              run: |
+                  if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then
+                    VERSION="${INPUT_VERSION}"
+                  else
+                    VERSION="${REF_NAME#refs/tags/v}"
+                  fi
+                  echo "version=${VERSION}" >> "$GITHUB_OUTPUT"
+                  echo "Version: ${VERSION}"
+
+            - name: Generate changelog
+              id: changelog
+              env:
+                  VERSION: ${{ steps.get_version.outputs.version }}
+              run: |
+                  # Extract changelog for this version
+                  if [ -f "CHANGELOG.md" ]; then
+                      CHANGELOG="$(sed -n "/## \[${VERSION}\]/,/## \[/p" CHANGELOG.md | sed '$ d')"
+                  else
+                      CHANGELOG="Release ${VERSION}"
+                  fi
+                  echo "changelog<<EOF" >> "$GITHUB_OUTPUT"
+                  echo "${CHANGELOG}" >> "$GITHUB_OUTPUT"
+                  echo "EOF" >> "$GITHUB_OUTPUT"
+
+            - name: Create GitHub Release
+              id: create_release
+              uses: actions/create-release@v1
+              env:
+                  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+              with:
+                  tag_name: v${{ steps.get_version.outputs.version }}
+                  release_name: Release ${{ steps.get_version.outputs.version }}
+                  body: ${{ steps.changelog.outputs.changelog }}
+                  draft: false
+                  prerelease: false
+
+    # Build CLI binaries for multiple platforms
+    build-cli:
+        name: Build CLI (${{ matrix.target }})
+        needs: create-release
+        strategy:
+            fail-fast: false
+            matrix:
+                include:
+                    # Linux x86_64
+                    - target: x86_64-unknown-linux-gnu
+                      os: ubuntu-latest
+                      cross: false
+                      strip: true
+
+                    # Linux x86_64 (musl for static linking)
+                    - target: x86_64-unknown-linux-musl
+                      os: ubuntu-latest
+                      cross: true
+                      strip: true
+
+                    # Linux ARM64
+                    - target: aarch64-unknown-linux-gnu
+                      os: ubuntu-latest
+                      cross: true
+                      strip: false
+
+                    # macOS x86_64
+                    - target: x86_64-apple-darwin
+                      os: macos-latest
+                      cross: false
+                      strip: true
+
+                    # macOS ARM64 (Apple Silicon)
+                    - target: aarch64-apple-darwin
+                      os: macos-latest
+                      cross: false
+                      strip: true
+
+                    # Windows x86_64
+                    - target: x86_64-pc-windows-msvc
+                      os: windows-latest
+                      cross: false
+                      strip: false
+                      ext: .exe
+
+        runs-on: ${{ matrix.os }}
+        steps:
+            - uses: actions/checkout@v4
+              with:
+                  submodules: recursive
+
+            - name: Install Rust
+              uses: dtolnay/rust-toolchain@stable
+              with:
+                  targets: ${{ matrix.target }}
+
+            - name: Cache Rust dependencies
+              uses: Swatinem/rust-cache@v2
+              with:
+                  key: ${{ matrix.target }}
+
+            - name: Install cross (if needed)
+              if: matrix.cross
+              run: cargo install cross --git https://github.com/cross-rs/cross
+
+            - name: Build release binary
+              env:
+                  TARGET: ${{ matrix.target }}
+                  USE_CROSS: ${{ matrix.cross }}
+              run: |
+                  if [ "${USE_CROSS}" == "true" ]; then
+                      cross build --release --target "${TARGET}" --features parallel,caching
+                  else
+                      cargo build --release --target "${TARGET}" --features parallel,caching
+                  fi
+              shell: bash
+
+            - name: Strip binary (if applicable)
+              if: matrix.strip
+              env:
+                  TARGET: ${{ matrix.target }}
+                  EXT: ${{ matrix.ext }}
+              run: |
+                  strip "target/${TARGET}/release/thread${EXT}"
+              shell: bash
+
+            - name: Create archive
+              id: archive
+              env:
+                  VERSION: ${{ needs.create-release.outputs.version }}
+                  TARGET: ${{ matrix.target }}
+                  OS_TYPE: ${{ matrix.os }}
+              run: |
+                  ARCHIVE_NAME="thread-${VERSION}-${TARGET}"
+                  if [ "${OS_TYPE}" == "windows-latest" ]; then
+                      7z a "${ARCHIVE_NAME}.zip" "./target/${TARGET}/release/thread.exe"
+                      echo "asset_path=${ARCHIVE_NAME}.zip" >> "$GITHUB_OUTPUT"
+                      echo "asset_content_type=application/zip" >> "$GITHUB_OUTPUT"
+                  else
+                      tar czf "${ARCHIVE_NAME}.tar.gz" -C "target/${TARGET}/release" thread
+                      echo "asset_path=${ARCHIVE_NAME}.tar.gz" >> "$GITHUB_OUTPUT"
+                      echo "asset_content_type=application/gzip" >> "$GITHUB_OUTPUT"
+                  fi
+                  echo "asset_name=${ARCHIVE_NAME}" >> "$GITHUB_OUTPUT"
+              shell: bash
+
+            - name: Upload release asset
+              uses: actions/upload-release-asset@v1
+              env:
+                  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+              with:
+                  upload_url: ${{ needs.create-release.outputs.upload_url }}
+                  asset_path: ${{ steps.archive.outputs.asset_path }}
+                  asset_name: ${{ steps.archive.outputs.asset_name }}${{ matrix.os == 'windows-latest' && '.zip' || '.tar.gz' }}
+                  asset_content_type: ${{ steps.archive.outputs.asset_content_type }}
+
+    # Build and publish WASM package
+    build-wasm:
+        name: Build & Publish WASM
+        needs: create-release
+        runs-on: ubuntu-latest
+        steps:
+            - uses: actions/checkout@v4
+              with:
+                  submodules: recursive
+
+            - name: Install Rust
+              uses: dtolnay/rust-toolchain@stable
+              with:
+                  targets: wasm32-unknown-unknown
+
+            - name: Cache Rust dependencies
+              uses: Swatinem/rust-cache@v2
+
+            - name: Install wasm-pack
+              uses: jetli/wasm-pack-action@v0.4.0
+
+            - name: Build WASM package
+              run: cargo run -p xtask build-wasm --release
+
+            - name: Create WASM archive
+              env:
+                  VERSION: ${{ needs.create-release.outputs.version }}
+              run: |
+                  ARCHIVE_NAME="thread-wasm-${VERSION}"
+                  tar czf "${ARCHIVE_NAME}.tar.gz" \
+                      thread_wasm_bg.wasm \
+                      thread_wasm.js \
+                      thread_wasm.d.ts \
+                      package.json
+
+            - name: Upload WASM archive
+              uses: actions/upload-release-asset@v1
+              env:
+                  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+                  VERSION: ${{ needs.create-release.outputs.version }}
+              with:
+                  upload_url: ${{ needs.create-release.outputs.upload_url }}
+                  asset_path: thread-wasm-${{ needs.create-release.outputs.version }}.tar.gz
+                  asset_name: thread-wasm-${{ needs.create-release.outputs.version }}.tar.gz
+                  asset_content_type: application/gzip
+
+    # Build Docker images
+    build-docker:
+        name: Build Docker Images
+        needs: create-release
+        runs-on: ubuntu-latest
+        steps:
+            - uses: actions/checkout@v4
+              with:
+                  submodules: recursive
+
+            - name: Set up Docker Buildx
+              uses: docker/setup-buildx-action@v3
+
+            - name: Login to GitHub Container Registry
+              uses: docker/login-action@v3
+              with:
+                  registry: ghcr.io
+                  username: ${{ github.actor }}
+                  password: ${{ secrets.GITHUB_TOKEN }}
+
+            - name: Build metadata
+              id: meta
+              uses: docker/metadata-action@v5
+              with:
+                  images: ghcr.io/${{ github.repository }}
+                  tags: |
+                      type=semver,pattern={{version}},value=v${{ needs.create-release.outputs.version }}
+                      type=semver,pattern={{major}}.{{minor}},value=v${{ needs.create-release.outputs.version }}
+                      type=semver,pattern={{major}},value=v${{ needs.create-release.outputs.version }}
+                      type=raw,value=latest
+
+            - name: Build and push
+              uses: docker/build-push-action@v5
+              with:
+                  context: .
+                  platforms: linux/amd64,linux/arm64
+                  push: true
+                  tags: ${{ steps.meta.outputs.tags }}
+                  labels: ${{ steps.meta.outputs.labels }}
+                  cache-from: type=gha
+                  cache-to: type=gha,mode=max
+
+    # Publish to crates.io (optional, requires CARGO_REGISTRY_TOKEN)
+    publish-crates:
+        name: Publish to crates.io
+        needs: [create-release, build-cli]
+        runs-on: ubuntu-latest
+        if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
+        steps:
+            - uses: actions/checkout@v4
+              with:
+                  submodules: recursive
+
+            - name: Install Rust
+              uses: dtolnay/rust-toolchain@stable
+
+            - name: Cache Rust dependencies
+              uses: Swatinem/rust-cache@v2
+
+            - name: Publish to crates.io
+              env:
+                  CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }}
+              run: |
+                  # Publish in dependency order
+                  cargo publish -p thread-utils --allow-dirty || echo "Package already published"
+                  cargo publish -p thread-language --allow-dirty || echo "Package already published"
+                  cargo publish -p thread-ast-engine --allow-dirty || echo "Package already published"
+                  cargo publish -p thread-rule-engine --allow-dirty || echo "Package already published"
+                  cargo publish -p thread-services --allow-dirty || echo "Package already published"
+                  cargo publish -p thread-flow --allow-dirty || echo "Package already published"
+                  cargo publish -p thread-wasm --allow-dirty || echo "Package already published"
+
+    # Deploy to Cloudflare Workers (Edge deployment)
+    deploy-edge:
+        name: Deploy to Cloudflare Edge
+        needs: [create-release, build-wasm]
+        runs-on: ubuntu-latest
+        if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
+        environment:
+            name: production-edge
+            url: https://thread.knit.li
+        steps:
+            - uses: actions/checkout@v4
+              with:
+                  submodules: recursive
+
+            - name: Install Rust
+              uses: dtolnay/rust-toolchain@stable
+              with:
+                  targets: wasm32-unknown-unknown
+
+            - name: Cache Rust dependencies
+              uses: Swatinem/rust-cache@v2
+
+            - name: Install wasm-pack
+              uses: jetli/wasm-pack-action@v0.4.0
+
+            - name: Build WASM for Workers
+              run: cargo run -p xtask build-wasm --release
+
+            - name: Deploy to Cloudflare Workers
+              uses: cloudflare/wrangler-action@v3
+              with:
+                  apiToken: ${{ secrets.CLOUDFLARE_API_TOKEN }}
+                  accountId: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }}
+                  command: deploy --env production
+
+    # Release notification
+    notify:
+        name: Release Notification
+        needs: [create-release, build-cli, build-wasm, build-docker]
+        runs-on: ubuntu-latest
+        if: always()
+        steps:
+            - name: Check release status
+              env:
+                  VERSION: ${{ needs.create-release.outputs.version }}
+                  CLI_RESULT: ${{ needs.build-cli.result }}
+                  WASM_RESULT: ${{ needs.build-wasm.result }}
+                  DOCKER_RESULT: ${{ needs.build-docker.result }}
+              run: |
+                  echo "Release v${VERSION} completed"
+                  echo "CLI builds: ${CLI_RESULT}"
+                  echo "WASM build: ${WASM_RESULT}"
+                  echo "Docker build: ${DOCKER_RESULT}"