Feat deploy zlogs with vm

backend/cmd/root.go

@@ -211,6 +211,10 @@ func addFlags() error {
 		return fmt.Errorf("failed to bind telemetry.otlp_endpoint flag: %w", err)
 	}
 
+	// === Zlog Output URL ===
+	if err := bindStringFlag(rootCmd, "zlog_output_url", "", "Zlog output URL for VM log streaming"); err != nil {
+		return fmt.Errorf("failed to bind zlog_output_url flag: %w", err)
+	}
 	return nil
 }
 

backend/config-example.json

@@ -71,7 +71,5 @@
       "host": ""
     }
   },
-  "telemetry": {
-    "otlp_endpoint": "jaeger:4317"
-  }
+  "zlog_output_url": "http://loki-address?tenant_id=zlog-vms"
 }

backend/go.mod

@@ -67,6 +67,7 @@ require (
 	github.com/pierrec/lz4/v4 v4.1.21 // indirect
 	github.com/segmentio/asm v1.2.0 // indirect
 	github.com/shopspring/decimal v1.4.0 // indirect
+	github.com/threefoldtech/zosbase v1.0.7-0.20251223152017-727e292afc8a // indirect
 	go.opentelemetry.io/proto/otlp v1.9.0 // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20251202230838-ff82c1b0f217 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217 // indirect
@@ -172,7 +173,6 @@ require (
 	github.com/stretchr/objx v0.5.2 // indirect
 	github.com/subosito/gotenv v1.6.0 // indirect
 	github.com/threefoldtech/tfgrid-sdk-go/rmb-sdk-go v0.17.5 // indirect
-	github.com/threefoldtech/zosbase v1.0.4 // indirect
 	github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
 	github.com/ugorji/go/codec v1.3.1 // indirect
 	github.com/vedhavyas/go-subkey/v2 v2.0.0 // indirect

backend/go.sum

@@ -371,8 +371,8 @@ github.com/threefoldtech/tfgrid-sdk-go/grid-proxy v0.17.6-0.20251209150615-a3bb9
 github.com/threefoldtech/tfgrid-sdk-go/grid-proxy v0.17.6-0.20251209150615-a3bb942f9860/go.mod h1:J57xHAagUOddwz2nonrkB91/8T9TJ9IKk/6wzKVB5EM=
 github.com/threefoldtech/tfgrid-sdk-go/rmb-sdk-go v0.17.5 h1:zp5iZOvtvcQrcR7Po3UZBNk2uBYi1i1VxMA/ENIvCZY=
 github.com/threefoldtech/tfgrid-sdk-go/rmb-sdk-go v0.17.5/go.mod h1:T+PZydVl3fxywqoUhCmzs+hUarfE1q9IMRl9xa+GIYo=
-github.com/threefoldtech/zosbase v1.0.4 h1:A4kFukh4IO5r5e9F51aqKgXOyTG5cWknxqEVGtQ647w=
-github.com/threefoldtech/zosbase v1.0.4/go.mod h1:ZZ1M8SZVr7k4tH2URr5DMEbcwZoQDpBZWgboHdiNE+k=
+github.com/threefoldtech/zosbase v1.0.7-0.20251223152017-727e292afc8a h1:JJEY41aZXr7cQkepsBWahvg81PlK8yc+XZD1cX6XzAM=
+github.com/threefoldtech/zosbase v1.0.7-0.20251223152017-727e292afc8a/go.mod h1:NQRib2wxtsMdvKpZlosjT9d/76VdxM89Fx5LmlwOP58=
 github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
 github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU=
 github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI=
@@ -478,8 +478,8 @@ golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=

backend/internal/api/app/app_dependencies.go

@@ -343,6 +343,7 @@ func (app *App) createHandlers() appHandlers {
 	deploymentService := services.NewDeploymentService(
 		app.core.appCtx, clusterRepo, userRepo, userNodesRepo, app.core.ewfEngine,
 		app.config.Debug, app.security.sshPublicKey, app.config.SSH.PrivateKeyPath, app.config.SystemAccount.Network,
+		app.config.ZlogOutputURL,
 	)
 
 	adminService := services.NewAdminService(

backend/internal/config/config.go

@@ -39,9 +39,10 @@ type Configuration struct {
 	CheckUserDebtIntervalInHours         int                           `json:"check_user_debt_interval_in_hours" validate:"gt=0" default:"48"`
 	NodeHealthCheck                      ReservedNodeHealthCheckConfig `json:"node_health_check" validate:"required,dive"`
 
-	Logger    LoggerConfig    `json:"logger"`
-	Loki      LokiConfig      `json:"loki"`
-	Telemetry TelemetryConfig `json:"telemetry"`
+	Logger        LoggerConfig    `json:"logger"`
+	Loki          LokiConfig      `json:"loki"`
+	Telemetry     TelemetryConfig `json:"telemetry"`
+	ZlogOutputURL string          `json:"zlog_output_url" validate:"required,url"`
 }
 
 type SSHConfig struct {

backend/internal/config/config_test.go

@@ -92,7 +92,8 @@ func TestLoadConfig(t *testing.T) {
 			"url": "http://localhost:3100/loki/api/v1/push",
 			"flush_interval_second": 5
 		},
-		"notification_config_path": "` + notificationConfigPath + `"
+		"notification_config_path": "` + notificationConfigPath + `",
+		"zlog_output_url": "http://[::1]:3100/loki/api/v1/push?tenant_id=zlog-vms"
 	}`
 
 	// Write the config files
@@ -203,7 +204,8 @@ func TestDefaultTagsInConfig(t *testing.T) {
 		},
 		"loki": {
 			"url": "http://loki:3100/loki/api/v1/push"
-		}
+		},
+		"zlog_output_url": "http://[::1]:3100/loki/api/v1/push?tenant_id=zlog-vms"
 	}`
 
 	// Write the config file
@@ -288,7 +290,8 @@ func TestLoadConfig_InvalidConfig(t *testing.T) {
 					"reserved_node_health_check_interval_in_hours": 1,
 					"reserved_node_health_check_timeout_in_minutes": 1,
 					"reserved_node_health_check_workers_num": 10
-				}
+				},
+				"zlog_output_url": "http://[::1]:3100/loki/api/v1/push?tenant_id=zlog-vms"
 			}`,
 			expectedErr: "validation error on field 'Configuration.Database.DSN': dsn",
 		},
@@ -321,7 +324,8 @@ func TestLoadConfig_InvalidConfig(t *testing.T) {
 					"reserved_node_health_check_interval_in_hours": 1,
 					"reserved_node_health_check_timeout_in_minutes": 1,
 					"reserved_node_health_check_workers_num": 10
-				}
+				},
+				"zlog_output_url": "http://[::1]:3100/loki/api/v1/push?tenant_id=zlog-vms"
 			}`,
 			expectedErr: "validation error on field 'Configuration.Database.MaxIdleConns': lteMaxOpenConns",
 		},
@@ -349,7 +353,8 @@ func TestLoadConfig_InvalidConfig(t *testing.T) {
 					"reserved_node_health_check_interval_in_hours": 1,
 					"reserved_node_health_check_timeout_in_minutes": 1,
 					"reserved_node_health_check_workers_num": 10
-				}
+				},
+				"zlog_output_url": "http://[::1]:3100/loki/api/v1/push?tenant_id=zlog-vms"
 			}`,
 			expectedErr: "validation error on field 'Configuration.JwtToken.Secret': required",
 		},
@@ -379,7 +384,8 @@ func TestLoadConfig_InvalidConfig(t *testing.T) {
 					"reserved_node_health_check_interval_in_hours": 1,
 					"reserved_node_health_check_timeout_in_minutes": 1,
 					"reserved_node_health_check_workers_num": 10
-				}
+				},
+				"zlog_output_url": "http://[::1]:3100/loki/api/v1/push?tenant_id=zlog-vms"
 			}`,
 			expectedErr: "validation error on field 'Configuration.Admins': required",
 		},
@@ -445,7 +451,8 @@ func TestSQLiteProductionCheck(t *testing.T) {
 				"ssh": {"private_key_path": "` + privateKeyPath + `", "public_key_path": "` + publicKeyPath + `"},
 				"monitor_balance_interval_in_minutes": 1,
 				"notify_admins_for_pending_records_in_hours": 1,
-				"node_health_check": {"reserved_node_health_check_interval_in_hours": 1, "reserved_node_health_check_timeout_in_minutes": 1, "reserved_node_health_check_workers_num": 1}
+				"node_health_check": {"reserved_node_health_check_interval_in_hours": 1, "reserved_node_health_check_timeout_in_minutes": 1, "reserved_node_health_check_workers_num": 1},
+				"zlog_output_url": "http://[::1]:3100/loki/api/v1/push?tenant_id=zlog-vms"
 			}`
 			err := os.WriteFile(configPath, []byte(configJSON), 0644)
 			require.NoError(t, err, "Failed to write test config file")

backend/internal/core/services/deployment_service.go

@@ -32,12 +32,13 @@ type DeploymentService struct {
 	sshPublicKey      string
 	sshPrivateKeyPath string
 	systemNetwork     string
+	zlogOutputURL     string
 }
 
 func NewDeploymentService(appCtx context.Context,
 	clusterRepo models.ClusterRepository, userRepo models.UserRepository,
 	userNodesRepo models.UserNodesRepository, ewfEngine *ewf.Engine,
-	debug bool, sshPublicKey, sshPrivateKeyPath, systemNetwork string,
+	debug bool, sshPublicKey, sshPrivateKeyPath, systemNetwork, zlogOutputURL string,
 ) DeploymentService {
 	return DeploymentService{
 		clusterRepo: clusterRepo,
@@ -51,6 +52,7 @@ func NewDeploymentService(appCtx context.Context,
 		sshPublicKey:      sshPublicKey,
 		sshPrivateKeyPath: sshPrivateKeyPath,
 		systemNetwork:     systemNetwork,
+		zlogOutputURL:     zlogOutputURL,
 	}
 }
 
@@ -179,11 +181,12 @@ func (svc *DeploymentService) GetClientConfig(userID int) (statemanager.ClientCo
 	}
 
 	return statemanager.ClientConfig{
-		SSHPublicKey: svc.sshPublicKey,
-		Mnemonic:     user.Mnemonic,
-		UserID:       userID,
-		Network:      svc.systemNetwork,
-		Debug:        svc.debug,
+		SSHPublicKey:  svc.sshPublicKey,
+		Mnemonic:      user.Mnemonic,
+		UserID:        userID,
+		Network:       svc.systemNetwork,
+		Debug:         svc.debug,
+		ZlogOutputURL: svc.zlogOutputURL,
 	}, nil
 }
 

backend/internal/deployment/kubedeployer/client.go

@@ -8,11 +8,12 @@ import (
 )
 
 type Client struct {
-	GridClient deployer.TFPluginClient
-	mnemonic   string
+	GridClient    deployer.TFPluginClient
+	mnemonic      string
+	zlogOutputURL string
 }
 
-func NewClient(mnemonic, gridNet string, debug bool, tp trace.TracerProvider) (*Client, error) {
+func NewClient(mnemonic, gridNet string, debug bool, tp trace.TracerProvider, zlogOutputURL string) (*Client, error) {
 	pluginOpts := []deployer.PluginOpt{
 		deployer.WithNetwork(gridNet),
 		deployer.WithDisableSentry(),
@@ -34,8 +35,9 @@ func NewClient(mnemonic, gridNet string, debug bool, tp trace.TracerProvider) (*
 	}
 
 	return &Client{
-		GridClient: tfplugin,
-		mnemonic:   mnemonic,
+		GridClient:    tfplugin,
+		mnemonic:      mnemonic,
+		zlogOutputURL: zlogOutputURL,
 	}, nil
 }
 

backend/internal/deployment/kubedeployer/converters.go

@@ -40,6 +40,7 @@ func deploymentFromNode(
 	masterSSH string,
 	mnemonic string,
 	gridNet string,
+	zlogOutputURL string,
 ) (workloads.Deployment, error) {
 	ipSeed, err := workloads.RandomMyceliumIPSeed()
 	if err != nil {
@@ -82,6 +83,13 @@ func deploymentFromNode(
 		node.EnvVars = make(map[string]string)
 	}
 
+	zlogs := []workloads.Zlog{
+		{
+			Zmachine: node.Name,
+			Output:   zlogOutputURL,
+		},
+	}
+
 	vm := workloads.VM{
 		Name:           node.Name,
 		NodeID:         node.NodeID,
@@ -96,6 +104,7 @@ func deploymentFromNode(
 		MyceliumIPSeed: ipSeed,
 		Mounts:         mounts,
 		GPUs:           gpus,
+		Zlogs:          zlogs,
 	}
 
 	vm.EnvVars["K3S_NODE_NAME"] = node.Name

backend/internal/deployment/kubedeployer/core.go

@@ -111,6 +111,7 @@ func (c *Client) DeployNode(ctx context.Context, cluster *Cluster, node Node, ma
 		masterPubKey,
 		c.mnemonic,
 		c.GridClient.Network,
+		c.zlogOutputURL,
 	)
 	if err != nil {
 		telemetry.RecordError(span, err)
@@ -243,6 +244,7 @@ func (c *Client) BatchDeployNodes(ctx context.Context, cluster *Cluster, nodes [
 			masterPubKey,
 			c.mnemonic,
 			c.GridClient.Network,
+			c.zlogOutputURL,
 		)
 		if err != nil {
 			telemetry.RecordError(span, err)

backend/internal/deployment/statemanager/client_manager.go

@@ -14,11 +14,12 @@ import (
 
 // ClientConfig represents the configuration needed to create a kubeclient
 type ClientConfig struct {
-	SSHPublicKey string `json:"ssh_public_key"`
-	Mnemonic     string `json:"mnemonic"`
-	UserID       int    `json:"user_id"`
-	Network      string `json:"network"`
-	Debug        bool   `json:"debug"`
+	SSHPublicKey  string `json:"ssh_public_key"`
+	Mnemonic      string `json:"mnemonic"`
+	UserID        int    `json:"user_id"`
+	Network       string `json:"network"`
+	Debug         bool   `json:"debug"`
+	ZlogOutputURL string `json:"zlog_output_url"`
 }
 
 // ValidateConfig validates the client configuration
@@ -66,7 +67,7 @@ func GetKubeClient(state ewf.State, config ClientConfig) (*kubedeployer.Client,
 	}
 
 	// Create new client
-	kubeClient, err := kubedeployer.NewClient(config.Mnemonic, config.Network, config.Debug, globalTp)
+	kubeClient, err := kubedeployer.NewClient(config.Mnemonic, config.Network, config.Debug, globalTp, config.ZlogOutputURL)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create kubeclient: %w", err)
 	}

docker-compose.yml

@@ -11,6 +11,7 @@ services:
       - /dev/net/tun:/dev/net/tun
     ports:
       - "8080:8080" # exposes the backend API
+      - "3100:3100" # exposes Loki
     networks:
       - app-network # connect to redis with network
     volumes:
@@ -112,14 +113,12 @@ services:
   loki:
     image: grafana/loki:latest
     container_name: loki
-    ports:
-      - "3100:3100"
     command: -config.file=/etc/loki/loki-config.yaml -target=all
     volumes:
       - ./loki/loki-config.yaml:/etc/loki/loki-config.yaml
       - loki-data:/loki
-    networks:
-      - app-network
+    network_mode: "service:mycelium"
+
   promtail:
     image: grafana/promtail:latest
     container_name: promtail
@@ -138,8 +137,8 @@ services:
     container_name: jaeger
     ports:
       - "16686:16686" # Jaeger UI
-      - "4317:4317"   # OTLP gRPC receiver
-      - "4318:4318"   # OTLP HTTP receiver
+      - "4317:4317" # OTLP gRPC receiver
+      - "4318:4318" # OTLP HTTP receiver
     environment:
       - COLLECTOR_OTLP_ENABLED=true
       - SPAN_STORAGE_TYPE=badger

grafana-gen/main.go

@@ -113,21 +113,39 @@ func main() {
 		map[string]any{
 			"id":    id + 26,
 			"type":  "logs",
-			"title": "Application Logs",
+			"title": "KubeCloud Application Logs",
 			"targets": []map[string]any{
 				{
 					"expr":       `{job="app-logs"}`,
 					"refId":      "A",
-					"datasource": "Loki",
+					"datasource": "Loki - KubeCloud",
 				},
 			},
 			"gridPos": map[string]int{
 				"h": 8,
-				"w": 24,
+				"w": 12,
 				"x": 0,
 				"y": y + 76,
 			},
 		},
+		map[string]any{
+			"id":    id + 27,
+			"type":  "logs",
+			"title": "VMs Logs",
+			"targets": []map[string]any{
+				{
+					"expr":       `{job="tailstream"}`,
+					"refId":      "A",
+					"datasource": "Loki - VMs",
+				},
+			},
+			"gridPos": map[string]int{
+				"h": 8,
+				"w": 12,
+				"x": 12,
+				"y": y + 76,
+			},
+		},
 	}
 
 	dashboard := map[string]any{

grafana/provisioning/datasources/datasource.yml

@@ -6,8 +6,25 @@ datasources:
     access: proxy
     url: http://prometheus:9090
     isDefault: true
-  - name: Loki
+
+  # KubeCloud internal logs
+  - name: Loki - KubeCloud
+    type: loki
+    access: proxy
+    url: http://mycelium:3100
+    isDefault: false
+    jsonData:
+      httpHeaderName1: "X-Scope-OrgID"
+    secureJsonData:
+      httpHeaderValue1: "kubecloud"
+
+  # Cluster VMs logs
+  - name: Loki - VMs
     type: loki
     access: proxy
-    url: http://loki:3100
+    url: http://mycelium:3100
     isDefault: false
+    jsonData:
+      httpHeaderName1: "X-Scope-OrgID"
+    secureJsonData:
+      httpHeaderValue1: "zlog-vms"