From ef8651410dd85e60a5ce14cead4d8ced1dc1d385 Mon Sep 17 00:00:00 2001 From: hongming Date: Mon, 25 May 2026 12:05:05 -0700 Subject: [PATCH] feat: refresh workspace templates from repo cache --- manifest.json | 2 +- scripts/clone-manifest.sh | 17 +- workspace-server/Dockerfile | 1 + workspace-server/Dockerfile.tenant | 1 + workspace-server/cmd/server/main.go | 62 +++++- .../internal/handlers/templates.go | 175 +++++++++++------ .../internal/handlers/templates_test.go | 65 +++++++ .../internal/handlers/workspace.go | 10 +- .../internal/handlers/workspace_provision.go | 13 +- .../handlers/workspace_provision_test.go | 26 +++ workspace-server/internal/router/router.go | 7 +- .../internal/templatecache/cache.go | 176 ++++++++++++++++++ .../internal/templatecache/cache_test.go | 16 ++ 13 files changed, 488 insertions(+), 83 deletions(-) create mode 100644 workspace-server/internal/templatecache/cache.go create mode 100644 workspace-server/internal/templatecache/cache_test.go diff --git a/manifest.json b/manifest.json index 81204ffd2..5ba8c48bc 100644 --- a/manifest.json +++ b/manifest.json @@ -1,5 +1,5 @@ { - "_comment": "OSS surface registry — every repo listed here MUST be public on git.moleculesai.app. Layer-3 customer/private templates are NOT registered here; they are handled at provision-time via the per-tenant credential resolver (see internal#102 RFC). 'main' refs are pinned to tags before broad rollout.", + "_comment": "Platform template registry. Repos may be public or platform-private; CI and runtime template-cache refresh clone them with the SSOT-managed template read token, then strip .git metadata before use. Customer/private tenant templates remain outside this platform manifest. 'main' refs are pinned to tags before broad rollout.", "version": 1, "plugins": [ {"name": "browser-automation", "repo": "molecule-ai/molecule-ai-plugin-browser-automation", "ref": "main"}, diff --git a/scripts/clone-manifest.sh b/scripts/clone-manifest.sh index 396d5f6bf..bbb4fd446 100755 --- a/scripts/clone-manifest.sh +++ b/scripts/clone-manifest.sh @@ -8,19 +8,10 @@ # Requires: git, jq (lighter than python3 — ~2MB vs ~50MB in Alpine) # # Auth (optional): -# Post-2026-05-08 (#192): every repo in manifest.json is public on -# git.moleculesai.app. Anonymous clone works for the entire registered -# set. The OSS-surface contract is recorded in manifest.json's _comment -# — Layer-3 customer/private templates (e.g. reno-stars) are NOT in the -# manifest; they are handled at provision-time via the per-tenant -# credential resolver (internal#102 RFC). -# -# MOLECULE_GITEA_TOKEN is therefore optional today. Kept supported for -# two reasons: (a) historical CI configs that still inject -# AUTO_SYNC_TOKEN remain harmless, (b) reserved for the case where a -# private internal-only template is later registered via a ci-readonly -# team grant — review must explicitly sign off on that, since it -# violates the public-OSS-surface contract. +# Repos in manifest.json may be public or platform-private. CI and +# operator refresh jobs should set MOLECULE_GITEA_TOKEN to the +# SSOT-managed template read token. Anonymous clone still works for +# public entries, but private platform templates depend on the token. # # The token (when set) never enters the Docker image: this script runs # in the trusted CI context BEFORE `docker buildx build`, populates diff --git a/workspace-server/Dockerfile b/workspace-server/Dockerfile index a5c42bb1c..adadfad35 100644 --- a/workspace-server/Dockerfile +++ b/workspace-server/Dockerfile @@ -71,6 +71,7 @@ RUN apk add --no-cache ca-certificates docker-cli docker-cli-buildx git tzdata w COPY --from=builder /platform /platform COPY --from=builder /memory-plugin /memory-plugin COPY workspace-server/migrations /migrations +COPY manifest.json /app/manifest.json # Templates + plugins (pre-cloned by scripts/clone-manifest.sh in the # trusted CI / operator-host context, .git already stripped). The Gitea # token used to clone them never enters this image — same shape as diff --git a/workspace-server/Dockerfile.tenant b/workspace-server/Dockerfile.tenant index 3e9d1d36f..69a054492 100644 --- a/workspace-server/Dockerfile.tenant +++ b/workspace-server/Dockerfile.tenant @@ -118,6 +118,7 @@ RUN deluser --remove-home node 2>/dev/null || true; \ COPY --from=go-builder /platform /platform COPY --from=go-builder /memory-plugin /memory-plugin COPY workspace-server/migrations /migrations +COPY manifest.json /app/manifest.json # Templates + plugins (pre-cloned by scripts/clone-manifest.sh in the # trusted CI / operator-host context, .git already stripped — see diff --git a/workspace-server/cmd/server/main.go b/workspace-server/cmd/server/main.go index bc4220d78..e8918db2f 100644 --- a/workspace-server/cmd/server/main.go +++ b/workspace-server/cmd/server/main.go @@ -50,6 +50,7 @@ import ( "git.moleculesai.app/molecule-ai/molecule-core/workspace-server/internal/router" "git.moleculesai.app/molecule-ai/molecule-core/workspace-server/internal/scheduler" "git.moleculesai.app/molecule-ai/molecule-core/workspace-server/internal/supervised" + "git.moleculesai.app/molecule-ai/molecule-core/workspace-server/internal/templatecache" "git.moleculesai.app/molecule-ai/molecule-core/workspace-server/internal/ws" // External plugins — each registers EnvMutator(s) that run at workspace @@ -58,6 +59,7 @@ import ( ghidentity "go.moleculesai.app/plugin/gh-identity/pluginloader" "git.moleculesai.app/molecule-ai/molecule-core/workspace-server/pkg/provisionhook" + "github.com/gin-gonic/gin" ) func main() { @@ -193,11 +195,28 @@ func main() { port := envOr("PORT", "8080") platformURL := envOr("PLATFORM_URL", fmt.Sprintf("http://host.docker.internal:%s", port)) configsDir := envOr("CONFIGS_DIR", findConfigsDir()) + templateCacheDir := envOr("TEMPLATE_CACHE_DIR", filepath.Join(os.TempDir(), "molecule-template-cache")) + manifestPath := findWorkspaceManifestPath() + templateToken := templateCacheToken() + refreshTemplates := func(ctx context.Context) (templatecache.RefreshReport, error) { + return templatecache.RefreshWorkspaceTemplates(ctx, manifestPath, templateCacheDir, templateToken) + } + if shouldRefreshTemplateCache(templateToken, manifestPath) { + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + report, err := refreshTemplates(ctx) + cancel() + if err != nil { + log.Printf("template cache refresh: %v (continuing with baked templates)", err) + } else { + log.Printf("template cache refresh: refreshed %d workspace templates into %s", len(report.Results), templateCacheDir) + } + } // Init order: wh → onWorkspaceOffline → liveness/healthSweep → router // WorkspaceHandler is created before the router so RestartByID can be wired into // the offline callbacks used by both the liveness monitor and the health sweep. - wh := handlers.NewWorkspaceHandler(broadcaster, prov, platformURL, configsDir) + wh := handlers.NewWorkspaceHandler(broadcaster, prov, platformURL, configsDir). + WithTemplateCacheDir(templateCacheDir) if cpProv != nil { wh.SetCPProvisioner(cpProv) } @@ -377,7 +396,12 @@ func main() { // require a plugins/ dir on disk (nil in CP/SaaS mode). pluginRegistry := plugins.NewRegistry() pluginRegistry.Register(plugins.NewGithubResolver()) - r := router.Setup(hub, broadcaster, prov, platformURL, configsDir, wh, channelMgr, memBundle, pluginRegistry) + refreshTemplatesHTTP := func(c *gin.Context) (any, error) { + ctx, cancel := context.WithTimeout(c.Request.Context(), 2*time.Minute) + defer cancel() + return refreshTemplates(ctx) + } + r := router.Setup(hub, broadcaster, prov, platformURL, configsDir, templateCacheDir, wh, channelMgr, memBundle, pluginRegistry, refreshTemplatesHTTP) // Plugin drift sweeper — periodic detection of upstream plugin version drift // (core#123). Scans workspace_plugins rows where tracked_ref != 'none', @@ -493,6 +517,40 @@ func findConfigsDir() string { return "workspace-configs-templates" } +func findWorkspaceManifestPath() string { + if v := os.Getenv("WORKSPACE_MANIFEST_PATH"); v != "" { + return v + } + for _, p := range []string{"/app/manifest.json", "manifest.json", "../manifest.json", "../../manifest.json"} { + if abs, err := filepath.Abs(p); err == nil { + if _, err := os.Stat(abs); err == nil { + return abs + } + } + } + return "" +} + +func templateCacheToken() string { + for _, key := range []string{"MOLECULE_TEMPLATE_GITEA_TOKEN", "MOLECULE_GITEA_TOKEN"} { + if v := strings.TrimSpace(os.Getenv(key)); v != "" { + return v + } + } + return "" +} + +func shouldRefreshTemplateCache(token, manifestPath string) bool { + switch strings.ToLower(strings.TrimSpace(os.Getenv("TEMPLATE_CACHE_REFRESH"))) { + case "0", "false", "off", "no": + return false + case "1", "true", "on", "yes": + return token != "" && manifestPath != "" + default: + return token != "" && manifestPath != "" + } +} + func findMigrationsDir() string { candidates := []string{ "migrations", diff --git a/workspace-server/internal/handlers/templates.go b/workspace-server/internal/handlers/templates.go index 7ccdde618..efc239b5f 100644 --- a/workspace-server/internal/handlers/templates.go +++ b/workspace-server/internal/handlers/templates.go @@ -54,6 +54,7 @@ const maxUploadFiles = 200 type TemplatesHandler struct { configsDir string + cacheDir string docker *client.Client // wh is used by Import and ReplaceFiles to call DefaultTier() so a // generated config.yaml's tier matches the SaaS-vs-self-hosted @@ -61,6 +62,11 @@ type TemplatesHandler struct { // the caller doesn't import templates that need a fresh config // generated. wh *WorkspaceHandler + // refreshCache is nil unless main wires a manifest-backed template + // cache refresher. POST /admin/templates/refresh uses this hook so a + // template repo merge can update the tenant catalog without rebuilding + // the full tenant image. + refreshCache func(ctx *gin.Context) (any, error) } // NewTemplatesHandler constructs a TemplatesHandler. wh may be nil for @@ -71,6 +77,16 @@ func NewTemplatesHandler(configsDir string, dockerCli *client.Client, wh *Worksp return &TemplatesHandler{configsDir: configsDir, docker: dockerCli, wh: wh} } +func (h *TemplatesHandler) WithCacheDir(cacheDir string) *TemplatesHandler { + h.cacheDir = cacheDir + return h +} + +func (h *TemplatesHandler) WithRefreshFunc(fn func(ctx *gin.Context) (any, error)) *TemplatesHandler { + h.refreshCache = fn + return h +} + // modelSpec describes a single supported model on a template: its id (sent // to the runtime), a human-readable label, and the env vars that must be // present for that model to work (e.g. API keys). @@ -161,6 +177,15 @@ type templateSummary struct { // Only resolves to actual templates (not ws-* dirs since those are now Docker volumes). // Returns empty string if no matching template is found. func (h *TemplatesHandler) resolveTemplateDir(wsName string) string { + if h.cacheDir != "" { + nameDir := filepath.Join(h.cacheDir, normalizeName(wsName)) + if _, err := os.Stat(nameDir); err == nil { + return nameDir + } + if tmpl := findTemplateByName(h.cacheDir, wsName); tmpl != "" { + return filepath.Join(h.cacheDir, tmpl) + } + } nameDir := filepath.Join(h.configsDir, normalizeName(wsName)) if _, err := os.Stat(nameDir); err == nil { return nameDir @@ -175,78 +200,104 @@ func (h *TemplatesHandler) resolveTemplateDir(wsName string) string { // List handles GET /templates func (h *TemplatesHandler) List(c *gin.Context) { templates := make([]templateSummary, 0) - walkTemplateConfigs(h.configsDir, func(id string, data []byte) { - var raw struct { - Name string `yaml:"name"` - Description string `yaml:"description"` - Tier int `yaml:"tier"` - Runtime string `yaml:"runtime"` - Model string `yaml:"model"` - Skills []string `yaml:"skills"` - // Top-level `providers:` block — structured registry. Distinct - // from runtime_config.providers (slug list) below. Both shapes - // coexist in production: claude-code ships the structured - // registry, hermes still uses the slug list. /templates surfaces - // both verbatim so each runtime owns its taxonomy. - Providers []providerRegistryEntry `yaml:"providers"` - RuntimeConfig struct { - Model string `yaml:"model"` - Models []modelSpec `yaml:"models"` - RequiredEnv []string `yaml:"required_env"` - RecommendedEnv []string `yaml:"recommended_env"` - Providers []string `yaml:"providers"` - ProvisionTimeoutSeconds int `yaml:"provision_timeout_seconds"` - } `yaml:"runtime_config"` - } - if err := yaml.Unmarshal(data, &raw); err != nil { - // Without this log a malformed config.yaml causes the - // template to silently disappear from /templates with no - // trace — the operator can't tell "excluded due to parse - // error" from "never existed." That matters more now that - // templates ship richer YAML shapes (top-level providers - // registry, models[] with required_env, etc.) where a - // type-shape mismatch on one field drops the whole entry. - log.Printf("templates list: skip %s: yaml.Unmarshal: %v", id, err) - return - } - runtime := strings.TrimSuffix(strings.TrimSpace(raw.Runtime), "-default") - if _, ok := knownRuntimes[runtime]; !ok { - log.Printf("templates list: skip %s: unsupported runtime %q", id, raw.Runtime) + seen := map[string]struct{}{} + walk := func(root string) { + if root == "" { return } + walkTemplateConfigs(root, func(id string, data []byte) { + if _, ok := seen[id]; ok { + return + } + seen[id] = struct{}{} + var raw struct { + Name string `yaml:"name"` + Description string `yaml:"description"` + Tier int `yaml:"tier"` + Runtime string `yaml:"runtime"` + Model string `yaml:"model"` + Skills []string `yaml:"skills"` + // Top-level `providers:` block — structured registry. Distinct + // from runtime_config.providers (slug list) below. Both shapes + // coexist in production: claude-code ships the structured + // registry, hermes still uses the slug list. /templates surfaces + // both verbatim so each runtime owns its taxonomy. + Providers []providerRegistryEntry `yaml:"providers"` + RuntimeConfig struct { + Model string `yaml:"model"` + Models []modelSpec `yaml:"models"` + RequiredEnv []string `yaml:"required_env"` + RecommendedEnv []string `yaml:"recommended_env"` + Providers []string `yaml:"providers"` + ProvisionTimeoutSeconds int `yaml:"provision_timeout_seconds"` + } `yaml:"runtime_config"` + } + if err := yaml.Unmarshal(data, &raw); err != nil { + // Without this log a malformed config.yaml causes the + // template to silently disappear from /templates with no + // trace — the operator can't tell "excluded due to parse + // error" from "never existed." That matters more now that + // templates ship richer YAML shapes (top-level providers + // registry, models[] with required_env, etc.) where a + // type-shape mismatch on one field drops the whole entry. + log.Printf("templates list: skip %s: yaml.Unmarshal: %v", id, err) + return + } + runtime := strings.TrimSuffix(strings.TrimSpace(raw.Runtime), "-default") + if _, ok := knownRuntimes[runtime]; !ok { + log.Printf("templates list: skip %s: unsupported runtime %q", id, raw.Runtime) + return + } - // Model comes from either top-level (legacy) or runtime_config.model (current). - model := raw.Model - if model == "" { - model = raw.RuntimeConfig.Model - } + // Model comes from either top-level (legacy) or runtime_config.model (current). + model := raw.Model + if model == "" { + model = raw.RuntimeConfig.Model + } - tier := raw.Tier - if h.wh != nil && h.wh.IsSaaS() { - tier = h.wh.DefaultTier() - } + tier := raw.Tier + if h.wh != nil && h.wh.IsSaaS() { + tier = h.wh.DefaultTier() + } - templates = append(templates, templateSummary{ - ID: id, - Name: raw.Name, - Description: raw.Description, - Tier: tier, - Runtime: raw.Runtime, - Model: model, - Models: raw.RuntimeConfig.Models, - RequiredEnv: raw.RuntimeConfig.RequiredEnv, - RecommendedEnv: raw.RuntimeConfig.RecommendedEnv, - Providers: raw.RuntimeConfig.Providers, - ProviderRegistry: raw.Providers, - Skills: raw.Skills, - SkillCount: len(raw.Skills), - ProvisionTimeoutSeconds: raw.RuntimeConfig.ProvisionTimeoutSeconds, + templates = append(templates, templateSummary{ + ID: id, + Name: raw.Name, + Description: raw.Description, + Tier: tier, + Runtime: raw.Runtime, + Model: model, + Models: raw.RuntimeConfig.Models, + RequiredEnv: raw.RuntimeConfig.RequiredEnv, + RecommendedEnv: raw.RuntimeConfig.RecommendedEnv, + Providers: raw.RuntimeConfig.Providers, + ProviderRegistry: raw.Providers, + Skills: raw.Skills, + SkillCount: len(raw.Skills), + ProvisionTimeoutSeconds: raw.RuntimeConfig.ProvisionTimeoutSeconds, + }) }) - }) + } + walk(h.cacheDir) + walk(h.configsDir) c.JSON(http.StatusOK, templates) } +// RefreshCache handles POST /admin/templates/refresh. +func (h *TemplatesHandler) RefreshCache(c *gin.Context) { + if h.refreshCache == nil { + c.JSON(http.StatusServiceUnavailable, gin.H{"error": "template cache refresh is not configured"}) + return + } + result, err := h.refreshCache(c) + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + return + } + c.JSON(http.StatusOK, result) +} + // ListFiles handles GET /workspaces/:id/files // Lists files inside the running container's /configs directory (or /workspace, etc.). // Falls back to host-side config templates directory when container isn't running. diff --git a/workspace-server/internal/handlers/templates_test.go b/workspace-server/internal/handlers/templates_test.go index 31508d207..f7d562667 100644 --- a/workspace-server/internal/handlers/templates_test.go +++ b/workspace-server/internal/handlers/templates_test.go @@ -133,6 +133,71 @@ skills: } } +func TestTemplatesList_CacheOverridesBakedTemplate(t *testing.T) { + setupTestDB(t) + setupTestRedis(t) + + bakedDir := t.TempDir() + cacheDir := t.TempDir() + + mustWriteTemplate := func(root, id, body string) { + t.Helper() + dir := filepath.Join(root, id) + if err := os.MkdirAll(dir, 0755); err != nil { + t.Fatalf("mkdir %s: %v", dir, err) + } + if err := os.WriteFile(filepath.Join(dir, "config.yaml"), []byte(body), 0644); err != nil { + t.Fatalf("write config: %v", err) + } + } + + mustWriteTemplate(bakedDir, "seo-agent", `name: SEO Agent +description: stale +tier: 4 +runtime: claude-code +model: old +runtime_config: + recommended_env: [TELEGRAM_BOT_TOKEN] +skills: [] +`) + mustWriteTemplate(cacheDir, "seo-agent", `name: SEO Agent +description: fresh +tier: 4 +runtime: claude-code +model: moonshot/kimi-k2.6 +runtime_config: + required_env: [TENANT_NAME] + recommended_env: [GOOGLE_GSC_SITE] +skills: [] +`) + + handler := NewTemplatesHandler(bakedDir, nil, nil).WithCacheDir(cacheDir) + w := httptest.NewRecorder() + c, _ := gin.CreateTestContext(w) + c.Request = httptest.NewRequest("GET", "/templates", nil) + handler.List(c) + + if w.Code != http.StatusOK { + t.Fatalf("expected 200, got %d", w.Code) + } + var resp []templateSummary + if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil { + t.Fatalf("parse: %v", err) + } + if len(resp) != 1 { + t.Fatalf("expected 1 template, got %d", len(resp)) + } + if resp[0].Description != "fresh" { + t.Fatalf("cache template should override baked copy, got description %q", resp[0].Description) + } + if !reflect.DeepEqual(resp[0].RequiredEnv, []string{"TENANT_NAME"}) { + t.Fatalf("RequiredEnv = %+v", resp[0].RequiredEnv) + } + if reflect.DeepEqual(resp[0].RecommendedEnv, []string{"TELEGRAM_BOT_TOKEN"}) { + t.Fatalf("stale baked recommended_env leaked through: %+v", resp[0].RecommendedEnv) + } +} + func TestTemplatesList_RuntimeAndModelsRegistry(t *testing.T) { setupTestDB(t) setupTestRedis(t) diff --git a/workspace-server/internal/handlers/workspace.go b/workspace-server/internal/handlers/workspace.go index 93d3978ab..592b40540 100644 --- a/workspace-server/internal/handlers/workspace.go +++ b/workspace-server/internal/handlers/workspace.go @@ -56,6 +56,7 @@ type WorkspaceHandler struct { cpProv provisioner.CPProvisionerAPI platformURL string configsDir string // path to workspace-configs-templates/ (for reading templates) + cacheDir string // optional runtime-refreshed template cache; overrides configsDir by template id // envMutators runs registered EnvMutator plugins right before // container Start, after built-in secret loads. Nil = no plugins // registered; Registry.Run handles a nil receiver as a no-op so the @@ -183,6 +184,11 @@ func NewWorkspaceHandler(b events.EventEmitter, p *provisioner.Provisioner, plat return h } +func (h *WorkspaceHandler) WithTemplateCacheDir(cacheDir string) *WorkspaceHandler { + h.cacheDir = cacheDir + return h +} + // WithNamespaceCleanup wires the I5 hook (RFC #2728) so workspace // purge can drop the plugin's `workspace:` namespace. main.go // passes a closure over plugin.DeleteNamespace; tests pass a stub. @@ -285,7 +291,7 @@ func (h *WorkspaceHandler) Create(c *gin.Context) { // #226: payload.Template is attacker-controllable. resolveInsideRoot // rejects absolute paths and any ".." that escapes configsDir so the // provisioner can't be pointed at host directories. - candidatePath, resolveErr := resolveInsideRoot(h.configsDir, payload.Template) + candidatePath, resolveErr := resolveWorkspaceTemplatePath(h.configsDir, h.cacheDir, payload.Template) if resolveErr != nil { log.Printf("Create: invalid template path %q: %v", payload.Template, resolveErr) c.JSON(http.StatusBadRequest, gin.H{"error": "invalid template"}) @@ -726,7 +732,7 @@ func (h *WorkspaceHandler) Create(c *gin.Context) { var templatePath string var configFiles map[string][]byte if payload.Template != "" { - candidatePath, resolveErr := resolveInsideRoot(h.configsDir, payload.Template) + candidatePath, resolveErr := resolveWorkspaceTemplatePath(h.configsDir, h.cacheDir, payload.Template) if resolveErr != nil { log.Printf("Create provision: rejecting template %q: %v", payload.Template, resolveErr) return diff --git a/workspace-server/internal/handlers/workspace_provision.go b/workspace-server/internal/handlers/workspace_provision.go index 7a3a4d41f..a31ac3c3a 100644 --- a/workspace-server/internal/handlers/workspace_provision.go +++ b/workspace-server/internal/handlers/workspace_provision.go @@ -485,6 +485,17 @@ func findTemplateByName(configsDir, name string) string { return "" } +func resolveWorkspaceTemplatePath(configsDir, cacheDir, template string) (string, error) { + if cacheDir != "" { + if p, err := resolveInsideRoot(cacheDir, template); err != nil { + return "", err + } else if _, statErr := os.Stat(p); statErr == nil { + return p, nil + } + } + return resolveInsideRoot(configsDir, template) +} + // resolveOrgTemplate looks for a matching role directory under // configsDir/org-templates/ and returns the absolute path and a short label // ("org-templates/"). Used by the restart handler's rebuild_config path @@ -658,7 +669,7 @@ func (h *WorkspaceHandler) defaultTemplateProvidersYAML(runtime string) string { return "" } templateName := runtime + "-default" - templatePath, err := resolveInsideRoot(h.configsDir, templateName) + templatePath, err := resolveWorkspaceTemplatePath(h.configsDir, h.cacheDir, templateName) if err != nil { log.Printf("Provisioner: default template providers skipped for runtime %s: %v", runtime, err) return "" diff --git a/workspace-server/internal/handlers/workspace_provision_test.go b/workspace-server/internal/handlers/workspace_provision_test.go index 7a4b2c8a2..05cf9b1bb 100644 --- a/workspace-server/internal/handlers/workspace_provision_test.go +++ b/workspace-server/internal/handlers/workspace_provision_test.go @@ -110,6 +110,32 @@ func TestFindTemplateByName_NotFound(t *testing.T) { } } +func TestResolveWorkspaceTemplatePath_PrefersCache(t *testing.T) { + bakedDir := t.TempDir() + cacheDir := t.TempDir() + + for _, root := range []string{bakedDir, cacheDir} { + if err := os.MkdirAll(filepath.Join(root, "seo-agent"), 0755); err != nil { + t.Fatalf("mkdir: %v", err) + } + } + + got, err := resolveWorkspaceTemplatePath(bakedDir, cacheDir, "seo-agent") + if err != nil { + t.Fatalf("resolveWorkspaceTemplatePath: %v", err) + } + want := filepath.Join(cacheDir, "seo-agent") + if got != want { + t.Fatalf("want cache path %q, got %q", want, got) + } +} + +func TestResolveWorkspaceTemplatePath_RejectsTraversal(t *testing.T) { + if _, err := resolveWorkspaceTemplatePath(t.TempDir(), t.TempDir(), "../seo-agent"); err == nil { + t.Fatal("expected traversal to be rejected") + } +} + func TestFindTemplateByName_SkipsWsPrefix(t *testing.T) { tmpDir := t.TempDir() diff --git a/workspace-server/internal/router/router.go b/workspace-server/internal/router/router.go index ba76a1f5d..ff6874fba 100644 --- a/workspace-server/internal/router/router.go +++ b/workspace-server/internal/router/router.go @@ -36,7 +36,7 @@ import ( // (main.go) gets the same pluginResolver instance so it can share scheme // enumeration if a deployment registers extra schemes externally. A nil // pluginResolver is harmless: plgh still works with its built-in defaults. -func Setup(hub *ws.Hub, broadcaster *events.Broadcaster, prov *provisioner.Provisioner, platformURL, configsDir string, wh *handlers.WorkspaceHandler, channelMgr *channels.Manager, memBundle *memwiring.Bundle, pluginResolver plugins.PluginResolver) *gin.Engine { +func Setup(hub *ws.Hub, broadcaster *events.Broadcaster, prov *provisioner.Provisioner, platformURL, configsDir string, templateCacheDir string, wh *handlers.WorkspaceHandler, channelMgr *channels.Manager, memBundle *memwiring.Bundle, pluginResolver plugins.PluginResolver, refreshTemplates func(ctx *gin.Context) (any, error)) *gin.Engine { r := gin.Default() // Issue #179 — trust no reverse-proxy headers. Without this call Gin's @@ -666,7 +666,9 @@ func Setup(hub *ws.Hub, broadcaster *events.Broadcaster, prov *provisioner.Provi // Templates — wh threaded so generateDefaultConfig picks the // SaaS-aware default tier in Import + ReplaceFiles (#2910 PR-B). - tmplh := handlers.NewTemplatesHandler(configsDir, dockerCli, wh) + tmplh := handlers.NewTemplatesHandler(configsDir, dockerCli, wh). + WithCacheDir(templateCacheDir). + WithRefreshFunc(refreshTemplates) // #686: GET /templates lists all template names+metadata from configsDir. // Open access lets unauthenticated callers enumerate org configurations and // installed plugins. AdminAuth-gate it alongside POST /templates/import. @@ -676,6 +678,7 @@ func Setup(hub *ws.Hub, broadcaster *events.Broadcaster, prov *provisioner.Provi tmplAdmin := r.Group("", middleware.AdminAuth(db.DB)) tmplAdmin.GET("/templates", tmplh.List) tmplAdmin.POST("/templates/import", tmplh.Import) + tmplAdmin.POST("/admin/templates/refresh", tmplh.RefreshCache) } wsAuth.PUT("/files", tmplh.ReplaceFiles) wsAuth.GET("/files", tmplh.ListFiles) diff --git a/workspace-server/internal/templatecache/cache.go b/workspace-server/internal/templatecache/cache.go new file mode 100644 index 000000000..f5a52253f --- /dev/null +++ b/workspace-server/internal/templatecache/cache.go @@ -0,0 +1,176 @@ +package templatecache + +import ( + "context" + "encoding/json" + "fmt" + "net/url" + "os" + "os/exec" + "path/filepath" + "strings" + "time" +) + +type ManifestEntry struct { + Name string `json:"name"` + Repo string `json:"repo"` + Ref string `json:"ref"` +} + +type manifestFile struct { + WorkspaceTemplates []ManifestEntry `json:"workspace_templates"` +} + +type TemplateResult struct { + Name string `json:"name"` + Repo string `json:"repo"` + Ref string `json:"ref"` + SHA string `json:"sha,omitempty"` + Status string `json:"status"` + Error string `json:"error,omitempty"` +} + +type RefreshReport struct { + ManifestPath string `json:"manifest_path"` + CacheDir string `json:"cache_dir"` + RefreshedAt time.Time `json:"refreshed_at"` + Results []TemplateResult `json:"results"` +} + +func RefreshWorkspaceTemplates(ctx context.Context, manifestPath, cacheDir, token string) (RefreshReport, error) { + report := RefreshReport{ + ManifestPath: manifestPath, + CacheDir: cacheDir, + RefreshedAt: time.Now().UTC(), + } + if strings.TrimSpace(token) == "" { + return report, fmt.Errorf("template cache refresh requires MOLECULE_TEMPLATE_GITEA_TOKEN or MOLECULE_GITEA_TOKEN") + } + data, err := os.ReadFile(manifestPath) + if err != nil { + return report, fmt.Errorf("read manifest: %w", err) + } + var manifest manifestFile + if err := json.Unmarshal(data, &manifest); err != nil { + return report, fmt.Errorf("parse manifest: %w", err) + } + if err := os.MkdirAll(cacheDir, 0o755); err != nil { + return report, fmt.Errorf("mkdir cache: %w", err) + } + for _, entry := range manifest.WorkspaceTemplates { + result := refreshOne(ctx, cacheDir, token, entry) + report.Results = append(report.Results, result) + } + return report, nil +} + +func refreshOne(ctx context.Context, cacheDir, token string, entry ManifestEntry) TemplateResult { + result := TemplateResult{Name: entry.Name, Repo: entry.Repo, Ref: entry.Ref} + if result.Ref == "" { + result.Ref = "main" + } + if !safeTemplateName(entry.Name) { + result.Status = "skipped" + result.Error = "invalid template name" + return result + } + if strings.TrimSpace(entry.Repo) == "" { + result.Status = "skipped" + result.Error = "missing repo" + return result + } + + tmp, err := os.MkdirTemp(cacheDir, ".tmp-"+entry.Name+"-") + if err != nil { + result.Status = "failed" + result.Error = err.Error() + return result + } + defer os.RemoveAll(tmp) + + cloneURL := authenticatedURL(entry.Repo, token) + for _, args := range [][]string{ + {"init", "-q", tmp}, + {"-C", tmp, "remote", "add", "origin", cloneURL}, + {"-C", tmp, "fetch", "--depth=1", "-q", "origin", result.Ref}, + {"-C", tmp, "checkout", "-q", "--detach", "FETCH_HEAD"}, + } { + cmd := exec.CommandContext(ctx, "git", args...) + if out, err := cmd.CombinedOutput(); err != nil { + result.Status = "failed" + result.Error = sanitizeGitError(out, err, token) + return result + } + } + shaCmd := exec.CommandContext(ctx, "git", "-C", tmp, "rev-parse", "HEAD") + if out, err := shaCmd.Output(); err == nil { + result.SHA = strings.TrimSpace(string(out)) + } + _ = os.RemoveAll(filepath.Join(tmp, ".git")) + + target := filepath.Join(cacheDir, entry.Name) + old := filepath.Join(cacheDir, ".old-"+entry.Name+"-"+fmt.Sprint(time.Now().UnixNano())) + if _, err := os.Stat(target); err == nil { + if err := os.Rename(target, old); err != nil { + result.Status = "failed" + result.Error = "replace old cache: " + err.Error() + return result + } + defer os.RemoveAll(old) + } + if err := os.Rename(tmp, target); err != nil { + if old != "" { + _ = os.Rename(old, target) + } + result.Status = "failed" + result.Error = "install cache: " + err.Error() + return result + } + result.Status = "refreshed" + return result +} + +func safeTemplateName(name string) bool { + if name == "" || name == "." || name == ".." { + return false + } + for _, r := range name { + if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') || r == '-' || r == '_' { + continue + } + return false + } + return true +} + +func authenticatedURL(repo, token string) string { + if strings.HasPrefix(repo, "http://") || strings.HasPrefix(repo, "https://") { + u, err := url.Parse(repo) + if err == nil { + u.User = url.UserPassword("oauth2", token) + return u.String() + } + } + u := &url.URL{ + Scheme: "https", + Host: "git.moleculesai.app", + Path: "/" + strings.TrimSuffix(repo, ".git") + ".git", + User: url.UserPassword("oauth2", token), + } + return u.String() +} + +func sanitizeGitError(out []byte, err error, token string) string { + msg := strings.TrimSpace(string(out)) + if msg == "" { + msg = err.Error() + } + if token != "" { + msg = strings.ReplaceAll(msg, token, "***") + } + if len(msg) > 300 { + msg = msg[:300] + } + return msg +} diff --git a/workspace-server/internal/templatecache/cache_test.go b/workspace-server/internal/templatecache/cache_test.go new file mode 100644 index 000000000..caef1a43d --- /dev/null +++ b/workspace-server/internal/templatecache/cache_test.go @@ -0,0 +1,16 @@ +package templatecache + +import "testing" + +func TestSafeTemplateName(t *testing.T) { + for _, name := range []string{"seo-agent", "claude_code", "T4"} { + if !safeTemplateName(name) { + t.Fatalf("%q should be safe", name) + } + } + for _, name := range []string{"", "../seo", "seo/agent", "seo.agent"} { + if safeTemplateName(name) { + t.Fatalf("%q should be rejected", name) + } + } +} -- 2.52.0