feat: refresh workspace templates from repo cache
Block internal-flavored paths / Block forbidden paths (pull_request) Waiting to run
ci-arm64-advisory / fast-checks (pull_request) Waiting to run
E2E API Smoke Test / detect-changes (pull_request) Waiting to run
E2E Chat / detect-changes (pull_request) Waiting to run
E2E Peer Visibility (literal MCP list_peers) / E2E Peer Visibility (local) (pull_request) Waiting to run
E2E Peer Visibility (literal MCP list_peers) / E2E Peer Visibility (pull_request) Waiting to run
E2E Staging Canvas (Playwright) / detect-changes (pull_request) Waiting to run
E2E Staging External Runtime / E2E Staging External Runtime (pull_request) Waiting to run
E2E Staging SaaS (full lifecycle) / pr-validate (pull_request) Waiting to run
E2E Staging SaaS (full lifecycle) / E2E Staging SaaS (pull_request) Waiting to run
Handlers Postgres Integration / detect-changes (pull_request) Waiting to run
Harness Replays / detect-changes (pull_request) Waiting to run
Lint forbidden tenant-env keys / Scan workspace_secrets writers for forbidden env keys (pull_request) Waiting to run
Lint no tenant GITEA or GITHUB token write / Scan for repo-host token write into tenant workspace surface (pull_request) Waiting to run
lint-required-no-paths / lint-required-no-paths (pull_request) Waiting to run
Secret scan / Scan diff for credential-shaped strings (pull_request) Waiting to run
Ops Scripts Tests / Ops scripts (unittest) (pull_request) Waiting to run
gate-check-v3 / gate-check (pull_request) Waiting to run
qa-review / approved (pull_request) Waiting to run
security-review / approved (pull_request) Waiting to run
sop-checklist / all-items-acked (pull_request) Waiting to run
sop-checklist / review-refire (pull_request) Waiting to run
sop-tier-check / tier-check (pull_request) Waiting to run
Lint shellcheck (arm64 pilot) / shellcheck-arm64 (pilot) (pull_request) Successful in 8s
audit-force-merge / audit (pull_request) Successful in 8s
CI / all-required (pull_request) Failing after 41m6s
CI / Platform (Go) (pull_request) Has been skipped
CI / Canvas (Next.js) (pull_request) Has been skipped
CI / Shellcheck (E2E scripts) (pull_request) Has been skipped
CI / Canvas Deploy Reminder (pull_request) Has been skipped
CI / Detect changes (pull_request) Has been cancelled
CI / Python Lint & Test (pull_request) Has been cancelled
E2E API Smoke Test / E2E API Smoke Test (pull_request) Has been cancelled
E2E Chat / E2E Chat (pull_request) Has been cancelled
E2E Staging Canvas (Playwright) / Canvas tabs E2E (pull_request) Has been cancelled
Handlers Postgres Integration / Handlers Postgres Integration (pull_request) Has been cancelled
Harness Replays / Harness Replays (pull_request) Has been cancelled

This commit is contained in:
2026-05-25 12:05:05 -07:00
parent 37972fa635
commit ef8651410d
13 changed files with 488 additions and 83 deletions
+1 -1
View File
@@ -1,5 +1,5 @@
{
"_comment": "OSS surface registry — every repo listed here MUST be public on git.moleculesai.app. Layer-3 customer/private templates are NOT registered here; they are handled at provision-time via the per-tenant credential resolver (see internal#102 RFC). 'main' refs are pinned to tags before broad rollout.",
"_comment": "Platform template registry. Repos may be public or platform-private; CI and runtime template-cache refresh clone them with the SSOT-managed template read token, then strip .git metadata before use. Customer/private tenant templates remain outside this platform manifest. 'main' refs are pinned to tags before broad rollout.",
"version": 1,
"plugins": [
{"name": "browser-automation", "repo": "molecule-ai/molecule-ai-plugin-browser-automation", "ref": "main"},
+4 -13
View File
@@ -8,19 +8,10 @@
# Requires: git, jq (lighter than python3 — ~2MB vs ~50MB in Alpine)
#
# Auth (optional):
# Post-2026-05-08 (#192): every repo in manifest.json is public on
# git.moleculesai.app. Anonymous clone works for the entire registered
# set. The OSS-surface contract is recorded in manifest.json's _comment
# — Layer-3 customer/private templates (e.g. reno-stars) are NOT in the
# manifest; they are handled at provision-time via the per-tenant
# credential resolver (internal#102 RFC).
#
# MOLECULE_GITEA_TOKEN is therefore optional today. Kept supported for
# two reasons: (a) historical CI configs that still inject
# AUTO_SYNC_TOKEN remain harmless, (b) reserved for the case where a
# private internal-only template is later registered via a ci-readonly
# team grant — review must explicitly sign off on that, since it
# violates the public-OSS-surface contract.
# Repos in manifest.json may be public or platform-private. CI and
# operator refresh jobs should set MOLECULE_GITEA_TOKEN to the
# SSOT-managed template read token. Anonymous clone still works for
# public entries, but private platform templates depend on the token.
#
# The token (when set) never enters the Docker image: this script runs
# in the trusted CI context BEFORE `docker buildx build`, populates
+1
View File
@@ -71,6 +71,7 @@ RUN apk add --no-cache ca-certificates docker-cli docker-cli-buildx git tzdata w
COPY --from=builder /platform /platform
COPY --from=builder /memory-plugin /memory-plugin
COPY workspace-server/migrations /migrations
COPY manifest.json /app/manifest.json
# Templates + plugins (pre-cloned by scripts/clone-manifest.sh in the
# trusted CI / operator-host context, .git already stripped). The Gitea
# token used to clone them never enters this image — same shape as
+1
View File
@@ -118,6 +118,7 @@ RUN deluser --remove-home node 2>/dev/null || true; \
COPY --from=go-builder /platform /platform
COPY --from=go-builder /memory-plugin /memory-plugin
COPY workspace-server/migrations /migrations
COPY manifest.json /app/manifest.json
# Templates + plugins (pre-cloned by scripts/clone-manifest.sh in the
# trusted CI / operator-host context, .git already stripped — see
+60 -2
View File
@@ -50,6 +50,7 @@ import (
"git.moleculesai.app/molecule-ai/molecule-core/workspace-server/internal/router"
"git.moleculesai.app/molecule-ai/molecule-core/workspace-server/internal/scheduler"
"git.moleculesai.app/molecule-ai/molecule-core/workspace-server/internal/supervised"
"git.moleculesai.app/molecule-ai/molecule-core/workspace-server/internal/templatecache"
"git.moleculesai.app/molecule-ai/molecule-core/workspace-server/internal/ws"
// External plugins — each registers EnvMutator(s) that run at workspace
@@ -58,6 +59,7 @@ import (
ghidentity "go.moleculesai.app/plugin/gh-identity/pluginloader"
"git.moleculesai.app/molecule-ai/molecule-core/workspace-server/pkg/provisionhook"
"github.com/gin-gonic/gin"
)
func main() {
@@ -193,11 +195,28 @@ func main() {
port := envOr("PORT", "8080")
platformURL := envOr("PLATFORM_URL", fmt.Sprintf("http://host.docker.internal:%s", port))
configsDir := envOr("CONFIGS_DIR", findConfigsDir())
templateCacheDir := envOr("TEMPLATE_CACHE_DIR", filepath.Join(os.TempDir(), "molecule-template-cache"))
manifestPath := findWorkspaceManifestPath()
templateToken := templateCacheToken()
refreshTemplates := func(ctx context.Context) (templatecache.RefreshReport, error) {
return templatecache.RefreshWorkspaceTemplates(ctx, manifestPath, templateCacheDir, templateToken)
}
if shouldRefreshTemplateCache(templateToken, manifestPath) {
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
report, err := refreshTemplates(ctx)
cancel()
if err != nil {
log.Printf("template cache refresh: %v (continuing with baked templates)", err)
} else {
log.Printf("template cache refresh: refreshed %d workspace templates into %s", len(report.Results), templateCacheDir)
}
}
// Init order: wh → onWorkspaceOffline → liveness/healthSweep → router
// WorkspaceHandler is created before the router so RestartByID can be wired into
// the offline callbacks used by both the liveness monitor and the health sweep.
wh := handlers.NewWorkspaceHandler(broadcaster, prov, platformURL, configsDir)
wh := handlers.NewWorkspaceHandler(broadcaster, prov, platformURL, configsDir).
WithTemplateCacheDir(templateCacheDir)
if cpProv != nil {
wh.SetCPProvisioner(cpProv)
}
@@ -377,7 +396,12 @@ func main() {
// require a plugins/ dir on disk (nil in CP/SaaS mode).
pluginRegistry := plugins.NewRegistry()
pluginRegistry.Register(plugins.NewGithubResolver())
r := router.Setup(hub, broadcaster, prov, platformURL, configsDir, wh, channelMgr, memBundle, pluginRegistry)
refreshTemplatesHTTP := func(c *gin.Context) (any, error) {
ctx, cancel := context.WithTimeout(c.Request.Context(), 2*time.Minute)
defer cancel()
return refreshTemplates(ctx)
}
r := router.Setup(hub, broadcaster, prov, platformURL, configsDir, templateCacheDir, wh, channelMgr, memBundle, pluginRegistry, refreshTemplatesHTTP)
// Plugin drift sweeper — periodic detection of upstream plugin version drift
// (core#123). Scans workspace_plugins rows where tracked_ref != 'none',
@@ -493,6 +517,40 @@ func findConfigsDir() string {
return "workspace-configs-templates"
}
func findWorkspaceManifestPath() string {
if v := os.Getenv("WORKSPACE_MANIFEST_PATH"); v != "" {
return v
}
for _, p := range []string{"/app/manifest.json", "manifest.json", "../manifest.json", "../../manifest.json"} {
if abs, err := filepath.Abs(p); err == nil {
if _, err := os.Stat(abs); err == nil {
return abs
}
}
}
return ""
}
func templateCacheToken() string {
for _, key := range []string{"MOLECULE_TEMPLATE_GITEA_TOKEN", "MOLECULE_GITEA_TOKEN"} {
if v := strings.TrimSpace(os.Getenv(key)); v != "" {
return v
}
}
return ""
}
func shouldRefreshTemplateCache(token, manifestPath string) bool {
switch strings.ToLower(strings.TrimSpace(os.Getenv("TEMPLATE_CACHE_REFRESH"))) {
case "0", "false", "off", "no":
return false
case "1", "true", "on", "yes":
return token != "" && manifestPath != ""
default:
return token != "" && manifestPath != ""
}
}
func findMigrationsDir() string {
candidates := []string{
"migrations",
+113 -62
View File
@@ -54,6 +54,7 @@ const maxUploadFiles = 200
type TemplatesHandler struct {
configsDir string
cacheDir string
docker *client.Client
// wh is used by Import and ReplaceFiles to call DefaultTier() so a
// generated config.yaml's tier matches the SaaS-vs-self-hosted
@@ -61,6 +62,11 @@ type TemplatesHandler struct {
// the caller doesn't import templates that need a fresh config
// generated.
wh *WorkspaceHandler
// refreshCache is nil unless main wires a manifest-backed template
// cache refresher. POST /admin/templates/refresh uses this hook so a
// template repo merge can update the tenant catalog without rebuilding
// the full tenant image.
refreshCache func(ctx *gin.Context) (any, error)
}
// NewTemplatesHandler constructs a TemplatesHandler. wh may be nil for
@@ -71,6 +77,16 @@ func NewTemplatesHandler(configsDir string, dockerCli *client.Client, wh *Worksp
return &TemplatesHandler{configsDir: configsDir, docker: dockerCli, wh: wh}
}
func (h *TemplatesHandler) WithCacheDir(cacheDir string) *TemplatesHandler {
h.cacheDir = cacheDir
return h
}
func (h *TemplatesHandler) WithRefreshFunc(fn func(ctx *gin.Context) (any, error)) *TemplatesHandler {
h.refreshCache = fn
return h
}
// modelSpec describes a single supported model on a template: its id (sent
// to the runtime), a human-readable label, and the env vars that must be
// present for that model to work (e.g. API keys).
@@ -161,6 +177,15 @@ type templateSummary struct {
// Only resolves to actual templates (not ws-* dirs since those are now Docker volumes).
// Returns empty string if no matching template is found.
func (h *TemplatesHandler) resolveTemplateDir(wsName string) string {
if h.cacheDir != "" {
nameDir := filepath.Join(h.cacheDir, normalizeName(wsName))
if _, err := os.Stat(nameDir); err == nil {
return nameDir
}
if tmpl := findTemplateByName(h.cacheDir, wsName); tmpl != "" {
return filepath.Join(h.cacheDir, tmpl)
}
}
nameDir := filepath.Join(h.configsDir, normalizeName(wsName))
if _, err := os.Stat(nameDir); err == nil {
return nameDir
@@ -175,78 +200,104 @@ func (h *TemplatesHandler) resolveTemplateDir(wsName string) string {
// List handles GET /templates
func (h *TemplatesHandler) List(c *gin.Context) {
templates := make([]templateSummary, 0)
walkTemplateConfigs(h.configsDir, func(id string, data []byte) {
var raw struct {
Name string `yaml:"name"`
Description string `yaml:"description"`
Tier int `yaml:"tier"`
Runtime string `yaml:"runtime"`
Model string `yaml:"model"`
Skills []string `yaml:"skills"`
// Top-level `providers:` block — structured registry. Distinct
// from runtime_config.providers (slug list) below. Both shapes
// coexist in production: claude-code ships the structured
// registry, hermes still uses the slug list. /templates surfaces
// both verbatim so each runtime owns its taxonomy.
Providers []providerRegistryEntry `yaml:"providers"`
RuntimeConfig struct {
Model string `yaml:"model"`
Models []modelSpec `yaml:"models"`
RequiredEnv []string `yaml:"required_env"`
RecommendedEnv []string `yaml:"recommended_env"`
Providers []string `yaml:"providers"`
ProvisionTimeoutSeconds int `yaml:"provision_timeout_seconds"`
} `yaml:"runtime_config"`
}
if err := yaml.Unmarshal(data, &raw); err != nil {
// Without this log a malformed config.yaml causes the
// template to silently disappear from /templates with no
// trace — the operator can't tell "excluded due to parse
// error" from "never existed." That matters more now that
// templates ship richer YAML shapes (top-level providers
// registry, models[] with required_env, etc.) where a
// type-shape mismatch on one field drops the whole entry.
log.Printf("templates list: skip %s: yaml.Unmarshal: %v", id, err)
return
}
runtime := strings.TrimSuffix(strings.TrimSpace(raw.Runtime), "-default")
if _, ok := knownRuntimes[runtime]; !ok {
log.Printf("templates list: skip %s: unsupported runtime %q", id, raw.Runtime)
seen := map[string]struct{}{}
walk := func(root string) {
if root == "" {
return
}
walkTemplateConfigs(root, func(id string, data []byte) {
if _, ok := seen[id]; ok {
return
}
seen[id] = struct{}{}
var raw struct {
Name string `yaml:"name"`
Description string `yaml:"description"`
Tier int `yaml:"tier"`
Runtime string `yaml:"runtime"`
Model string `yaml:"model"`
Skills []string `yaml:"skills"`
// Top-level `providers:` block — structured registry. Distinct
// from runtime_config.providers (slug list) below. Both shapes
// coexist in production: claude-code ships the structured
// registry, hermes still uses the slug list. /templates surfaces
// both verbatim so each runtime owns its taxonomy.
Providers []providerRegistryEntry `yaml:"providers"`
RuntimeConfig struct {
Model string `yaml:"model"`
Models []modelSpec `yaml:"models"`
RequiredEnv []string `yaml:"required_env"`
RecommendedEnv []string `yaml:"recommended_env"`
Providers []string `yaml:"providers"`
ProvisionTimeoutSeconds int `yaml:"provision_timeout_seconds"`
} `yaml:"runtime_config"`
}
if err := yaml.Unmarshal(data, &raw); err != nil {
// Without this log a malformed config.yaml causes the
// template to silently disappear from /templates with no
// trace — the operator can't tell "excluded due to parse
// error" from "never existed." That matters more now that
// templates ship richer YAML shapes (top-level providers
// registry, models[] with required_env, etc.) where a
// type-shape mismatch on one field drops the whole entry.
log.Printf("templates list: skip %s: yaml.Unmarshal: %v", id, err)
return
}
runtime := strings.TrimSuffix(strings.TrimSpace(raw.Runtime), "-default")
if _, ok := knownRuntimes[runtime]; !ok {
log.Printf("templates list: skip %s: unsupported runtime %q", id, raw.Runtime)
return
}
// Model comes from either top-level (legacy) or runtime_config.model (current).
model := raw.Model
if model == "" {
model = raw.RuntimeConfig.Model
}
// Model comes from either top-level (legacy) or runtime_config.model (current).
model := raw.Model
if model == "" {
model = raw.RuntimeConfig.Model
}
tier := raw.Tier
if h.wh != nil && h.wh.IsSaaS() {
tier = h.wh.DefaultTier()
}
tier := raw.Tier
if h.wh != nil && h.wh.IsSaaS() {
tier = h.wh.DefaultTier()
}
templates = append(templates, templateSummary{
ID: id,
Name: raw.Name,
Description: raw.Description,
Tier: tier,
Runtime: raw.Runtime,
Model: model,
Models: raw.RuntimeConfig.Models,
RequiredEnv: raw.RuntimeConfig.RequiredEnv,
RecommendedEnv: raw.RuntimeConfig.RecommendedEnv,
Providers: raw.RuntimeConfig.Providers,
ProviderRegistry: raw.Providers,
Skills: raw.Skills,
SkillCount: len(raw.Skills),
ProvisionTimeoutSeconds: raw.RuntimeConfig.ProvisionTimeoutSeconds,
templates = append(templates, templateSummary{
ID: id,
Name: raw.Name,
Description: raw.Description,
Tier: tier,
Runtime: raw.Runtime,
Model: model,
Models: raw.RuntimeConfig.Models,
RequiredEnv: raw.RuntimeConfig.RequiredEnv,
RecommendedEnv: raw.RuntimeConfig.RecommendedEnv,
Providers: raw.RuntimeConfig.Providers,
ProviderRegistry: raw.Providers,
Skills: raw.Skills,
SkillCount: len(raw.Skills),
ProvisionTimeoutSeconds: raw.RuntimeConfig.ProvisionTimeoutSeconds,
})
})
})
}
walk(h.cacheDir)
walk(h.configsDir)
c.JSON(http.StatusOK, templates)
}
// RefreshCache handles POST /admin/templates/refresh.
func (h *TemplatesHandler) RefreshCache(c *gin.Context) {
if h.refreshCache == nil {
c.JSON(http.StatusServiceUnavailable, gin.H{"error": "template cache refresh is not configured"})
return
}
result, err := h.refreshCache(c)
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
return
}
c.JSON(http.StatusOK, result)
}
// ListFiles handles GET /workspaces/:id/files
// Lists files inside the running container's /configs directory (or /workspace, etc.).
// Falls back to host-side config templates directory when container isn't running.
@@ -133,6 +133,71 @@ skills:
}
}
func TestTemplatesList_CacheOverridesBakedTemplate(t *testing.T) {
setupTestDB(t)
setupTestRedis(t)
bakedDir := t.TempDir()
cacheDir := t.TempDir()
mustWriteTemplate := func(root, id, body string) {
t.Helper()
dir := filepath.Join(root, id)
if err := os.MkdirAll(dir, 0755); err != nil {
t.Fatalf("mkdir %s: %v", dir, err)
}
if err := os.WriteFile(filepath.Join(dir, "config.yaml"), []byte(body), 0644); err != nil {
t.Fatalf("write config: %v", err)
}
}
mustWriteTemplate(bakedDir, "seo-agent", `name: SEO Agent
description: stale
tier: 4
runtime: claude-code
model: old
runtime_config:
recommended_env: [TELEGRAM_BOT_TOKEN]
skills: []
`)
mustWriteTemplate(cacheDir, "seo-agent", `name: SEO Agent
description: fresh
tier: 4
runtime: claude-code
model: moonshot/kimi-k2.6
runtime_config:
required_env: [TENANT_NAME]
recommended_env: [GOOGLE_GSC_SITE]
skills: []
`)
handler := NewTemplatesHandler(bakedDir, nil, nil).WithCacheDir(cacheDir)
w := httptest.NewRecorder()
c, _ := gin.CreateTestContext(w)
c.Request = httptest.NewRequest("GET", "/templates", nil)
handler.List(c)
if w.Code != http.StatusOK {
t.Fatalf("expected 200, got %d", w.Code)
}
var resp []templateSummary
if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
t.Fatalf("parse: %v", err)
}
if len(resp) != 1 {
t.Fatalf("expected 1 template, got %d", len(resp))
}
if resp[0].Description != "fresh" {
t.Fatalf("cache template should override baked copy, got description %q", resp[0].Description)
}
if !reflect.DeepEqual(resp[0].RequiredEnv, []string{"TENANT_NAME"}) {
t.Fatalf("RequiredEnv = %+v", resp[0].RequiredEnv)
}
if reflect.DeepEqual(resp[0].RecommendedEnv, []string{"TELEGRAM_BOT_TOKEN"}) {
t.Fatalf("stale baked recommended_env leaked through: %+v", resp[0].RecommendedEnv)
}
}
func TestTemplatesList_RuntimeAndModelsRegistry(t *testing.T) {
setupTestDB(t)
setupTestRedis(t)
@@ -56,6 +56,7 @@ type WorkspaceHandler struct {
cpProv provisioner.CPProvisionerAPI
platformURL string
configsDir string // path to workspace-configs-templates/ (for reading templates)
cacheDir string // optional runtime-refreshed template cache; overrides configsDir by template id
// envMutators runs registered EnvMutator plugins right before
// container Start, after built-in secret loads. Nil = no plugins
// registered; Registry.Run handles a nil receiver as a no-op so the
@@ -183,6 +184,11 @@ func NewWorkspaceHandler(b events.EventEmitter, p *provisioner.Provisioner, plat
return h
}
func (h *WorkspaceHandler) WithTemplateCacheDir(cacheDir string) *WorkspaceHandler {
h.cacheDir = cacheDir
return h
}
// WithNamespaceCleanup wires the I5 hook (RFC #2728) so workspace
// purge can drop the plugin's `workspace:<id>` namespace. main.go
// passes a closure over plugin.DeleteNamespace; tests pass a stub.
@@ -285,7 +291,7 @@ func (h *WorkspaceHandler) Create(c *gin.Context) {
// #226: payload.Template is attacker-controllable. resolveInsideRoot
// rejects absolute paths and any ".." that escapes configsDir so the
// provisioner can't be pointed at host directories.
candidatePath, resolveErr := resolveInsideRoot(h.configsDir, payload.Template)
candidatePath, resolveErr := resolveWorkspaceTemplatePath(h.configsDir, h.cacheDir, payload.Template)
if resolveErr != nil {
log.Printf("Create: invalid template path %q: %v", payload.Template, resolveErr)
c.JSON(http.StatusBadRequest, gin.H{"error": "invalid template"})
@@ -726,7 +732,7 @@ func (h *WorkspaceHandler) Create(c *gin.Context) {
var templatePath string
var configFiles map[string][]byte
if payload.Template != "" {
candidatePath, resolveErr := resolveInsideRoot(h.configsDir, payload.Template)
candidatePath, resolveErr := resolveWorkspaceTemplatePath(h.configsDir, h.cacheDir, payload.Template)
if resolveErr != nil {
log.Printf("Create provision: rejecting template %q: %v", payload.Template, resolveErr)
return
@@ -485,6 +485,17 @@ func findTemplateByName(configsDir, name string) string {
return ""
}
func resolveWorkspaceTemplatePath(configsDir, cacheDir, template string) (string, error) {
if cacheDir != "" {
if p, err := resolveInsideRoot(cacheDir, template); err != nil {
return "", err
} else if _, statErr := os.Stat(p); statErr == nil {
return p, nil
}
}
return resolveInsideRoot(configsDir, template)
}
// resolveOrgTemplate looks for a matching role directory under
// configsDir/org-templates/ and returns the absolute path and a short label
// ("org-templates/<dir>"). Used by the restart handler's rebuild_config path
@@ -658,7 +669,7 @@ func (h *WorkspaceHandler) defaultTemplateProvidersYAML(runtime string) string {
return ""
}
templateName := runtime + "-default"
templatePath, err := resolveInsideRoot(h.configsDir, templateName)
templatePath, err := resolveWorkspaceTemplatePath(h.configsDir, h.cacheDir, templateName)
if err != nil {
log.Printf("Provisioner: default template providers skipped for runtime %s: %v", runtime, err)
return ""
@@ -110,6 +110,32 @@ func TestFindTemplateByName_NotFound(t *testing.T) {
}
}
func TestResolveWorkspaceTemplatePath_PrefersCache(t *testing.T) {
bakedDir := t.TempDir()
cacheDir := t.TempDir()
for _, root := range []string{bakedDir, cacheDir} {
if err := os.MkdirAll(filepath.Join(root, "seo-agent"), 0755); err != nil {
t.Fatalf("mkdir: %v", err)
}
}
got, err := resolveWorkspaceTemplatePath(bakedDir, cacheDir, "seo-agent")
if err != nil {
t.Fatalf("resolveWorkspaceTemplatePath: %v", err)
}
want := filepath.Join(cacheDir, "seo-agent")
if got != want {
t.Fatalf("want cache path %q, got %q", want, got)
}
}
func TestResolveWorkspaceTemplatePath_RejectsTraversal(t *testing.T) {
if _, err := resolveWorkspaceTemplatePath(t.TempDir(), t.TempDir(), "../seo-agent"); err == nil {
t.Fatal("expected traversal to be rejected")
}
}
func TestFindTemplateByName_SkipsWsPrefix(t *testing.T) {
tmpDir := t.TempDir()
+5 -2
View File
@@ -36,7 +36,7 @@ import (
// (main.go) gets the same pluginResolver instance so it can share scheme
// enumeration if a deployment registers extra schemes externally. A nil
// pluginResolver is harmless: plgh still works with its built-in defaults.
func Setup(hub *ws.Hub, broadcaster *events.Broadcaster, prov *provisioner.Provisioner, platformURL, configsDir string, wh *handlers.WorkspaceHandler, channelMgr *channels.Manager, memBundle *memwiring.Bundle, pluginResolver plugins.PluginResolver) *gin.Engine {
func Setup(hub *ws.Hub, broadcaster *events.Broadcaster, prov *provisioner.Provisioner, platformURL, configsDir string, templateCacheDir string, wh *handlers.WorkspaceHandler, channelMgr *channels.Manager, memBundle *memwiring.Bundle, pluginResolver plugins.PluginResolver, refreshTemplates func(ctx *gin.Context) (any, error)) *gin.Engine {
r := gin.Default()
// Issue #179 — trust no reverse-proxy headers. Without this call Gin's
@@ -666,7 +666,9 @@ func Setup(hub *ws.Hub, broadcaster *events.Broadcaster, prov *provisioner.Provi
// Templates — wh threaded so generateDefaultConfig picks the
// SaaS-aware default tier in Import + ReplaceFiles (#2910 PR-B).
tmplh := handlers.NewTemplatesHandler(configsDir, dockerCli, wh)
tmplh := handlers.NewTemplatesHandler(configsDir, dockerCli, wh).
WithCacheDir(templateCacheDir).
WithRefreshFunc(refreshTemplates)
// #686: GET /templates lists all template names+metadata from configsDir.
// Open access lets unauthenticated callers enumerate org configurations and
// installed plugins. AdminAuth-gate it alongside POST /templates/import.
@@ -676,6 +678,7 @@ func Setup(hub *ws.Hub, broadcaster *events.Broadcaster, prov *provisioner.Provi
tmplAdmin := r.Group("", middleware.AdminAuth(db.DB))
tmplAdmin.GET("/templates", tmplh.List)
tmplAdmin.POST("/templates/import", tmplh.Import)
tmplAdmin.POST("/admin/templates/refresh", tmplh.RefreshCache)
}
wsAuth.PUT("/files", tmplh.ReplaceFiles)
wsAuth.GET("/files", tmplh.ListFiles)
@@ -0,0 +1,176 @@
package templatecache
import (
"context"
"encoding/json"
"fmt"
"net/url"
"os"
"os/exec"
"path/filepath"
"strings"
"time"
)
type ManifestEntry struct {
Name string `json:"name"`
Repo string `json:"repo"`
Ref string `json:"ref"`
}
type manifestFile struct {
WorkspaceTemplates []ManifestEntry `json:"workspace_templates"`
}
type TemplateResult struct {
Name string `json:"name"`
Repo string `json:"repo"`
Ref string `json:"ref"`
SHA string `json:"sha,omitempty"`
Status string `json:"status"`
Error string `json:"error,omitempty"`
}
type RefreshReport struct {
ManifestPath string `json:"manifest_path"`
CacheDir string `json:"cache_dir"`
RefreshedAt time.Time `json:"refreshed_at"`
Results []TemplateResult `json:"results"`
}
func RefreshWorkspaceTemplates(ctx context.Context, manifestPath, cacheDir, token string) (RefreshReport, error) {
report := RefreshReport{
ManifestPath: manifestPath,
CacheDir: cacheDir,
RefreshedAt: time.Now().UTC(),
}
if strings.TrimSpace(token) == "" {
return report, fmt.Errorf("template cache refresh requires MOLECULE_TEMPLATE_GITEA_TOKEN or MOLECULE_GITEA_TOKEN")
}
data, err := os.ReadFile(manifestPath)
if err != nil {
return report, fmt.Errorf("read manifest: %w", err)
}
var manifest manifestFile
if err := json.Unmarshal(data, &manifest); err != nil {
return report, fmt.Errorf("parse manifest: %w", err)
}
if err := os.MkdirAll(cacheDir, 0o755); err != nil {
return report, fmt.Errorf("mkdir cache: %w", err)
}
for _, entry := range manifest.WorkspaceTemplates {
result := refreshOne(ctx, cacheDir, token, entry)
report.Results = append(report.Results, result)
}
return report, nil
}
func refreshOne(ctx context.Context, cacheDir, token string, entry ManifestEntry) TemplateResult {
result := TemplateResult{Name: entry.Name, Repo: entry.Repo, Ref: entry.Ref}
if result.Ref == "" {
result.Ref = "main"
}
if !safeTemplateName(entry.Name) {
result.Status = "skipped"
result.Error = "invalid template name"
return result
}
if strings.TrimSpace(entry.Repo) == "" {
result.Status = "skipped"
result.Error = "missing repo"
return result
}
tmp, err := os.MkdirTemp(cacheDir, ".tmp-"+entry.Name+"-")
if err != nil {
result.Status = "failed"
result.Error = err.Error()
return result
}
defer os.RemoveAll(tmp)
cloneURL := authenticatedURL(entry.Repo, token)
for _, args := range [][]string{
{"init", "-q", tmp},
{"-C", tmp, "remote", "add", "origin", cloneURL},
{"-C", tmp, "fetch", "--depth=1", "-q", "origin", result.Ref},
{"-C", tmp, "checkout", "-q", "--detach", "FETCH_HEAD"},
} {
cmd := exec.CommandContext(ctx, "git", args...)
if out, err := cmd.CombinedOutput(); err != nil {
result.Status = "failed"
result.Error = sanitizeGitError(out, err, token)
return result
}
}
shaCmd := exec.CommandContext(ctx, "git", "-C", tmp, "rev-parse", "HEAD")
if out, err := shaCmd.Output(); err == nil {
result.SHA = strings.TrimSpace(string(out))
}
_ = os.RemoveAll(filepath.Join(tmp, ".git"))
target := filepath.Join(cacheDir, entry.Name)
old := filepath.Join(cacheDir, ".old-"+entry.Name+"-"+fmt.Sprint(time.Now().UnixNano()))
if _, err := os.Stat(target); err == nil {
if err := os.Rename(target, old); err != nil {
result.Status = "failed"
result.Error = "replace old cache: " + err.Error()
return result
}
defer os.RemoveAll(old)
}
if err := os.Rename(tmp, target); err != nil {
if old != "" {
_ = os.Rename(old, target)
}
result.Status = "failed"
result.Error = "install cache: " + err.Error()
return result
}
result.Status = "refreshed"
return result
}
func safeTemplateName(name string) bool {
if name == "" || name == "." || name == ".." {
return false
}
for _, r := range name {
if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') || r == '-' || r == '_' {
continue
}
return false
}
return true
}
func authenticatedURL(repo, token string) string {
if strings.HasPrefix(repo, "http://") || strings.HasPrefix(repo, "https://") {
u, err := url.Parse(repo)
if err == nil {
u.User = url.UserPassword("oauth2", token)
return u.String()
}
}
u := &url.URL{
Scheme: "https",
Host: "git.moleculesai.app",
Path: "/" + strings.TrimSuffix(repo, ".git") + ".git",
User: url.UserPassword("oauth2", token),
}
return u.String()
}
func sanitizeGitError(out []byte, err error, token string) string {
msg := strings.TrimSpace(string(out))
if msg == "" {
msg = err.Error()
}
if token != "" {
msg = strings.ReplaceAll(msg, token, "***")
}
if len(msg) > 300 {
msg = msg[:300]
}
return msg
}
@@ -0,0 +1,16 @@
package templatecache
import "testing"
func TestSafeTemplateName(t *testing.T) {
for _, name := range []string{"seo-agent", "claude_code", "T4"} {
if !safeTemplateName(name) {
t.Fatalf("%q should be safe", name)
}
}
for _, name := range []string{"", "../seo", "seo/agent", "seo.agent"} {
if safeTemplateName(name) {
t.Fatalf("%q should be rejected", name)
}
}
}