feat: refresh workspace templates from repo cache
Block internal-flavored paths / Block forbidden paths (pull_request) Waiting to run
ci-arm64-advisory / fast-checks (pull_request) Waiting to run
E2E API Smoke Test / detect-changes (pull_request) Waiting to run
E2E Chat / detect-changes (pull_request) Waiting to run
E2E Peer Visibility (literal MCP list_peers) / E2E Peer Visibility (local) (pull_request) Waiting to run
E2E Peer Visibility (literal MCP list_peers) / E2E Peer Visibility (pull_request) Waiting to run
E2E Staging Canvas (Playwright) / detect-changes (pull_request) Waiting to run
E2E Staging External Runtime / E2E Staging External Runtime (pull_request) Waiting to run
E2E Staging SaaS (full lifecycle) / pr-validate (pull_request) Waiting to run
E2E Staging SaaS (full lifecycle) / E2E Staging SaaS (pull_request) Waiting to run
Handlers Postgres Integration / detect-changes (pull_request) Waiting to run
Harness Replays / detect-changes (pull_request) Waiting to run
Lint forbidden tenant-env keys / Scan workspace_secrets writers for forbidden env keys (pull_request) Waiting to run
Lint no tenant GITEA or GITHUB token write / Scan for repo-host token write into tenant workspace surface (pull_request) Waiting to run
lint-required-no-paths / lint-required-no-paths (pull_request) Waiting to run
Secret scan / Scan diff for credential-shaped strings (pull_request) Waiting to run
Ops Scripts Tests / Ops scripts (unittest) (pull_request) Waiting to run
gate-check-v3 / gate-check (pull_request) Waiting to run
qa-review / approved (pull_request) Waiting to run
security-review / approved (pull_request) Waiting to run
sop-checklist / all-items-acked (pull_request) Waiting to run
sop-checklist / review-refire (pull_request) Waiting to run
sop-tier-check / tier-check (pull_request) Waiting to run
Lint shellcheck (arm64 pilot) / shellcheck-arm64 (pilot) (pull_request) Successful in 8s
audit-force-merge / audit (pull_request) Successful in 8s
CI / all-required (pull_request) Failing after 41m6s
CI / Platform (Go) (pull_request) Has been skipped
CI / Canvas (Next.js) (pull_request) Has been skipped
CI / Shellcheck (E2E scripts) (pull_request) Has been skipped
CI / Canvas Deploy Reminder (pull_request) Has been skipped
CI / Detect changes (pull_request) Has been cancelled
CI / Python Lint & Test (pull_request) Has been cancelled
E2E API Smoke Test / E2E API Smoke Test (pull_request) Has been cancelled
E2E Chat / E2E Chat (pull_request) Has been cancelled
E2E Staging Canvas (Playwright) / Canvas tabs E2E (pull_request) Has been cancelled
Handlers Postgres Integration / Handlers Postgres Integration (pull_request) Has been cancelled
Harness Replays / Harness Replays (pull_request) Has been cancelled
Block internal-flavored paths / Block forbidden paths (pull_request) Waiting to run
ci-arm64-advisory / fast-checks (pull_request) Waiting to run
E2E API Smoke Test / detect-changes (pull_request) Waiting to run
E2E Chat / detect-changes (pull_request) Waiting to run
E2E Peer Visibility (literal MCP list_peers) / E2E Peer Visibility (local) (pull_request) Waiting to run
E2E Peer Visibility (literal MCP list_peers) / E2E Peer Visibility (pull_request) Waiting to run
E2E Staging Canvas (Playwright) / detect-changes (pull_request) Waiting to run
E2E Staging External Runtime / E2E Staging External Runtime (pull_request) Waiting to run
E2E Staging SaaS (full lifecycle) / pr-validate (pull_request) Waiting to run
E2E Staging SaaS (full lifecycle) / E2E Staging SaaS (pull_request) Waiting to run
Handlers Postgres Integration / detect-changes (pull_request) Waiting to run
Harness Replays / detect-changes (pull_request) Waiting to run
Lint forbidden tenant-env keys / Scan workspace_secrets writers for forbidden env keys (pull_request) Waiting to run
Lint no tenant GITEA or GITHUB token write / Scan for repo-host token write into tenant workspace surface (pull_request) Waiting to run
lint-required-no-paths / lint-required-no-paths (pull_request) Waiting to run
Secret scan / Scan diff for credential-shaped strings (pull_request) Waiting to run
Ops Scripts Tests / Ops scripts (unittest) (pull_request) Waiting to run
gate-check-v3 / gate-check (pull_request) Waiting to run
qa-review / approved (pull_request) Waiting to run
security-review / approved (pull_request) Waiting to run
sop-checklist / all-items-acked (pull_request) Waiting to run
sop-checklist / review-refire (pull_request) Waiting to run
sop-tier-check / tier-check (pull_request) Waiting to run
Lint shellcheck (arm64 pilot) / shellcheck-arm64 (pilot) (pull_request) Successful in 8s
audit-force-merge / audit (pull_request) Successful in 8s
CI / all-required (pull_request) Failing after 41m6s
CI / Platform (Go) (pull_request) Has been skipped
CI / Canvas (Next.js) (pull_request) Has been skipped
CI / Shellcheck (E2E scripts) (pull_request) Has been skipped
CI / Canvas Deploy Reminder (pull_request) Has been skipped
CI / Detect changes (pull_request) Has been cancelled
CI / Python Lint & Test (pull_request) Has been cancelled
E2E API Smoke Test / E2E API Smoke Test (pull_request) Has been cancelled
E2E Chat / E2E Chat (pull_request) Has been cancelled
E2E Staging Canvas (Playwright) / Canvas tabs E2E (pull_request) Has been cancelled
Handlers Postgres Integration / Handlers Postgres Integration (pull_request) Has been cancelled
Harness Replays / Harness Replays (pull_request) Has been cancelled
This commit is contained in:
+1
-1
@@ -1,5 +1,5 @@
|
||||
{
|
||||
"_comment": "OSS surface registry — every repo listed here MUST be public on git.moleculesai.app. Layer-3 customer/private templates are NOT registered here; they are handled at provision-time via the per-tenant credential resolver (see internal#102 RFC). 'main' refs are pinned to tags before broad rollout.",
|
||||
"_comment": "Platform template registry. Repos may be public or platform-private; CI and runtime template-cache refresh clone them with the SSOT-managed template read token, then strip .git metadata before use. Customer/private tenant templates remain outside this platform manifest. 'main' refs are pinned to tags before broad rollout.",
|
||||
"version": 1,
|
||||
"plugins": [
|
||||
{"name": "browser-automation", "repo": "molecule-ai/molecule-ai-plugin-browser-automation", "ref": "main"},
|
||||
|
||||
@@ -8,19 +8,10 @@
|
||||
# Requires: git, jq (lighter than python3 — ~2MB vs ~50MB in Alpine)
|
||||
#
|
||||
# Auth (optional):
|
||||
# Post-2026-05-08 (#192): every repo in manifest.json is public on
|
||||
# git.moleculesai.app. Anonymous clone works for the entire registered
|
||||
# set. The OSS-surface contract is recorded in manifest.json's _comment
|
||||
# — Layer-3 customer/private templates (e.g. reno-stars) are NOT in the
|
||||
# manifest; they are handled at provision-time via the per-tenant
|
||||
# credential resolver (internal#102 RFC).
|
||||
#
|
||||
# MOLECULE_GITEA_TOKEN is therefore optional today. Kept supported for
|
||||
# two reasons: (a) historical CI configs that still inject
|
||||
# AUTO_SYNC_TOKEN remain harmless, (b) reserved for the case where a
|
||||
# private internal-only template is later registered via a ci-readonly
|
||||
# team grant — review must explicitly sign off on that, since it
|
||||
# violates the public-OSS-surface contract.
|
||||
# Repos in manifest.json may be public or platform-private. CI and
|
||||
# operator refresh jobs should set MOLECULE_GITEA_TOKEN to the
|
||||
# SSOT-managed template read token. Anonymous clone still works for
|
||||
# public entries, but private platform templates depend on the token.
|
||||
#
|
||||
# The token (when set) never enters the Docker image: this script runs
|
||||
# in the trusted CI context BEFORE `docker buildx build`, populates
|
||||
|
||||
@@ -71,6 +71,7 @@ RUN apk add --no-cache ca-certificates docker-cli docker-cli-buildx git tzdata w
|
||||
COPY --from=builder /platform /platform
|
||||
COPY --from=builder /memory-plugin /memory-plugin
|
||||
COPY workspace-server/migrations /migrations
|
||||
COPY manifest.json /app/manifest.json
|
||||
# Templates + plugins (pre-cloned by scripts/clone-manifest.sh in the
|
||||
# trusted CI / operator-host context, .git already stripped). The Gitea
|
||||
# token used to clone them never enters this image — same shape as
|
||||
|
||||
@@ -118,6 +118,7 @@ RUN deluser --remove-home node 2>/dev/null || true; \
|
||||
COPY --from=go-builder /platform /platform
|
||||
COPY --from=go-builder /memory-plugin /memory-plugin
|
||||
COPY workspace-server/migrations /migrations
|
||||
COPY manifest.json /app/manifest.json
|
||||
|
||||
# Templates + plugins (pre-cloned by scripts/clone-manifest.sh in the
|
||||
# trusted CI / operator-host context, .git already stripped — see
|
||||
|
||||
@@ -50,6 +50,7 @@ import (
|
||||
"git.moleculesai.app/molecule-ai/molecule-core/workspace-server/internal/router"
|
||||
"git.moleculesai.app/molecule-ai/molecule-core/workspace-server/internal/scheduler"
|
||||
"git.moleculesai.app/molecule-ai/molecule-core/workspace-server/internal/supervised"
|
||||
"git.moleculesai.app/molecule-ai/molecule-core/workspace-server/internal/templatecache"
|
||||
"git.moleculesai.app/molecule-ai/molecule-core/workspace-server/internal/ws"
|
||||
|
||||
// External plugins — each registers EnvMutator(s) that run at workspace
|
||||
@@ -58,6 +59,7 @@ import (
|
||||
ghidentity "go.moleculesai.app/plugin/gh-identity/pluginloader"
|
||||
|
||||
"git.moleculesai.app/molecule-ai/molecule-core/workspace-server/pkg/provisionhook"
|
||||
"github.com/gin-gonic/gin"
|
||||
)
|
||||
|
||||
func main() {
|
||||
@@ -193,11 +195,28 @@ func main() {
|
||||
port := envOr("PORT", "8080")
|
||||
platformURL := envOr("PLATFORM_URL", fmt.Sprintf("http://host.docker.internal:%s", port))
|
||||
configsDir := envOr("CONFIGS_DIR", findConfigsDir())
|
||||
templateCacheDir := envOr("TEMPLATE_CACHE_DIR", filepath.Join(os.TempDir(), "molecule-template-cache"))
|
||||
manifestPath := findWorkspaceManifestPath()
|
||||
templateToken := templateCacheToken()
|
||||
refreshTemplates := func(ctx context.Context) (templatecache.RefreshReport, error) {
|
||||
return templatecache.RefreshWorkspaceTemplates(ctx, manifestPath, templateCacheDir, templateToken)
|
||||
}
|
||||
if shouldRefreshTemplateCache(templateToken, manifestPath) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
|
||||
report, err := refreshTemplates(ctx)
|
||||
cancel()
|
||||
if err != nil {
|
||||
log.Printf("template cache refresh: %v (continuing with baked templates)", err)
|
||||
} else {
|
||||
log.Printf("template cache refresh: refreshed %d workspace templates into %s", len(report.Results), templateCacheDir)
|
||||
}
|
||||
}
|
||||
|
||||
// Init order: wh → onWorkspaceOffline → liveness/healthSweep → router
|
||||
// WorkspaceHandler is created before the router so RestartByID can be wired into
|
||||
// the offline callbacks used by both the liveness monitor and the health sweep.
|
||||
wh := handlers.NewWorkspaceHandler(broadcaster, prov, platformURL, configsDir)
|
||||
wh := handlers.NewWorkspaceHandler(broadcaster, prov, platformURL, configsDir).
|
||||
WithTemplateCacheDir(templateCacheDir)
|
||||
if cpProv != nil {
|
||||
wh.SetCPProvisioner(cpProv)
|
||||
}
|
||||
@@ -377,7 +396,12 @@ func main() {
|
||||
// require a plugins/ dir on disk (nil in CP/SaaS mode).
|
||||
pluginRegistry := plugins.NewRegistry()
|
||||
pluginRegistry.Register(plugins.NewGithubResolver())
|
||||
r := router.Setup(hub, broadcaster, prov, platformURL, configsDir, wh, channelMgr, memBundle, pluginRegistry)
|
||||
refreshTemplatesHTTP := func(c *gin.Context) (any, error) {
|
||||
ctx, cancel := context.WithTimeout(c.Request.Context(), 2*time.Minute)
|
||||
defer cancel()
|
||||
return refreshTemplates(ctx)
|
||||
}
|
||||
r := router.Setup(hub, broadcaster, prov, platformURL, configsDir, templateCacheDir, wh, channelMgr, memBundle, pluginRegistry, refreshTemplatesHTTP)
|
||||
|
||||
// Plugin drift sweeper — periodic detection of upstream plugin version drift
|
||||
// (core#123). Scans workspace_plugins rows where tracked_ref != 'none',
|
||||
@@ -493,6 +517,40 @@ func findConfigsDir() string {
|
||||
return "workspace-configs-templates"
|
||||
}
|
||||
|
||||
func findWorkspaceManifestPath() string {
|
||||
if v := os.Getenv("WORKSPACE_MANIFEST_PATH"); v != "" {
|
||||
return v
|
||||
}
|
||||
for _, p := range []string{"/app/manifest.json", "manifest.json", "../manifest.json", "../../manifest.json"} {
|
||||
if abs, err := filepath.Abs(p); err == nil {
|
||||
if _, err := os.Stat(abs); err == nil {
|
||||
return abs
|
||||
}
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func templateCacheToken() string {
|
||||
for _, key := range []string{"MOLECULE_TEMPLATE_GITEA_TOKEN", "MOLECULE_GITEA_TOKEN"} {
|
||||
if v := strings.TrimSpace(os.Getenv(key)); v != "" {
|
||||
return v
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func shouldRefreshTemplateCache(token, manifestPath string) bool {
|
||||
switch strings.ToLower(strings.TrimSpace(os.Getenv("TEMPLATE_CACHE_REFRESH"))) {
|
||||
case "0", "false", "off", "no":
|
||||
return false
|
||||
case "1", "true", "on", "yes":
|
||||
return token != "" && manifestPath != ""
|
||||
default:
|
||||
return token != "" && manifestPath != ""
|
||||
}
|
||||
}
|
||||
|
||||
func findMigrationsDir() string {
|
||||
candidates := []string{
|
||||
"migrations",
|
||||
|
||||
@@ -54,6 +54,7 @@ const maxUploadFiles = 200
|
||||
|
||||
type TemplatesHandler struct {
|
||||
configsDir string
|
||||
cacheDir string
|
||||
docker *client.Client
|
||||
// wh is used by Import and ReplaceFiles to call DefaultTier() so a
|
||||
// generated config.yaml's tier matches the SaaS-vs-self-hosted
|
||||
@@ -61,6 +62,11 @@ type TemplatesHandler struct {
|
||||
// the caller doesn't import templates that need a fresh config
|
||||
// generated.
|
||||
wh *WorkspaceHandler
|
||||
// refreshCache is nil unless main wires a manifest-backed template
|
||||
// cache refresher. POST /admin/templates/refresh uses this hook so a
|
||||
// template repo merge can update the tenant catalog without rebuilding
|
||||
// the full tenant image.
|
||||
refreshCache func(ctx *gin.Context) (any, error)
|
||||
}
|
||||
|
||||
// NewTemplatesHandler constructs a TemplatesHandler. wh may be nil for
|
||||
@@ -71,6 +77,16 @@ func NewTemplatesHandler(configsDir string, dockerCli *client.Client, wh *Worksp
|
||||
return &TemplatesHandler{configsDir: configsDir, docker: dockerCli, wh: wh}
|
||||
}
|
||||
|
||||
func (h *TemplatesHandler) WithCacheDir(cacheDir string) *TemplatesHandler {
|
||||
h.cacheDir = cacheDir
|
||||
return h
|
||||
}
|
||||
|
||||
func (h *TemplatesHandler) WithRefreshFunc(fn func(ctx *gin.Context) (any, error)) *TemplatesHandler {
|
||||
h.refreshCache = fn
|
||||
return h
|
||||
}
|
||||
|
||||
// modelSpec describes a single supported model on a template: its id (sent
|
||||
// to the runtime), a human-readable label, and the env vars that must be
|
||||
// present for that model to work (e.g. API keys).
|
||||
@@ -161,6 +177,15 @@ type templateSummary struct {
|
||||
// Only resolves to actual templates (not ws-* dirs since those are now Docker volumes).
|
||||
// Returns empty string if no matching template is found.
|
||||
func (h *TemplatesHandler) resolveTemplateDir(wsName string) string {
|
||||
if h.cacheDir != "" {
|
||||
nameDir := filepath.Join(h.cacheDir, normalizeName(wsName))
|
||||
if _, err := os.Stat(nameDir); err == nil {
|
||||
return nameDir
|
||||
}
|
||||
if tmpl := findTemplateByName(h.cacheDir, wsName); tmpl != "" {
|
||||
return filepath.Join(h.cacheDir, tmpl)
|
||||
}
|
||||
}
|
||||
nameDir := filepath.Join(h.configsDir, normalizeName(wsName))
|
||||
if _, err := os.Stat(nameDir); err == nil {
|
||||
return nameDir
|
||||
@@ -175,78 +200,104 @@ func (h *TemplatesHandler) resolveTemplateDir(wsName string) string {
|
||||
// List handles GET /templates
|
||||
func (h *TemplatesHandler) List(c *gin.Context) {
|
||||
templates := make([]templateSummary, 0)
|
||||
walkTemplateConfigs(h.configsDir, func(id string, data []byte) {
|
||||
var raw struct {
|
||||
Name string `yaml:"name"`
|
||||
Description string `yaml:"description"`
|
||||
Tier int `yaml:"tier"`
|
||||
Runtime string `yaml:"runtime"`
|
||||
Model string `yaml:"model"`
|
||||
Skills []string `yaml:"skills"`
|
||||
// Top-level `providers:` block — structured registry. Distinct
|
||||
// from runtime_config.providers (slug list) below. Both shapes
|
||||
// coexist in production: claude-code ships the structured
|
||||
// registry, hermes still uses the slug list. /templates surfaces
|
||||
// both verbatim so each runtime owns its taxonomy.
|
||||
Providers []providerRegistryEntry `yaml:"providers"`
|
||||
RuntimeConfig struct {
|
||||
Model string `yaml:"model"`
|
||||
Models []modelSpec `yaml:"models"`
|
||||
RequiredEnv []string `yaml:"required_env"`
|
||||
RecommendedEnv []string `yaml:"recommended_env"`
|
||||
Providers []string `yaml:"providers"`
|
||||
ProvisionTimeoutSeconds int `yaml:"provision_timeout_seconds"`
|
||||
} `yaml:"runtime_config"`
|
||||
}
|
||||
if err := yaml.Unmarshal(data, &raw); err != nil {
|
||||
// Without this log a malformed config.yaml causes the
|
||||
// template to silently disappear from /templates with no
|
||||
// trace — the operator can't tell "excluded due to parse
|
||||
// error" from "never existed." That matters more now that
|
||||
// templates ship richer YAML shapes (top-level providers
|
||||
// registry, models[] with required_env, etc.) where a
|
||||
// type-shape mismatch on one field drops the whole entry.
|
||||
log.Printf("templates list: skip %s: yaml.Unmarshal: %v", id, err)
|
||||
return
|
||||
}
|
||||
runtime := strings.TrimSuffix(strings.TrimSpace(raw.Runtime), "-default")
|
||||
if _, ok := knownRuntimes[runtime]; !ok {
|
||||
log.Printf("templates list: skip %s: unsupported runtime %q", id, raw.Runtime)
|
||||
seen := map[string]struct{}{}
|
||||
walk := func(root string) {
|
||||
if root == "" {
|
||||
return
|
||||
}
|
||||
walkTemplateConfigs(root, func(id string, data []byte) {
|
||||
if _, ok := seen[id]; ok {
|
||||
return
|
||||
}
|
||||
seen[id] = struct{}{}
|
||||
var raw struct {
|
||||
Name string `yaml:"name"`
|
||||
Description string `yaml:"description"`
|
||||
Tier int `yaml:"tier"`
|
||||
Runtime string `yaml:"runtime"`
|
||||
Model string `yaml:"model"`
|
||||
Skills []string `yaml:"skills"`
|
||||
// Top-level `providers:` block — structured registry. Distinct
|
||||
// from runtime_config.providers (slug list) below. Both shapes
|
||||
// coexist in production: claude-code ships the structured
|
||||
// registry, hermes still uses the slug list. /templates surfaces
|
||||
// both verbatim so each runtime owns its taxonomy.
|
||||
Providers []providerRegistryEntry `yaml:"providers"`
|
||||
RuntimeConfig struct {
|
||||
Model string `yaml:"model"`
|
||||
Models []modelSpec `yaml:"models"`
|
||||
RequiredEnv []string `yaml:"required_env"`
|
||||
RecommendedEnv []string `yaml:"recommended_env"`
|
||||
Providers []string `yaml:"providers"`
|
||||
ProvisionTimeoutSeconds int `yaml:"provision_timeout_seconds"`
|
||||
} `yaml:"runtime_config"`
|
||||
}
|
||||
if err := yaml.Unmarshal(data, &raw); err != nil {
|
||||
// Without this log a malformed config.yaml causes the
|
||||
// template to silently disappear from /templates with no
|
||||
// trace — the operator can't tell "excluded due to parse
|
||||
// error" from "never existed." That matters more now that
|
||||
// templates ship richer YAML shapes (top-level providers
|
||||
// registry, models[] with required_env, etc.) where a
|
||||
// type-shape mismatch on one field drops the whole entry.
|
||||
log.Printf("templates list: skip %s: yaml.Unmarshal: %v", id, err)
|
||||
return
|
||||
}
|
||||
runtime := strings.TrimSuffix(strings.TrimSpace(raw.Runtime), "-default")
|
||||
if _, ok := knownRuntimes[runtime]; !ok {
|
||||
log.Printf("templates list: skip %s: unsupported runtime %q", id, raw.Runtime)
|
||||
return
|
||||
}
|
||||
|
||||
// Model comes from either top-level (legacy) or runtime_config.model (current).
|
||||
model := raw.Model
|
||||
if model == "" {
|
||||
model = raw.RuntimeConfig.Model
|
||||
}
|
||||
// Model comes from either top-level (legacy) or runtime_config.model (current).
|
||||
model := raw.Model
|
||||
if model == "" {
|
||||
model = raw.RuntimeConfig.Model
|
||||
}
|
||||
|
||||
tier := raw.Tier
|
||||
if h.wh != nil && h.wh.IsSaaS() {
|
||||
tier = h.wh.DefaultTier()
|
||||
}
|
||||
tier := raw.Tier
|
||||
if h.wh != nil && h.wh.IsSaaS() {
|
||||
tier = h.wh.DefaultTier()
|
||||
}
|
||||
|
||||
templates = append(templates, templateSummary{
|
||||
ID: id,
|
||||
Name: raw.Name,
|
||||
Description: raw.Description,
|
||||
Tier: tier,
|
||||
Runtime: raw.Runtime,
|
||||
Model: model,
|
||||
Models: raw.RuntimeConfig.Models,
|
||||
RequiredEnv: raw.RuntimeConfig.RequiredEnv,
|
||||
RecommendedEnv: raw.RuntimeConfig.RecommendedEnv,
|
||||
Providers: raw.RuntimeConfig.Providers,
|
||||
ProviderRegistry: raw.Providers,
|
||||
Skills: raw.Skills,
|
||||
SkillCount: len(raw.Skills),
|
||||
ProvisionTimeoutSeconds: raw.RuntimeConfig.ProvisionTimeoutSeconds,
|
||||
templates = append(templates, templateSummary{
|
||||
ID: id,
|
||||
Name: raw.Name,
|
||||
Description: raw.Description,
|
||||
Tier: tier,
|
||||
Runtime: raw.Runtime,
|
||||
Model: model,
|
||||
Models: raw.RuntimeConfig.Models,
|
||||
RequiredEnv: raw.RuntimeConfig.RequiredEnv,
|
||||
RecommendedEnv: raw.RuntimeConfig.RecommendedEnv,
|
||||
Providers: raw.RuntimeConfig.Providers,
|
||||
ProviderRegistry: raw.Providers,
|
||||
Skills: raw.Skills,
|
||||
SkillCount: len(raw.Skills),
|
||||
ProvisionTimeoutSeconds: raw.RuntimeConfig.ProvisionTimeoutSeconds,
|
||||
})
|
||||
})
|
||||
})
|
||||
}
|
||||
walk(h.cacheDir)
|
||||
walk(h.configsDir)
|
||||
|
||||
c.JSON(http.StatusOK, templates)
|
||||
}
|
||||
|
||||
// RefreshCache handles POST /admin/templates/refresh.
|
||||
func (h *TemplatesHandler) RefreshCache(c *gin.Context) {
|
||||
if h.refreshCache == nil {
|
||||
c.JSON(http.StatusServiceUnavailable, gin.H{"error": "template cache refresh is not configured"})
|
||||
return
|
||||
}
|
||||
result, err := h.refreshCache(c)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
c.JSON(http.StatusOK, result)
|
||||
}
|
||||
|
||||
// ListFiles handles GET /workspaces/:id/files
|
||||
// Lists files inside the running container's /configs directory (or /workspace, etc.).
|
||||
// Falls back to host-side config templates directory when container isn't running.
|
||||
|
||||
@@ -133,6 +133,71 @@ skills:
|
||||
}
|
||||
}
|
||||
|
||||
func TestTemplatesList_CacheOverridesBakedTemplate(t *testing.T) {
|
||||
setupTestDB(t)
|
||||
setupTestRedis(t)
|
||||
|
||||
bakedDir := t.TempDir()
|
||||
cacheDir := t.TempDir()
|
||||
|
||||
mustWriteTemplate := func(root, id, body string) {
|
||||
t.Helper()
|
||||
dir := filepath.Join(root, id)
|
||||
if err := os.MkdirAll(dir, 0755); err != nil {
|
||||
t.Fatalf("mkdir %s: %v", dir, err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(dir, "config.yaml"), []byte(body), 0644); err != nil {
|
||||
t.Fatalf("write config: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
mustWriteTemplate(bakedDir, "seo-agent", `name: SEO Agent
|
||||
description: stale
|
||||
tier: 4
|
||||
runtime: claude-code
|
||||
model: old
|
||||
runtime_config:
|
||||
recommended_env: [TELEGRAM_BOT_TOKEN]
|
||||
skills: []
|
||||
`)
|
||||
mustWriteTemplate(cacheDir, "seo-agent", `name: SEO Agent
|
||||
description: fresh
|
||||
tier: 4
|
||||
runtime: claude-code
|
||||
model: moonshot/kimi-k2.6
|
||||
runtime_config:
|
||||
required_env: [TENANT_NAME]
|
||||
recommended_env: [GOOGLE_GSC_SITE]
|
||||
skills: []
|
||||
`)
|
||||
|
||||
handler := NewTemplatesHandler(bakedDir, nil, nil).WithCacheDir(cacheDir)
|
||||
w := httptest.NewRecorder()
|
||||
c, _ := gin.CreateTestContext(w)
|
||||
c.Request = httptest.NewRequest("GET", "/templates", nil)
|
||||
handler.List(c)
|
||||
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("expected 200, got %d", w.Code)
|
||||
}
|
||||
var resp []templateSummary
|
||||
if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
|
||||
t.Fatalf("parse: %v", err)
|
||||
}
|
||||
if len(resp) != 1 {
|
||||
t.Fatalf("expected 1 template, got %d", len(resp))
|
||||
}
|
||||
if resp[0].Description != "fresh" {
|
||||
t.Fatalf("cache template should override baked copy, got description %q", resp[0].Description)
|
||||
}
|
||||
if !reflect.DeepEqual(resp[0].RequiredEnv, []string{"TENANT_NAME"}) {
|
||||
t.Fatalf("RequiredEnv = %+v", resp[0].RequiredEnv)
|
||||
}
|
||||
if reflect.DeepEqual(resp[0].RecommendedEnv, []string{"TELEGRAM_BOT_TOKEN"}) {
|
||||
t.Fatalf("stale baked recommended_env leaked through: %+v", resp[0].RecommendedEnv)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTemplatesList_RuntimeAndModelsRegistry(t *testing.T) {
|
||||
setupTestDB(t)
|
||||
setupTestRedis(t)
|
||||
|
||||
@@ -56,6 +56,7 @@ type WorkspaceHandler struct {
|
||||
cpProv provisioner.CPProvisionerAPI
|
||||
platformURL string
|
||||
configsDir string // path to workspace-configs-templates/ (for reading templates)
|
||||
cacheDir string // optional runtime-refreshed template cache; overrides configsDir by template id
|
||||
// envMutators runs registered EnvMutator plugins right before
|
||||
// container Start, after built-in secret loads. Nil = no plugins
|
||||
// registered; Registry.Run handles a nil receiver as a no-op so the
|
||||
@@ -183,6 +184,11 @@ func NewWorkspaceHandler(b events.EventEmitter, p *provisioner.Provisioner, plat
|
||||
return h
|
||||
}
|
||||
|
||||
func (h *WorkspaceHandler) WithTemplateCacheDir(cacheDir string) *WorkspaceHandler {
|
||||
h.cacheDir = cacheDir
|
||||
return h
|
||||
}
|
||||
|
||||
// WithNamespaceCleanup wires the I5 hook (RFC #2728) so workspace
|
||||
// purge can drop the plugin's `workspace:<id>` namespace. main.go
|
||||
// passes a closure over plugin.DeleteNamespace; tests pass a stub.
|
||||
@@ -285,7 +291,7 @@ func (h *WorkspaceHandler) Create(c *gin.Context) {
|
||||
// #226: payload.Template is attacker-controllable. resolveInsideRoot
|
||||
// rejects absolute paths and any ".." that escapes configsDir so the
|
||||
// provisioner can't be pointed at host directories.
|
||||
candidatePath, resolveErr := resolveInsideRoot(h.configsDir, payload.Template)
|
||||
candidatePath, resolveErr := resolveWorkspaceTemplatePath(h.configsDir, h.cacheDir, payload.Template)
|
||||
if resolveErr != nil {
|
||||
log.Printf("Create: invalid template path %q: %v", payload.Template, resolveErr)
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "invalid template"})
|
||||
@@ -726,7 +732,7 @@ func (h *WorkspaceHandler) Create(c *gin.Context) {
|
||||
var templatePath string
|
||||
var configFiles map[string][]byte
|
||||
if payload.Template != "" {
|
||||
candidatePath, resolveErr := resolveInsideRoot(h.configsDir, payload.Template)
|
||||
candidatePath, resolveErr := resolveWorkspaceTemplatePath(h.configsDir, h.cacheDir, payload.Template)
|
||||
if resolveErr != nil {
|
||||
log.Printf("Create provision: rejecting template %q: %v", payload.Template, resolveErr)
|
||||
return
|
||||
|
||||
@@ -485,6 +485,17 @@ func findTemplateByName(configsDir, name string) string {
|
||||
return ""
|
||||
}
|
||||
|
||||
func resolveWorkspaceTemplatePath(configsDir, cacheDir, template string) (string, error) {
|
||||
if cacheDir != "" {
|
||||
if p, err := resolveInsideRoot(cacheDir, template); err != nil {
|
||||
return "", err
|
||||
} else if _, statErr := os.Stat(p); statErr == nil {
|
||||
return p, nil
|
||||
}
|
||||
}
|
||||
return resolveInsideRoot(configsDir, template)
|
||||
}
|
||||
|
||||
// resolveOrgTemplate looks for a matching role directory under
|
||||
// configsDir/org-templates/ and returns the absolute path and a short label
|
||||
// ("org-templates/<dir>"). Used by the restart handler's rebuild_config path
|
||||
@@ -658,7 +669,7 @@ func (h *WorkspaceHandler) defaultTemplateProvidersYAML(runtime string) string {
|
||||
return ""
|
||||
}
|
||||
templateName := runtime + "-default"
|
||||
templatePath, err := resolveInsideRoot(h.configsDir, templateName)
|
||||
templatePath, err := resolveWorkspaceTemplatePath(h.configsDir, h.cacheDir, templateName)
|
||||
if err != nil {
|
||||
log.Printf("Provisioner: default template providers skipped for runtime %s: %v", runtime, err)
|
||||
return ""
|
||||
|
||||
@@ -110,6 +110,32 @@ func TestFindTemplateByName_NotFound(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveWorkspaceTemplatePath_PrefersCache(t *testing.T) {
|
||||
bakedDir := t.TempDir()
|
||||
cacheDir := t.TempDir()
|
||||
|
||||
for _, root := range []string{bakedDir, cacheDir} {
|
||||
if err := os.MkdirAll(filepath.Join(root, "seo-agent"), 0755); err != nil {
|
||||
t.Fatalf("mkdir: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
got, err := resolveWorkspaceTemplatePath(bakedDir, cacheDir, "seo-agent")
|
||||
if err != nil {
|
||||
t.Fatalf("resolveWorkspaceTemplatePath: %v", err)
|
||||
}
|
||||
want := filepath.Join(cacheDir, "seo-agent")
|
||||
if got != want {
|
||||
t.Fatalf("want cache path %q, got %q", want, got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveWorkspaceTemplatePath_RejectsTraversal(t *testing.T) {
|
||||
if _, err := resolveWorkspaceTemplatePath(t.TempDir(), t.TempDir(), "../seo-agent"); err == nil {
|
||||
t.Fatal("expected traversal to be rejected")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFindTemplateByName_SkipsWsPrefix(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
|
||||
|
||||
@@ -36,7 +36,7 @@ import (
|
||||
// (main.go) gets the same pluginResolver instance so it can share scheme
|
||||
// enumeration if a deployment registers extra schemes externally. A nil
|
||||
// pluginResolver is harmless: plgh still works with its built-in defaults.
|
||||
func Setup(hub *ws.Hub, broadcaster *events.Broadcaster, prov *provisioner.Provisioner, platformURL, configsDir string, wh *handlers.WorkspaceHandler, channelMgr *channels.Manager, memBundle *memwiring.Bundle, pluginResolver plugins.PluginResolver) *gin.Engine {
|
||||
func Setup(hub *ws.Hub, broadcaster *events.Broadcaster, prov *provisioner.Provisioner, platformURL, configsDir string, templateCacheDir string, wh *handlers.WorkspaceHandler, channelMgr *channels.Manager, memBundle *memwiring.Bundle, pluginResolver plugins.PluginResolver, refreshTemplates func(ctx *gin.Context) (any, error)) *gin.Engine {
|
||||
r := gin.Default()
|
||||
|
||||
// Issue #179 — trust no reverse-proxy headers. Without this call Gin's
|
||||
@@ -666,7 +666,9 @@ func Setup(hub *ws.Hub, broadcaster *events.Broadcaster, prov *provisioner.Provi
|
||||
|
||||
// Templates — wh threaded so generateDefaultConfig picks the
|
||||
// SaaS-aware default tier in Import + ReplaceFiles (#2910 PR-B).
|
||||
tmplh := handlers.NewTemplatesHandler(configsDir, dockerCli, wh)
|
||||
tmplh := handlers.NewTemplatesHandler(configsDir, dockerCli, wh).
|
||||
WithCacheDir(templateCacheDir).
|
||||
WithRefreshFunc(refreshTemplates)
|
||||
// #686: GET /templates lists all template names+metadata from configsDir.
|
||||
// Open access lets unauthenticated callers enumerate org configurations and
|
||||
// installed plugins. AdminAuth-gate it alongside POST /templates/import.
|
||||
@@ -676,6 +678,7 @@ func Setup(hub *ws.Hub, broadcaster *events.Broadcaster, prov *provisioner.Provi
|
||||
tmplAdmin := r.Group("", middleware.AdminAuth(db.DB))
|
||||
tmplAdmin.GET("/templates", tmplh.List)
|
||||
tmplAdmin.POST("/templates/import", tmplh.Import)
|
||||
tmplAdmin.POST("/admin/templates/refresh", tmplh.RefreshCache)
|
||||
}
|
||||
wsAuth.PUT("/files", tmplh.ReplaceFiles)
|
||||
wsAuth.GET("/files", tmplh.ListFiles)
|
||||
|
||||
@@ -0,0 +1,176 @@
|
||||
package templatecache
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
type ManifestEntry struct {
|
||||
Name string `json:"name"`
|
||||
Repo string `json:"repo"`
|
||||
Ref string `json:"ref"`
|
||||
}
|
||||
|
||||
type manifestFile struct {
|
||||
WorkspaceTemplates []ManifestEntry `json:"workspace_templates"`
|
||||
}
|
||||
|
||||
type TemplateResult struct {
|
||||
Name string `json:"name"`
|
||||
Repo string `json:"repo"`
|
||||
Ref string `json:"ref"`
|
||||
SHA string `json:"sha,omitempty"`
|
||||
Status string `json:"status"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
type RefreshReport struct {
|
||||
ManifestPath string `json:"manifest_path"`
|
||||
CacheDir string `json:"cache_dir"`
|
||||
RefreshedAt time.Time `json:"refreshed_at"`
|
||||
Results []TemplateResult `json:"results"`
|
||||
}
|
||||
|
||||
func RefreshWorkspaceTemplates(ctx context.Context, manifestPath, cacheDir, token string) (RefreshReport, error) {
|
||||
report := RefreshReport{
|
||||
ManifestPath: manifestPath,
|
||||
CacheDir: cacheDir,
|
||||
RefreshedAt: time.Now().UTC(),
|
||||
}
|
||||
if strings.TrimSpace(token) == "" {
|
||||
return report, fmt.Errorf("template cache refresh requires MOLECULE_TEMPLATE_GITEA_TOKEN or MOLECULE_GITEA_TOKEN")
|
||||
}
|
||||
data, err := os.ReadFile(manifestPath)
|
||||
if err != nil {
|
||||
return report, fmt.Errorf("read manifest: %w", err)
|
||||
}
|
||||
var manifest manifestFile
|
||||
if err := json.Unmarshal(data, &manifest); err != nil {
|
||||
return report, fmt.Errorf("parse manifest: %w", err)
|
||||
}
|
||||
if err := os.MkdirAll(cacheDir, 0o755); err != nil {
|
||||
return report, fmt.Errorf("mkdir cache: %w", err)
|
||||
}
|
||||
for _, entry := range manifest.WorkspaceTemplates {
|
||||
result := refreshOne(ctx, cacheDir, token, entry)
|
||||
report.Results = append(report.Results, result)
|
||||
}
|
||||
return report, nil
|
||||
}
|
||||
|
||||
func refreshOne(ctx context.Context, cacheDir, token string, entry ManifestEntry) TemplateResult {
|
||||
result := TemplateResult{Name: entry.Name, Repo: entry.Repo, Ref: entry.Ref}
|
||||
if result.Ref == "" {
|
||||
result.Ref = "main"
|
||||
}
|
||||
if !safeTemplateName(entry.Name) {
|
||||
result.Status = "skipped"
|
||||
result.Error = "invalid template name"
|
||||
return result
|
||||
}
|
||||
if strings.TrimSpace(entry.Repo) == "" {
|
||||
result.Status = "skipped"
|
||||
result.Error = "missing repo"
|
||||
return result
|
||||
}
|
||||
|
||||
tmp, err := os.MkdirTemp(cacheDir, ".tmp-"+entry.Name+"-")
|
||||
if err != nil {
|
||||
result.Status = "failed"
|
||||
result.Error = err.Error()
|
||||
return result
|
||||
}
|
||||
defer os.RemoveAll(tmp)
|
||||
|
||||
cloneURL := authenticatedURL(entry.Repo, token)
|
||||
for _, args := range [][]string{
|
||||
{"init", "-q", tmp},
|
||||
{"-C", tmp, "remote", "add", "origin", cloneURL},
|
||||
{"-C", tmp, "fetch", "--depth=1", "-q", "origin", result.Ref},
|
||||
{"-C", tmp, "checkout", "-q", "--detach", "FETCH_HEAD"},
|
||||
} {
|
||||
cmd := exec.CommandContext(ctx, "git", args...)
|
||||
if out, err := cmd.CombinedOutput(); err != nil {
|
||||
result.Status = "failed"
|
||||
result.Error = sanitizeGitError(out, err, token)
|
||||
return result
|
||||
}
|
||||
}
|
||||
shaCmd := exec.CommandContext(ctx, "git", "-C", tmp, "rev-parse", "HEAD")
|
||||
if out, err := shaCmd.Output(); err == nil {
|
||||
result.SHA = strings.TrimSpace(string(out))
|
||||
}
|
||||
_ = os.RemoveAll(filepath.Join(tmp, ".git"))
|
||||
|
||||
target := filepath.Join(cacheDir, entry.Name)
|
||||
old := filepath.Join(cacheDir, ".old-"+entry.Name+"-"+fmt.Sprint(time.Now().UnixNano()))
|
||||
if _, err := os.Stat(target); err == nil {
|
||||
if err := os.Rename(target, old); err != nil {
|
||||
result.Status = "failed"
|
||||
result.Error = "replace old cache: " + err.Error()
|
||||
return result
|
||||
}
|
||||
defer os.RemoveAll(old)
|
||||
}
|
||||
if err := os.Rename(tmp, target); err != nil {
|
||||
if old != "" {
|
||||
_ = os.Rename(old, target)
|
||||
}
|
||||
result.Status = "failed"
|
||||
result.Error = "install cache: " + err.Error()
|
||||
return result
|
||||
}
|
||||
result.Status = "refreshed"
|
||||
return result
|
||||
}
|
||||
|
||||
func safeTemplateName(name string) bool {
|
||||
if name == "" || name == "." || name == ".." {
|
||||
return false
|
||||
}
|
||||
for _, r := range name {
|
||||
if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') || r == '-' || r == '_' {
|
||||
continue
|
||||
}
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func authenticatedURL(repo, token string) string {
|
||||
if strings.HasPrefix(repo, "http://") || strings.HasPrefix(repo, "https://") {
|
||||
u, err := url.Parse(repo)
|
||||
if err == nil {
|
||||
u.User = url.UserPassword("oauth2", token)
|
||||
return u.String()
|
||||
}
|
||||
}
|
||||
u := &url.URL{
|
||||
Scheme: "https",
|
||||
Host: "git.moleculesai.app",
|
||||
Path: "/" + strings.TrimSuffix(repo, ".git") + ".git",
|
||||
User: url.UserPassword("oauth2", token),
|
||||
}
|
||||
return u.String()
|
||||
}
|
||||
|
||||
func sanitizeGitError(out []byte, err error, token string) string {
|
||||
msg := strings.TrimSpace(string(out))
|
||||
if msg == "" {
|
||||
msg = err.Error()
|
||||
}
|
||||
if token != "" {
|
||||
msg = strings.ReplaceAll(msg, token, "***")
|
||||
}
|
||||
if len(msg) > 300 {
|
||||
msg = msg[:300]
|
||||
}
|
||||
return msg
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
package templatecache
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestSafeTemplateName(t *testing.T) {
|
||||
for _, name := range []string{"seo-agent", "claude_code", "T4"} {
|
||||
if !safeTemplateName(name) {
|
||||
t.Fatalf("%q should be safe", name)
|
||||
}
|
||||
}
|
||||
for _, name := range []string{"", "../seo", "seo/agent", "seo.agent"} {
|
||||
if safeTemplateName(name) {
|
||||
t.Fatalf("%q should be rejected", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user