From d552b5bfa1b0c42f7dc29bee5bfed57dbfe3a2bf Mon Sep 17 00:00:00 2001 From: Min Min Date: Wed, 27 May 2026 00:32:18 +0800 Subject: [PATCH 1/2] add custom resource support for deployment Signed-off-by: Min Min --- .../aslan/core/common/service/kube/parse.go | 58 +++++++++++++++++++ .../aslan/core/common/service/kube/render.go | 29 +++++++++- .../aslan/core/common/util/service.go | 31 +++++++++- 3 files changed, 116 insertions(+), 2 deletions(-) diff --git a/pkg/microservice/aslan/core/common/service/kube/parse.go b/pkg/microservice/aslan/core/common/service/kube/parse.go index 9c71df0edb..1ff2411acb 100644 --- a/pkg/microservice/aslan/core/common/service/kube/parse.go +++ b/pkg/microservice/aslan/core/common/service/kube/parse.go @@ -17,8 +17,11 @@ limitations under the License. package kube import ( + "fmt" "regexp" "strings" + + commonmodels "github.com/koderover/zadig/v2/pkg/microservice/aslan/core/common/repository/models" ) const ( @@ -30,6 +33,11 @@ const ( envRegexString = `\$EnvName\$` productRegexString = `\$Product\$` serviceRegexString = `\$Service\$` + + // $-image$ — per service-module image substitution. + // Module names follow the K8s container name spec; we allow the broader + // [A-Za-z0-9_-]+ to match what parseContainer already accepts. + moduleImageRegexString = `\$([A-Za-z0-9_-]+)-image\$` ) var ( @@ -41,6 +49,8 @@ var ( productRegex = regexp.MustCompile(productRegexString) envNameRegex = regexp.MustCompile(envRegexString) serviceRegex = regexp.MustCompile(serviceRegexString) + + moduleImageRegex = regexp.MustCompile(moduleImageRegexString) ) // ParseSysKeys 渲染系统变量键值 @@ -51,3 +61,51 @@ func ParseSysKeys(namespace, envName, productName, serviceName, ori string) stri ori = serviceRegex.ReplaceAllLiteralString(ori, strings.ToLower(serviceName)) return ori } + +// IsModuleImagePlaceholder reports whether s is exactly a $-image$ +// placeholder (no surrounding text). Auto-detection uses this to avoid +// storing placeholder strings as resolved image URIs. +func IsModuleImagePlaceholder(s string) bool { + loc := moduleImageRegex.FindStringIndex(s) + return loc != nil && loc[0] == 0 && loc[1] == len(s) +} + +// ParseModuleImageKeys substitutes $-image$ placeholders in yaml using +// the provided container list (container.Name -> container.Image). Containers +// with an empty or placeholder-shaped Image are skipped — they cannot resolve +// anything. +// +// If allowUnresolved is false, any $-image$ placeholder still present +// after substitution causes an error naming the offending module(s). Callers +// on the initial-deploy path (no built image yet) should pass true. +func ParseModuleImageKeys(yaml string, containers []*commonmodels.Container, allowUnresolved bool) (string, error) { + for _, c := range containers { + if c == nil || c.Name == "" || c.Image == "" { + continue + } + if IsModuleImagePlaceholder(c.Image) { + continue + } + pattern := regexp.MustCompile(`\$` + regexp.QuoteMeta(c.Name) + `-image\$`) + yaml = pattern.ReplaceAllLiteralString(yaml, c.Image) + } + + if allowUnresolved { + return yaml, nil + } + + matches := moduleImageRegex.FindAllStringSubmatch(yaml, -1) + if len(matches) == 0 { + return yaml, nil + } + seen := make(map[string]struct{}, len(matches)) + unresolved := make([]string, 0, len(matches)) + for _, m := range matches { + if _, ok := seen[m[1]]; ok { + continue + } + seen[m[1]] = struct{}{} + unresolved = append(unresolved, m[1]) + } + return yaml, fmt.Errorf("unresolved $-image$ placeholder(s): %s", strings.Join(unresolved, ", ")) +} diff --git a/pkg/microservice/aslan/core/common/service/kube/render.go b/pkg/microservice/aslan/core/common/service/kube/render.go index 939c8dadd0..88ab8fc813 100644 --- a/pkg/microservice/aslan/core/common/service/kube/render.go +++ b/pkg/microservice/aslan/core/common/service/kube/render.go @@ -65,6 +65,10 @@ type GeneSvcYamlOption struct { IgnoreCurrentReplicaOverrides bool UnInstall bool Containers []*models.Container + // AllowUnresolvedModuleImages skips the post-render check that errors + // when a $-image$ placeholder remains in the YAML. Set this on + // initial-deploy code paths where the image hasn't been built yet. + AllowUnresolvedModuleImages bool } type WorkloadResource struct { @@ -418,6 +422,10 @@ func FetchCurrentAppliedYaml(option *GeneSvcYamlOption) (string, int, error) { if err != nil { return "", 0, err } + fullRenderedYaml, err = ParseModuleImageKeys(fullRenderedYaml, mergedContainers, option.AllowUnresolvedModuleImages) + if err != nil { + return "", 0, err + } replicaOverrides := resolveReplicaOverrides(curProductSvc.WorkLoads, option.ReplicaOverrides, option.IgnoreCurrentReplicaOverrides) fullRenderedYaml, err = ApplyReplicaOverrides(fullRenderedYaml, replicaOverrides) @@ -743,7 +751,15 @@ func FetchImportedManifests(option *GeneSvcYamlOption, productInfo *models.Produ manifestArr = append(manifestArr, string(workloadBs)) } } - return ReplaceWorkloadImages(util.JoinYamls(manifestArr), option.Containers) + replacedYaml, workloadResource, err := ReplaceWorkloadImages(util.JoinYamls(manifestArr), option.Containers) + if err != nil { + return "", nil, err + } + replacedYaml, err = ParseModuleImageKeys(replacedYaml, option.Containers, option.AllowUnresolvedModuleImages) + if err != nil { + return "", nil, err + } + return replacedYaml, workloadResource, nil } func variableYamlNil(variableYaml string) bool { @@ -871,6 +887,13 @@ func GenerateRenderedYaml(option *GeneSvcYamlOption) (string, int, []*WorkloadRe if err != nil { return "", 0, nil, fmt.Errorf("failed to replace workload images: %v", err) } + // Second image-substitution pass — text-level $-image$ replacement + // for any workload kind not handled by ReplaceWorkloadImages (e.g. DaemonSets, + // Argo Rollouts, other CRDs). + fullRenderedYaml, err = ParseModuleImageKeys(fullRenderedYaml, mergedContainers, option.AllowUnresolvedModuleImages) + if err != nil { + return "", 0, nil, fmt.Errorf("failed to parse module image keys: %v", err) + } var currentReplicaOverrides []*commonmodels.WorkLoad if curProductSvc != nil { @@ -986,5 +1009,9 @@ func RenderEnvServiceWithTempl(prod *commonmodels.Product, serviceRender *templa if err != nil { return "", err } + parsedYaml, err = ParseModuleImageKeys(parsedYaml, service.Containers, false) + if err != nil { + return "", err + } return ApplyReplicaOverrides(parsedYaml, service.WorkLoads) } diff --git a/pkg/microservice/aslan/core/common/util/service.go b/pkg/microservice/aslan/core/common/util/service.go index c16f7ec96d..84be6be297 100644 --- a/pkg/microservice/aslan/core/common/util/service.go +++ b/pkg/microservice/aslan/core/common/util/service.go @@ -19,6 +19,7 @@ package util import ( "errors" "fmt" + "regexp" "strings" "gopkg.in/yaml.v2" @@ -43,6 +44,10 @@ var ( {setting.PathSearchComponentImage: "image.repository", setting.PathSearchComponentTag: "image.tag"}, {setting.PathSearchComponentImage: "image"}, } + + // Duplicated from kube.moduleImageRegex — kept here to avoid an import + // cycle (kube imports util). Keep both regexes in sync. + moduleImagePlaceholderRegex = regexp.MustCompile(`^\$[A-Za-z0-9_-]+-image\$$`) ) func GetServiceDeployStrategy(serviceName string, strategyMap map[string]setting.ServiceDeployStrategy) setting.ServiceDeployStrategy { @@ -168,6 +173,16 @@ func SetChartServiceDeployStrategyImport(strategyMap map[string]setting.ServiceD } func SetCurrentContainerImages(args *commonmodels.Service) error { + // Snapshot prior images so we can preserve resolved URIs when the + // rendered YAML contains a $-image$ placeholder (which would + // otherwise overwrite the stored image with the literal placeholder). + priorByName := make(map[string]*commonmodels.Container, len(args.Containers)) + for _, c := range args.Containers { + if c != nil { + priorByName[c.Name] = c + } + } + var srvContainers []*commonmodels.Container for _, data := range args.KubeYamls { yamlDataArray := util.SplitYaml(data) @@ -211,7 +226,21 @@ func SetCurrentContainerImages(args *commonmodels.Service) error { } } - args.Containers = uniqueSlice(srvContainers) + deduped := uniqueSlice(srvContainers) + for _, c := range deduped { + if c == nil || !moduleImagePlaceholderRegex.MatchString(c.Image) { + continue + } + // The rendered YAML still carries the placeholder for this module + // (e.g. an unknown workload kind where ReplaceWorkloadImages can't + // substitute by name+kind). Carry the previously-resolved image + // forward so we never persist a placeholder as a stored image URI. + if prior, ok := priorByName[c.Name]; ok && prior.Image != "" && !moduleImagePlaceholderRegex.MatchString(prior.Image) { + c.Image = prior.Image + c.ImageName = prior.ImageName + } + } + args.Containers = deduped return nil } From 6b488642bffb70f30e36e41f977a393f2f9e896c Mon Sep 17 00:00:00 2001 From: Min Min Date: Mon, 8 Jun 2026 11:57:16 +0800 Subject: [PATCH 2/2] service module implementation & bugfix Signed-off-by: Min Min --- pkg/cli/initconfig/cmd/init.go | 2 + pkg/cli/upgradeassistant/cmd/migrate/500.go | 180 +++++++++++ .../internal/repository/models/migration.go | 1 + .../aslan/core/build/service/build.go | 11 +- .../aslan/core/build/service/target.go | 20 +- .../core/common/repository/models/service.go | 11 +- .../repository/models/service_module.go | 84 ++++++ .../repository/mongodb/service_module.go | 270 +++++++++++++++++ .../aslan/core/common/service/environment.go | 11 +- .../aslan/core/common/service/helm/helm.go | 10 +- .../aslan/core/common/service/kube/parse.go | 23 +- .../aslan/core/common/service/kube/render.go | 50 +++- .../core/common/service/repository/service.go | 96 +++++- .../service/repository/service_module.go | 212 +++++++++++++ .../aslan/core/common/service/service.go | 12 +- .../jobcontroller/job_deploy.go | 12 + .../aslan/core/common/util/service.go | 69 ++++- .../aslan/core/delivery/service/openapi.go | 9 +- .../core/environment/service/environment.go | 14 +- .../aslan/core/environment/service/k8s.go | 32 +- .../aslan/core/environment/service/product.go | 9 +- .../core/environment/service/revision.go | 48 ++- .../aslan/core/project/service/openapi.go | 15 +- .../aslan/core/project/service/product.go | 10 +- .../core/service/handler/manual_module.go | 208 +++++++++++++ .../aslan/core/service/handler/router.go | 8 + .../aslan/core/service/service/helm.go | 8 +- .../core/service/service/manual_module.go | 282 ++++++++++++++++++ .../aslan/core/service/service/service.go | 31 +- .../core/workflow/service/webhook/gerrit.go | 3 + .../workflow/controller/job/job_build.go | 27 +- .../workflow/controller/job/job_deploy.go | 80 ++++- .../controller/job/job_distribute_image.go | 9 +- .../service/workflow/workflow_task_v4.go | 9 +- 34 files changed, 1807 insertions(+), 69 deletions(-) create mode 100644 pkg/cli/upgradeassistant/cmd/migrate/500.go create mode 100644 pkg/microservice/aslan/core/common/repository/models/service_module.go create mode 100644 pkg/microservice/aslan/core/common/repository/mongodb/service_module.go create mode 100644 pkg/microservice/aslan/core/common/service/repository/service_module.go create mode 100644 pkg/microservice/aslan/core/service/handler/manual_module.go create mode 100644 pkg/microservice/aslan/core/service/service/manual_module.go diff --git a/pkg/cli/initconfig/cmd/init.go b/pkg/cli/initconfig/cmd/init.go index 8ab70ad404..0ac2942ffb 100644 --- a/pkg/cli/initconfig/cmd/init.go +++ b/pkg/cli/initconfig/cmd/init.go @@ -154,6 +154,8 @@ func createOrUpdateMongodbIndex(ctx context.Context) { commonrepo.NewS3StorageColl(), commonrepo.NewServiceColl(), commonrepo.NewProductionServiceColl(), + commonrepo.NewServiceModuleColl(), + commonrepo.NewProductionServiceModuleColl(), commonrepo.NewStrategyColl(), commonrepo.NewStatsColl(), commonrepo.NewSubscriptionColl(), diff --git a/pkg/cli/upgradeassistant/cmd/migrate/500.go b/pkg/cli/upgradeassistant/cmd/migrate/500.go new file mode 100644 index 0000000000..23ca986519 --- /dev/null +++ b/pkg/cli/upgradeassistant/cmd/migrate/500.go @@ -0,0 +1,180 @@ +/* +Copyright 2026 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrate + +import ( + "context" + "fmt" + + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + + internalmodels "github.com/koderover/zadig/v2/pkg/cli/upgradeassistant/internal/repository/models" + internalmongodb "github.com/koderover/zadig/v2/pkg/cli/upgradeassistant/internal/repository/mongodb" + "github.com/koderover/zadig/v2/pkg/cli/upgradeassistant/internal/upgradepath" + commonmodels "github.com/koderover/zadig/v2/pkg/microservice/aslan/core/common/repository/models" + commonrepo "github.com/koderover/zadig/v2/pkg/microservice/aslan/core/common/repository/mongodb" + "github.com/koderover/zadig/v2/pkg/microservice/aslan/core/common/service/repository" + "github.com/koderover/zadig/v2/pkg/tool/log" +) + +const migration500ProgressEvery = 200 + +// legacyServiceForMigration500 is a local-to-this-migration view of a service +// template document. We deliberately do NOT reuse commonmodels.Service here: +// that struct now has bson:"-" on Containers (5.0.0 deprecation), so a normal +// decode would silently drop the legacy "containers" field that pre-5.0.0 +// documents carry — which is exactly the field this migration needs to read +// to backfill service_module. +// +// Only the fields SyncAutoServiceModules touches are declared. Everything +// else in the legacy document is ignored. +type legacyServiceForMigration500 struct { + ServiceName string `bson:"service_name"` + ProductName string `bson:"product_name"` + Revision int64 `bson:"revision"` + Type string `bson:"type"` + Containers []*commonmodels.Container `bson:"containers,omitempty"` +} + +func init() { + upgradepath.RegisterHandler("4.3.0", "5.0.0", V430ToV500) + upgradepath.RegisterHandler("5.0.0", "4.3.0", V500ToV430) +} + +// V430ToV500 backfills the new service_module collection from the legacy +// Service.Containers field on every existing service template revision. +// +// Phase 3 already dual-writes for new revisions persisted after the deploy. +// This migration plugs the gap for everything that existed before. +// +// Idempotent: ReplaceAutoForRevision deletes-then-inserts per (service, +// revision), so a re-run from a partial migration produces the same result. +func V430ToV500() error { + migrationInfo, err := getMigrationInfo() + if err != nil { + return fmt.Errorf("failed to get migration info from db, err: %s", err) + } + + defer func() { + updateMigrationError(migrationInfo.ID, err) + }() + + err = migrateServiceModule500(migrationInfo) + if err != nil { + return err + } + + return nil +} + +func V500ToV430() error { + // Rollback: the new collection is additive — leaving the data in place + // is safe because the legacy Service.Containers field is still populated + // and authoritative on the 4.3.0 code path. Deferring an actual cleanup + // to avoid wiping records a re-roll-forward would rebuild. + return nil +} + +// migrateServiceModule500 walks both template_service and +// production_template_service collections, mirroring each Service's +// Containers slice into service_module / production_service_module as +// auto records bound to the corresponding revision. +// +// Skipped when the migration flag is already set; backfill is otherwise +// idempotent and safe to re-run on partial completion. +func migrateServiceModule500(migrationInfo *internalmodels.Migration) error { + if migrationInfo.Migration500ServiceModule { + log.Infof("migration 5.0.0: service_module backfill already completed, skipping") + return nil + } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + testCount, err := backfillServiceModulesForCollection500(ctx, commonrepo.NewServiceColl().Collection, "template_service", false) + if err != nil { + return fmt.Errorf("failed to backfill service modules from template_service, err: %s", err) + } + + prodCount, err := backfillServiceModulesForCollection500(ctx, commonrepo.NewProductionServiceColl().Collection, "production_template_service", true) + if err != nil { + return fmt.Errorf("failed to backfill service modules from production_template_service, err: %s", err) + } + + log.Infof("migration 5.0.0: backfilled %d test + %d production service revisions into service_module", testCount, prodCount) + + return internalmongodb.NewMigrationColl().UpdateMigrationStatus(migrationInfo.ID, map[string]interface{}{ + getMigrationFieldBsonTag(migrationInfo, &migrationInfo.Migration500ServiceModule): true, + }) +} + +// backfillServiceModulesForCollection500 streams every document in the given +// service-template collection and mirrors its Containers slice into the new +// service_module collection (production-side picked by `production`). +// +// Per-document failures are logged and skipped so a single bad record +// (corrupt yaml, dead project) doesn't halt the whole migration. The +// returned count is the number of revisions successfully mirrored. +func backfillServiceModulesForCollection500(ctx context.Context, coll *mongo.Collection, label string, production bool) (int, error) { + cursor, err := coll.Find(ctx, bson.M{}) + if err != nil { + return 0, fmt.Errorf("failed to open cursor over %s: %s", label, err) + } + defer cursor.Close(ctx) + + migrated := 0 + skipped := 0 + for cursor.Next(ctx) { + // Decode into the local legacy view (see legacyServiceForMigration500 + // above) — commonmodels.Service has bson:"-" on Containers and would + // silently drop the legacy field on decode. + var legacy legacyServiceForMigration500 + if decodeErr := cursor.Decode(&legacy); decodeErr != nil { + log.Warnf("migration 5.0.0: failed to decode %s document, skipping: %s", label, decodeErr) + skipped++ + continue + } + svc := &commonmodels.Service{ + ServiceName: legacy.ServiceName, + ProductName: legacy.ProductName, + Revision: legacy.Revision, + Type: legacy.Type, + Containers: legacy.Containers, + } + // SyncAutoServiceModules tolerates empty Containers (no-op) and + // validates required fields itself. Errors here are logged but not + // fatal — one corrupt service shouldn't block the rest. + if syncErr := repository.SyncAutoServiceModules(ctx, svc, production); syncErr != nil { + log.Warnf("migration 5.0.0: failed to sync %s %s/%s rev %d: %s", + label, svc.ProductName, svc.ServiceName, svc.Revision, syncErr) + skipped++ + continue + } + migrated++ + if migrated%migration500ProgressEvery == 0 { + log.Infof("migration 5.0.0: %s progress — %d revisions mirrored, %d skipped", label, migrated, skipped) + } + } + if err := cursor.Err(); err != nil { + return migrated, fmt.Errorf("cursor over %s ended in error: %s", label, err) + } + if skipped > 0 { + log.Warnf("migration 5.0.0: %s complete — %d mirrored, %d skipped (inspect warn logs above)", label, migrated, skipped) + } + return migrated, nil +} diff --git a/pkg/cli/upgradeassistant/internal/repository/models/migration.go b/pkg/cli/upgradeassistant/internal/repository/models/migration.go index 7437c2de92..da3718396b 100644 --- a/pkg/cli/upgradeassistant/internal/repository/models/migration.go +++ b/pkg/cli/upgradeassistant/internal/repository/models/migration.go @@ -44,6 +44,7 @@ type Migration struct { Migration430GlobalReadOnlyRole bool `bson:"migration_430_global_read_only_role"` Migration430ScalePermission bool `bson:"migration_430_scale_permission"` Migration430CollaborationScalePermission bool `bson:"migration_430_collaboration_scale_permission"` + Migration500ServiceModule bool `bson:"migration_500_service_module"` Error string `bson:"error"` } diff --git a/pkg/microservice/aslan/core/build/service/build.go b/pkg/microservice/aslan/core/build/service/build.go index a7d05583eb..6adf9f2318 100644 --- a/pkg/microservice/aslan/core/build/service/build.go +++ b/pkg/microservice/aslan/core/build/service/build.go @@ -17,6 +17,7 @@ limitations under the License. package service import ( + "context" "errors" "fmt" "strings" @@ -29,6 +30,7 @@ import ( commonmodels "github.com/koderover/zadig/v2/pkg/microservice/aslan/core/common/repository/models" commonrepo "github.com/koderover/zadig/v2/pkg/microservice/aslan/core/common/repository/mongodb" commonservice "github.com/koderover/zadig/v2/pkg/microservice/aslan/core/common/service" + "github.com/koderover/zadig/v2/pkg/microservice/aslan/core/common/service/repository" commonutil "github.com/koderover/zadig/v2/pkg/microservice/aslan/core/common/util" "github.com/koderover/zadig/v2/pkg/setting" "github.com/koderover/zadig/v2/pkg/shared/client/systemconfig" @@ -195,7 +197,14 @@ func ListBuildModulesByServiceModule(encryptedKey, productName, envName string, ModuleBuilds: resp, }) } else { - for _, container := range serviceTmpl.Containers { + // Service.Containers no longer persisted — pull merged modules + // (non-production: this function targets the test service path + // per its NewServiceColl().ListMaxRevisionsByProduct call above). + resolved, _, rerr := repository.ResolveServiceModules(context.Background(), serviceTmpl.ProductName, serviceTmpl.ServiceName, false, serviceTmpl.Revision) + if rerr != nil { + return nil, e.ErrListBuildModule.AddErr(rerr) + } + for _, container := range resolved { opt := &commonrepo.BuildListOption{ ServiceName: serviceTmpl.ServiceName, Targets: []string{container.Name}, diff --git a/pkg/microservice/aslan/core/build/service/target.go b/pkg/microservice/aslan/core/build/service/target.go index 5be80a220a..b650f30e56 100644 --- a/pkg/microservice/aslan/core/build/service/target.go +++ b/pkg/microservice/aslan/core/build/service/target.go @@ -17,12 +17,14 @@ limitations under the License. package service import ( + "context" "fmt" "go.uber.org/zap" commonmodels "github.com/koderover/zadig/v2/pkg/microservice/aslan/core/common/repository/models" commonrepo "github.com/koderover/zadig/v2/pkg/microservice/aslan/core/common/repository/mongodb" + "github.com/koderover/zadig/v2/pkg/microservice/aslan/core/common/service/repository" "github.com/koderover/zadig/v2/pkg/setting" e "github.com/koderover/zadig/v2/pkg/tool/errors" ) @@ -39,7 +41,14 @@ func ListDeployTarget(productName string, log *zap.SugaredLogger) ([]*commonmode for _, svc := range services { switch svc.Type { case setting.K8SDeployType, setting.HelmDeployType: - for _, container := range svc.Containers { + // Service.Containers no longer persisted — pull merged modules + // from service_module table. + resolved, _, rerr := repository.ResolveServiceModules(context.Background(), svc.ProductName, svc.ServiceName, false, svc.Revision) + if rerr != nil { + log.Errorf("failed to resolve modules for %s/%s rev %d: %s", svc.ProductName, svc.ServiceName, svc.Revision, rerr) + continue + } + for _, container := range resolved { serviceObjects = append(serviceObjects, &commonmodels.ServiceModuleTarget{ ProductName: svc.ProductName, ServiceWithModule: commonmodels.ServiceWithModule{ @@ -85,7 +94,14 @@ func ListContainers(productName string, log *zap.SugaredLogger) ([]*commonmodels log.Errorf("ServiceTmpl.Find error: %v", err) continue } - for _, container := range serviceDetail.Containers { + // Service.Containers no longer persisted — read modules from + // the service_module table. + resolved, _, rerr := repository.ResolveServiceModules(context.Background(), serviceDetail.ProductName, serviceDetail.ServiceName, false, serviceDetail.Revision) + if rerr != nil { + log.Errorf("failed to resolve modules for %s/%s rev %d: %s", serviceDetail.ProductName, serviceDetail.ServiceName, serviceDetail.Revision, rerr) + continue + } + for _, container := range resolved { containerList = append(containerList, &commonmodels.ServiceModuleTarget{ ProductName: service.ProductName, ServiceWithModule: commonmodels.ServiceWithModule{ diff --git a/pkg/microservice/aslan/core/common/repository/models/service.go b/pkg/microservice/aslan/core/common/repository/models/service.go index df0588ff77..9dbb29faa8 100644 --- a/pkg/microservice/aslan/core/common/repository/models/service.go +++ b/pkg/microservice/aslan/core/common/repository/models/service.go @@ -48,7 +48,16 @@ type Service struct { Hash string `bson:"hash256,omitempty" json:"hash256,omitempty"` CreateTime int64 `bson:"create_time" json:"create_time"` CreateBy string `bson:"create_by" json:"create_by"` - Containers []*Container `bson:"containers,omitempty" json:"containers,omitempty"` + // Containers is an in-memory transit field used by the parsing helpers + // (SetCurrentContainerImages, parseContainer) and JSON responses for the + // frontend. Authoritative storage lives in the service_module collection + // — read via repository.ResolveServiceModules and write via + // repository.SyncAutoServiceModules. The bson tag is `-` so MongoDB does + // not persist this field; fresh DB loads will have it nil. + // + // Deprecated: do not add new readers of this field. Use + // repository.ResolveServiceModules instead. + Containers []*Container `bson:"-" json:"containers,omitempty"` Description string `bson:"description,omitempty" json:"description,omitempty"` Visibility string `bson:"visibility,omitempty" json:"visibility,omitempty"` // DEPRECATED since 1.17.0 Status string `bson:"status,omitempty" json:"status,omitempty"` diff --git a/pkg/microservice/aslan/core/common/repository/models/service_module.go b/pkg/microservice/aslan/core/common/repository/models/service_module.go new file mode 100644 index 0000000000..7b3f182851 --- /dev/null +++ b/pkg/microservice/aslan/core/common/repository/models/service_module.go @@ -0,0 +1,84 @@ +/* +Copyright 2026 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "go.mongodb.org/mongo-driver/bson/primitive" + + "github.com/koderover/zadig/v2/pkg/setting" +) + +// ServiceModule is a first-class record of a deployable unit ("module" / +// "container") attached to a service template. Modules can originate from: +// +// - Auto-discovery: parsed out of a service template's KubeYamls for the +// workload kinds we recognize (Deployment, StatefulSet, Job, CronJob, +// CloneSet). Auto records are bound to a specific service revision via +// RevisionBound and are re-derived every time SetCurrentContainerImages +// re-parses YAML. +// +// - Manual declaration: added through the manual-module API by users for +// workload kinds we cannot parse (CRDs, DaemonSets, Argo Rollouts, etc.). +// Manual records carry RevisionBound = 0 — they are version-agnostic and +// load for every revision of the service. +// +// Read-merge rule (see ResolveServiceModules): records are unioned by Name with +// time-of-creation precedence ("first-come-first-served" by CreateTime). +// Conflicts are recorded and surfaced to the caller so the UI can warn users. +type ServiceModule struct { + ID primitive.ObjectID `bson:"_id,omitempty" json:"id,omitempty"` + ProjectName string `bson:"project_name" json:"project_name"` + ServiceName string `bson:"service_name" json:"service_name"` + + // IsManual: true if added explicitly by the user via the manual-module API. + // Manual records MUST have RevisionBound = 0. + IsManual bool `bson:"is_manual" json:"is_manual"` + + // RevisionBound: the service revision this record belongs to. + // 0 -> manual / version-agnostic, loads for every revision + // >0 -> auto-discovered, scoped to that revision only + RevisionBound int64 `bson:"revision_bound" json:"revision_bound"` + + // Name is the module identifier. For auto records it equals the parsed + // container's Name; for manual records it is the user-supplied name that + // $-image$ placeholders in YAML resolve against. + Name string `bson:"name" json:"name"` + + // Type carries the legacy Container.Type distinction (normal vs init + // container). No K8s deployment logic in this codebase branches on this + // field — it is preserved for OpenAPI parity. Manual records default to + // ContainerTypeNormal (""). + Type setting.ContainerType `bson:"type,omitempty" json:"type,omitempty"` + + // Image is the resolved image URI. For auto records it is whatever the + // parsed YAML had; for manual records it is what the user typed. Either + // may be empty at creation time (an unbuilt module). + Image string `bson:"image,omitempty" json:"image,omitempty"` + + // ImageName is the registry-friendly name used for build target paths and + // image webhook matching. Required for manual records (enforced at the + // service layer); for auto records it is derived from Image via + // ExtractImageName, falling back to Name. + ImageName string `bson:"image_name,omitempty" json:"image_name,omitempty"` + + CreateTime int64 `bson:"create_time" json:"create_time"` + UpdateTime int64 `bson:"update_time,omitempty" json:"update_time,omitempty"` +} + +func (ServiceModule) TableName() string { + return "service_module" +} diff --git a/pkg/microservice/aslan/core/common/repository/mongodb/service_module.go b/pkg/microservice/aslan/core/common/repository/mongodb/service_module.go new file mode 100644 index 0000000000..79bb5875bc --- /dev/null +++ b/pkg/microservice/aslan/core/common/repository/mongodb/service_module.go @@ -0,0 +1,270 @@ +/* +Copyright 2026 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mongodb + +import ( + "context" + "time" + + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + + "github.com/koderover/zadig/v2/pkg/microservice/aslan/config" + "github.com/koderover/zadig/v2/pkg/microservice/aslan/core/common/repository/models" + mongotool "github.com/koderover/zadig/v2/pkg/tool/mongo" +) + +const ( + serviceModuleCollName = "service_module" + productionServiceModuleCollName = "production_service_module" +) + +// ServiceModuleColl is the storage for first-class module records. +// Production and non-production records live in separate MongoDB collections +// (same shape) — pick the right constructor for the context, mirroring how +// ServiceColl / ProductionServiceColl is structured. +type ServiceModuleColl struct { + *mongo.Collection + mongo.Session + coll string +} + +func NewServiceModuleColl() *ServiceModuleColl { + return newServiceModuleColl(serviceModuleCollName, nil) +} + +func NewProductionServiceModuleColl() *ServiceModuleColl { + return newServiceModuleColl(productionServiceModuleCollName, nil) +} + +func NewServiceModuleCollWithSession(session mongo.Session) *ServiceModuleColl { + return newServiceModuleColl(serviceModuleCollName, session) +} + +func NewProductionServiceModuleCollWithSession(session mongo.Session) *ServiceModuleColl { + return newServiceModuleColl(productionServiceModuleCollName, session) +} + +func newServiceModuleColl(name string, session mongo.Session) *ServiceModuleColl { + return &ServiceModuleColl{ + Collection: mongotool.Database(config.MongoDatabase()).Collection(name), + Session: session, + coll: name, + } +} + +func (c *ServiceModuleColl) GetCollectionName() string { + return c.coll +} + +func (c *ServiceModuleColl) EnsureIndex(ctx context.Context) error { + mod := []mongo.IndexModel{ + // Uniqueness: a given slot (manual or per-revision auto) holds at most + // one record per name. Manual and auto with the same name coexist — + // the merge layer (ResolveServiceModules) reconciles them. + { + Keys: bson.D{ + bson.E{Key: "project_name", Value: 1}, + bson.E{Key: "service_name", Value: 1}, + bson.E{Key: "is_manual", Value: 1}, + bson.E{Key: "revision_bound", Value: 1}, + bson.E{Key: "name", Value: 1}, + }, + Options: options.Index().SetUnique(true), + }, + // Hot path: list-all-modules-for-a-service. + { + Keys: bson.D{ + bson.E{Key: "project_name", Value: 1}, + bson.E{Key: "service_name", Value: 1}, + }, + Options: options.Index().SetUnique(false), + }, + } + _, err := c.Indexes().CreateMany(ctx, mod, mongotool.CreateIndexOptions(ctx)) + return err +} + +// ListByServiceRevision returns the records relevant to a single read: +// every manual record for the service (RevisionBound = 0) plus the auto +// records bound to the supplied revision. The merge logic in +// ResolveServiceModules deduplicates and applies the precedence rule. +func (c *ServiceModuleColl) ListByServiceRevision(ctx context.Context, projectName, serviceName string, revision int64) ([]*models.ServiceModule, error) { + query := bson.M{ + "project_name": projectName, + "service_name": serviceName, + "$or": bson.A{ + bson.M{"is_manual": true}, + bson.M{"is_manual": false, "revision_bound": revision}, + }, + } + return c.findAll(ctx, query, options.Find().SetSort(bson.D{ + bson.E{Key: "create_time", Value: 1}, + bson.E{Key: "_id", Value: 1}, + })) +} + +// ListManual returns every manual record for a service. Used by the manual- +// module CRUD API to list user-declared modules independently of any revision. +func (c *ServiceModuleColl) ListManual(ctx context.Context, projectName, serviceName string) ([]*models.ServiceModule, error) { + query := bson.M{ + "project_name": projectName, + "service_name": serviceName, + "is_manual": true, + } + return c.findAll(ctx, query, options.Find().SetSort(bson.D{ + bson.E{Key: "create_time", Value: 1}, + bson.E{Key: "_id", Value: 1}, + })) +} + +// ListAutoByRevision returns just the auto-discovered records for one revision. +// Used by the write side (SetCurrentContainerImages flow) when it needs to +// reconcile parsed-from-YAML records against the existing snapshot. +func (c *ServiceModuleColl) ListAutoByRevision(ctx context.Context, projectName, serviceName string, revision int64) ([]*models.ServiceModule, error) { + query := bson.M{ + "project_name": projectName, + "service_name": serviceName, + "is_manual": false, + "revision_bound": revision, + } + return c.findAll(ctx, query, nil) +} + +func (c *ServiceModuleColl) findAll(ctx context.Context, query bson.M, opts *options.FindOptions) ([]*models.ServiceModule, error) { + cursor, err := c.Collection.Find(mongotool.SessionContext(ctx, c.Session), query, opts) + if err != nil { + return nil, err + } + defer cursor.Close(ctx) + resp := make([]*models.ServiceModule, 0) + if err := cursor.All(ctx, &resp); err != nil { + return nil, err + } + return resp, nil +} + +// CreateManual inserts a single user-declared module. The caller is +// responsible for validating that ImageName is non-empty and that no manual +// record with the same name already exists (uniqueness index will also enforce +// this and surface a duplicate-key error). +// +// ImageName is defensively defaulted to Name if empty — the API layer should +// already require it, but persisting an empty value would break read paths +// that no longer carry the legacy GetImageNameFromContainerInfo fallback. +func (c *ServiceModuleColl) CreateManual(ctx context.Context, m *models.ServiceModule) error { + m.IsManual = true + m.RevisionBound = 0 + if m.ImageName == "" { + m.ImageName = m.Name + } + now := time.Now().Unix() + if m.CreateTime == 0 { + m.CreateTime = now + } + m.UpdateTime = now + _, err := c.Collection.InsertOne(mongotool.SessionContext(ctx, c.Session), m) + return err +} + +// UpdateManual replaces a manual record's mutable fields by ID. +func (c *ServiceModuleColl) UpdateManual(ctx context.Context, id primitive.ObjectID, image, imageName string) error { + update := bson.M{ + "$set": bson.M{ + "image": image, + "image_name": imageName, + "update_time": time.Now().Unix(), + }, + } + _, err := c.Collection.UpdateOne( + mongotool.SessionContext(ctx, c.Session), + bson.M{"_id": id, "is_manual": true}, + update, + ) + return err +} + +// DeleteByID removes a single record by ObjectID. Used for manual-module +// deletion from the API. +func (c *ServiceModuleColl) DeleteByID(ctx context.Context, id primitive.ObjectID) error { + _, err := c.Collection.DeleteOne(mongotool.SessionContext(ctx, c.Session), bson.M{"_id": id}) + return err +} + +// ReplaceAutoForRevision atomically replaces the auto-discovered records for +// one (service, revision). Used by SetCurrentContainerImages on every YAML +// re-parse. Manual records are not touched. +// +// "Atomic" here means within the deletion-and-insertion window: if there is +// no session, a concurrent reader of the same (service, revision) may observe +// an empty auto set for a brief moment. Pass a session via *WithSession to +// guarantee snapshot consistency. +func (c *ServiceModuleColl) ReplaceAutoForRevision(ctx context.Context, projectName, serviceName string, revision int64, records []*models.ServiceModule) error { + sessCtx := mongotool.SessionContext(ctx, c.Session) + + if _, err := c.Collection.DeleteMany(sessCtx, bson.M{ + "project_name": projectName, + "service_name": serviceName, + "is_manual": false, + "revision_bound": revision, + }); err != nil { + return err + } + if len(records) == 0 { + return nil + } + + now := time.Now().Unix() + docs := make([]interface{}, 0, len(records)) + for _, r := range records { + r.ProjectName = projectName + r.ServiceName = serviceName + r.IsManual = false + r.RevisionBound = revision + if r.CreateTime == 0 { + r.CreateTime = now + } + r.UpdateTime = now + docs = append(docs, r) + } + _, err := c.Collection.InsertMany(sessCtx, docs) + return err +} + +// DeleteAutoByRevision drops every auto record for a specific revision. Used +// when a service revision is hard-deleted (rare cleanup path). +func (c *ServiceModuleColl) DeleteAutoByRevision(ctx context.Context, projectName, serviceName string, revision int64) error { + _, err := c.Collection.DeleteMany(mongotool.SessionContext(ctx, c.Session), bson.M{ + "project_name": projectName, + "service_name": serviceName, + "is_manual": false, + "revision_bound": revision, + }) + return err +} + +// DeleteByService cascades: removes every record (manual and auto, all +// revisions) for one service. Called when a service template is deleted. +func (c *ServiceModuleColl) DeleteByService(ctx context.Context, projectName, serviceName string) error { + _, err := c.Collection.DeleteMany(mongotool.SessionContext(ctx, c.Session), bson.M{ + "project_name": projectName, + "service_name": serviceName, + }) + return err +} diff --git a/pkg/microservice/aslan/core/common/service/environment.go b/pkg/microservice/aslan/core/common/service/environment.go index c4389a3334..f50758de7d 100644 --- a/pkg/microservice/aslan/core/common/service/environment.go +++ b/pkg/microservice/aslan/core/common/service/environment.go @@ -1045,7 +1045,16 @@ func QueryPodsStatus(productInfo *commonmodels.Product, serviceTmpl *commonmodel resp.Ingress = svcResp.Ingress resp.Workloads = svcResp.Workloads - if len(serviceTmpl.Containers) == 0 { + // Service.Containers no longer persisted — check the module count via the + // merged view. Pure-CRD services with only manual modules still report + // PodReady here because they have no native workload readiness signal + // (intentional, matching the prior "no parsed containers" branch). + resolvedContainers, _, rerr := repository.ResolveServiceModules(context.Background(), serviceTmpl.ProductName, serviceTmpl.ServiceName, productInfo.Production, serviceTmpl.Revision) + if rerr != nil { + log.Errorf("failed to resolve modules for %s/%s rev %d: %s", serviceTmpl.ProductName, serviceTmpl.ServiceName, serviceTmpl.Revision, rerr) + // Fall through: treat as no-modules to avoid blocking status display. + } + if len(resolvedContainers) == 0 { resp.PodStatus = setting.PodSucceeded resp.Ready = setting.PodReady return resp diff --git a/pkg/microservice/aslan/core/common/service/helm/helm.go b/pkg/microservice/aslan/core/common/service/helm/helm.go index 340487b92f..ef38abab29 100644 --- a/pkg/microservice/aslan/core/common/service/helm/helm.go +++ b/pkg/microservice/aslan/core/common/service/helm/helm.go @@ -540,6 +540,12 @@ func (s *HelmDeployService) GenNewEnvService(prod *commonmodels.Product, service return nil, nil, errors.Wrapf(err, "failed to find service %s/%d in product %s", serviceName, svcFindOption.Revision, prod.ProductName) } + // Service.Containers no longer persisted — pull modules for the + // loaded template revision from the service_module table. + tmplContainers, _, rerr := repository.ResolveServiceModules(context.Background(), tmplSvc.ProductName, tmplSvc.ServiceName, prod.Production, tmplSvc.Revision) + if rerr != nil { + return nil, nil, errors.Wrapf(rerr, "failed to resolve modules for %s/%s rev %d", tmplSvc.ProductName, tmplSvc.ServiceName, tmplSvc.Revision) + } if prodSvc == nil { prodSvc = &commonmodels.ProductService{ ServiceName: serviceName, @@ -547,7 +553,7 @@ func (s *HelmDeployService) GenNewEnvService(prod *commonmodels.Product, service ProductName: prod.ProductName, Type: tmplSvc.Type, Revision: tmplSvc.Revision, - Containers: tmplSvc.Containers, + Containers: tmplContainers, } } else { prodSvc.Revision = tmplSvc.Revision @@ -557,7 +563,7 @@ func (s *HelmDeployService) GenNewEnvService(prod *commonmodels.Product, service containerMap[container.Name] = container } - for _, templateContainer := range tmplSvc.Containers { + for _, templateContainer := range tmplContainers { if containerMap[templateContainer.Name] == nil { prodSvc.Containers = append(prodSvc.Containers, templateContainer) } else { diff --git a/pkg/microservice/aslan/core/common/service/kube/parse.go b/pkg/microservice/aslan/core/common/service/kube/parse.go index 1ff2411acb..41a28a1946 100644 --- a/pkg/microservice/aslan/core/common/service/kube/parse.go +++ b/pkg/microservice/aslan/core/common/service/kube/parse.go @@ -22,6 +22,7 @@ import ( "strings" commonmodels "github.com/koderover/zadig/v2/pkg/microservice/aslan/core/common/repository/models" + commonutil "github.com/koderover/zadig/v2/pkg/microservice/aslan/core/common/util" ) const ( @@ -62,14 +63,6 @@ func ParseSysKeys(namespace, envName, productName, serviceName, ori string) stri return ori } -// IsModuleImagePlaceholder reports whether s is exactly a $-image$ -// placeholder (no surrounding text). Auto-detection uses this to avoid -// storing placeholder strings as resolved image URIs. -func IsModuleImagePlaceholder(s string) bool { - loc := moduleImageRegex.FindStringIndex(s) - return loc != nil && loc[0] == 0 && loc[1] == len(s) -} - // ParseModuleImageKeys substitutes $-image$ placeholders in yaml using // the provided container list (container.Name -> container.Image). Containers // with an empty or placeholder-shaped Image are skipped — they cannot resolve @@ -83,7 +76,19 @@ func ParseModuleImageKeys(yaml string, containers []*commonmodels.Container, all if c == nil || c.Name == "" || c.Image == "" { continue } - if IsModuleImagePlaceholder(c.Image) { + if commonutil.IsModuleImagePlaceholder(c.Image) { + continue + } + // Also skip the Zadig workflow-task-create placeholder convention + // (e.g. "{{ NOT BE RENDERED }}"). Substituting it into the YAML + // produces flow-style garbage that downstream YAML decoders choke + // on. For built-in workload kinds, ReplaceWorkloadImages still + // writes this placeholder into container.image structurally, so the + // preview yaml round-trips. For non-built-in kinds (CRDs, etc.), + // the $-image$ placeholder stays as text — paired with + // AllowUnresolvedModuleImages on the caller, this is fine. + trimmedImage := strings.TrimSpace(c.Image) + if strings.HasPrefix(trimmedImage, "{{") && strings.HasSuffix(trimmedImage, "}}") { continue } pattern := regexp.MustCompile(`\$` + regexp.QuoteMeta(c.Name) + `-image\$`) diff --git a/pkg/microservice/aslan/core/common/service/kube/render.go b/pkg/microservice/aslan/core/common/service/kube/render.go index 88ab8fc813..c818a811d7 100644 --- a/pkg/microservice/aslan/core/common/service/kube/render.go +++ b/pkg/microservice/aslan/core/common/service/kube/render.go @@ -18,6 +18,7 @@ package kube import ( "bytes" + "context" "fmt" "regexp" "sort" @@ -417,6 +418,19 @@ func FetchCurrentAppliedYaml(option *GeneSvcYamlOption) (string, int, error) { return "", 0, err } fullRenderedYaml = ParseSysKeys(productInfo.Namespace, productInfo.EnvName, option.ProductName, option.ServiceName, fullRenderedYaml) + + // Post-deprecation: Service.Containers is bson:"-", so the DB-loaded + // prodSvcTemplate.Containers is nil. Re-parse the rendered YAML to + // recover env-resolved auto entries — same pattern as + // GenerateRenderedYaml. No manual fold here: this function surfaces + // what's currently applied in env, and curProductSvc.Containers is + // the authoritative env snapshot (which upsertService now writes + // correctly, including manual modules). Manuals added after the + // last reconcile aren't applied yet and intentionally don't show + // here — that's "current applied", not "what would be applied". + prodSvcTemplate.KubeYamls = util.SplitYaml(fullRenderedYaml) + commonutil.SetCurrentContainerImages(prodSvcTemplate) + mergedContainers := mergeContainers(prodSvcTemplate.Containers, curProductSvc.Containers) fullRenderedYaml, _, err = ReplaceWorkloadImages(fullRenderedYaml, mergedContainers) if err != nil { @@ -882,6 +896,18 @@ func GenerateRenderedYaml(option *GeneSvcYamlOption) (string, int, []*WorkloadRe latestSvcTemplate.KubeYamls = util.SplitYaml(fullRenderedYaml) commonutil.SetCurrentContainerImages(latestSvcTemplate) + // Fold user-declared manual modules into the freshly-parsed slice so + // latestSvcTemplate.Containers represents the complete module set for this + // render. The legacy mergeContainers priority chain + // (cur → tmpl → env-touched → workflow option) is then preserved verbatim: + // workflow option still overrides, env-touched still wins second, manuals + // participate at the template-baseline level. + manualContainers, err := repository.ListManualServiceModules(context.Background(), option.ProductName, option.ServiceName, productInfo.Production) + if err != nil { + return "", 0, nil, fmt.Errorf("failed to list manual service modules: %v", err) + } + latestSvcTemplate.Containers = commonutil.FoldManualModulesInto(latestSvcTemplate.Containers, manualContainers) + mergedContainers := mergeContainers(curContainers, latestSvcTemplate.Containers, svcContainersInProduct, option.Containers) fullRenderedYaml, workloadResource, err := ReplaceWorkloadImages(fullRenderedYaml, mergedContainers) if err != nil { @@ -1005,11 +1031,31 @@ func RenderEnvServiceWithTempl(prod *commonmodels.Product, serviceRender *templa return "", err } parsedYaml = ParseSysKeys(prod.Namespace, prod.EnvName, prod.ProductName, service.ServiceName, parsedYaml) - parsedYaml, _, err = ReplaceWorkloadImages(parsedYaml, service.Containers) + + // Fold manual modules into service.Containers before substitution. + // + // service.Containers nominally reflects the env's runtime snapshot, but + // callers like buildPreviewCandidateOverrides clone a ProductService and + // only override Revision — Containers still reflect the OLD revision's + // state. When the new revision's YAML carries $-image$ for a + // module that wasn't in the env before, substitution fails without this + // re-merge. + // + // For "true env snapshot" callers (e.g. genuinely rendering what's in + // env), the fold is a no-op when env state is consistent. The only + // observable difference is in the candidate-render path, which is what + // we want. + manualContainers, err := repository.ListManualServiceModules(context.Background(), prod.ProductName, service.ServiceName, prod.Production) + if err != nil { + return "", fmt.Errorf("failed to list manual service modules: %v", err) + } + allContainers := commonutil.FoldManualModulesInto(service.Containers, manualContainers) + + parsedYaml, _, err = ReplaceWorkloadImages(parsedYaml, allContainers) if err != nil { return "", err } - parsedYaml, err = ParseModuleImageKeys(parsedYaml, service.Containers, false) + parsedYaml, err = ParseModuleImageKeys(parsedYaml, allContainers, false) if err != nil { return "", err } diff --git a/pkg/microservice/aslan/core/common/service/repository/service.go b/pkg/microservice/aslan/core/common/service/repository/service.go index 675e975729..048d60b315 100644 --- a/pkg/microservice/aslan/core/common/service/repository/service.go +++ b/pkg/microservice/aslan/core/common/service/repository/service.go @@ -17,9 +17,13 @@ limitations under the License. package repository import ( + "context" + + "go.mongodb.org/mongo-driver/mongo" + "github.com/koderover/zadig/v2/pkg/microservice/aslan/core/common/repository/models" "github.com/koderover/zadig/v2/pkg/microservice/aslan/core/common/repository/mongodb" - "go.mongodb.org/mongo-driver/mongo" + "github.com/koderover/zadig/v2/pkg/tool/log" ) type IServiceColl interface { @@ -29,10 +33,47 @@ type IServiceColl interface { } func ServiceCollWithSession(production bool, session mongo.Session) IServiceColl { + var inner IServiceColl if !production { - return mongodb.NewServiceCollWithSession(session) + inner = mongodb.NewServiceCollWithSession(session) + } else { + inner = mongodb.NewProductionServiceCollWithSession(session) + } + return &serviceCollWithModuleSync{IServiceColl: inner, production: production} +} + +// serviceCollWithModuleSync wraps an IServiceColl with side-effects that keep +// the service_module collection in sync. The wrapper is transparent for +// reads; Create / Delete are intercepted to mirror auto records into the new +// table (best-effort during Phase 3 — old field stays authoritative). +// +// Caveat: the wrapped writes here happen outside the mongo session if the +// underlying coll was constructed with one. A transaction rollback on the +// inner Create will leave the synced auto records in place. Acceptable while +// the new table is non-authoritative; revisit once the read switch lands. +type serviceCollWithModuleSync struct { + IServiceColl + production bool +} + +func (s *serviceCollWithModuleSync) Create(args *models.Service) error { + if err := s.IServiceColl.Create(args); err != nil { + return err } - return mongodb.NewProductionServiceCollWithSession(session) + syncAutoModulesBestEffort(args, s.production, "Create") + return nil +} + +func (s *serviceCollWithModuleSync) Delete(serviceName, serviceType, productName, status string, revision int64) error { + if err := s.IServiceColl.Delete(serviceName, serviceType, productName, status, revision); err != nil { + return err + } + if revision > 0 { + if dErr := DeleteAutoServiceModulesForRevision(context.Background(), productName, serviceName, s.production, revision); dErr != nil { + log.Warnf("service_module: Delete failed to drop auto records for %s/%s rev %d: %s", productName, serviceName, revision, dErr) + } + } + return nil } func QueryTemplateService(option *mongodb.ServiceFindOption, production bool) (*models.Service, error) { @@ -106,11 +147,17 @@ func UpdateServiceVariables(args *models.Service, production bool) error { } func UpdateServiceContainers(args *models.Service, production bool) error { + var err error if !production { - return mongodb.NewServiceColl().UpdateServiceContainers(args) + err = mongodb.NewServiceColl().UpdateServiceContainers(args) } else { - return mongodb.NewProductionServiceColl().UpdateServiceContainers(args) + err = mongodb.NewProductionServiceColl().UpdateServiceContainers(args) + } + if err != nil { + return err } + syncAutoModulesBestEffort(args, production, "UpdateServiceContainers") + return nil } func UpdateStatus(serviceName, productName, status string, production bool) error { @@ -138,17 +185,48 @@ func UpdateWithSession(service *models.Service, production bool, session mongo.S } func Create(service *models.Service, production bool) error { + var err error if !production { - return mongodb.NewServiceColl().Create(service) + err = mongodb.NewServiceColl().Create(service) } else { - return mongodb.NewProductionServiceColl().Create(service) + err = mongodb.NewProductionServiceColl().Create(service) + } + if err != nil { + return err } + syncAutoModulesBestEffort(service, production, "Create") + return nil } func Delete(serviceName, serviceType, productName, status string, revision int64, production bool) error { + var err error if !production { - return mongodb.NewServiceColl().Delete(serviceName, serviceType, productName, status, revision) + err = mongodb.NewServiceColl().Delete(serviceName, serviceType, productName, status, revision) } else { - return mongodb.NewProductionServiceColl().Delete(serviceName, serviceType, productName, status, revision) + err = mongodb.NewProductionServiceColl().Delete(serviceName, serviceType, productName, status, revision) + } + if err != nil { + return err + } + // Per-revision delete — drop the matching auto records, leave manual alone. + // Revision 0 means "all revisions" in some callers; skip the new-table + // cleanup in that case to avoid wiping cross-revision auto data, which + // DeleteAllServiceModulesForService should handle instead. + if revision > 0 { + if dErr := DeleteAutoServiceModulesForRevision(context.Background(), productName, serviceName, production, revision); dErr != nil { + log.Warnf("service_module: failed to delete auto records for %s/%s rev %d: %s", productName, serviceName, revision, dErr) + } + } + return nil +} + +// syncAutoModulesBestEffort writes the auto-discovered modules to the new +// service_module collection. Failure is logged but not propagated — during +// Phase 3 the legacy Service.Containers field remains authoritative; the new +// table is being populated for the read-path switch later. +func syncAutoModulesBestEffort(svc *models.Service, production bool, callSite string) { + if err := SyncAutoServiceModules(context.Background(), svc, production); err != nil { + log.Warnf("service_module: %s failed to sync auto modules for %s/%s rev %d: %s", + callSite, svc.ProductName, svc.ServiceName, svc.Revision, err) } } diff --git a/pkg/microservice/aslan/core/common/service/repository/service_module.go b/pkg/microservice/aslan/core/common/service/repository/service_module.go new file mode 100644 index 0000000000..c0caee1462 --- /dev/null +++ b/pkg/microservice/aslan/core/common/service/repository/service_module.go @@ -0,0 +1,212 @@ +/* +Copyright 2026 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package repository + +import ( + "context" + + "github.com/koderover/zadig/v2/pkg/microservice/aslan/core/common/repository/models" + "github.com/koderover/zadig/v2/pkg/microservice/aslan/core/common/repository/mongodb" +) + +// ModuleConflict reports that two records share a Name. The Winner is the +// record currently in effect under the precedence rule (see +// ResolveServiceModules); Shadowed lists the records that were displaced. +// Callers (typically API handlers) should surface these to the UI so users +// can disambiguate — e.g., "module 'api' has both a manual declaration and +// an auto-discovered entry; the manual one is winning." +type ModuleConflict struct { + Name string + Winner *models.ServiceModule + Shadowed []*models.ServiceModule +} + +// ResolveServiceModules returns the merged module list for one (project, +// service, revision) plus any name conflicts. Production picks the right +// underlying collection (service_module vs production_service_module). +// +// Merge rule: time-of-creation precedence ("first-come-first-served"). For +// every Name in the union of (manual records, auto records bound to +// `revision`), the record with the smallest CreateTime wins; later records +// with the same Name are reported as shadowed. Ties (same CreateTime) are +// broken by ObjectID order — deterministic, but "shouldn't happen in +// practice." +// +// The returned []*models.Container is a drop-in replacement for the legacy +// Service.Containers field, so callers migrating off the field only need to +// swap their data source. +func ResolveServiceModules(ctx context.Context, projectName, serviceName string, production bool, revision int64) ([]*models.Container, []ModuleConflict, error) { + coll := pickServiceModuleColl(production) + records, err := coll.ListByServiceRevision(ctx, projectName, serviceName, revision) + if err != nil { + return nil, nil, err + } + return mergeServiceModules(records), conflictsFromMerge(records), nil +} + +// SyncAutoServiceModules upserts the parsed containers from svc.Containers +// into the service_module collection as auto records bound to svc.Revision. +// Idempotent: replaces the existing auto records for (service, revision) +// atomically. +// +// Called from Create / UpdateServiceContainers in this package — see those +// functions for the integration points. External callers (e.g. backfill in +// UpgradeAssistant) can call this directly. +// +// Manual records for the service are untouched. +func SyncAutoServiceModules(ctx context.Context, svc *models.Service, production bool) error { + if svc == nil || svc.ServiceName == "" || svc.ProductName == "" || svc.Revision == 0 { + return nil + } + coll := pickServiceModuleColl(production) + records := containersToAutoRecords(svc.ProductName, svc.ServiceName, svc.Revision, svc.Containers) + return coll.ReplaceAutoForRevision(ctx, svc.ProductName, svc.ServiceName, svc.Revision, records) +} + +// ListManualServiceModules returns just the manual records for a service, +// mapped to Container shape. Render paths use this to inject user-declared +// modules into the substitution pipeline alongside the in-memory parsed +// containers (which only cover built-in workload kinds). +func ListManualServiceModules(ctx context.Context, projectName, serviceName string, production bool) ([]*models.Container, error) { + if projectName == "" || serviceName == "" { + return nil, nil + } + records, err := pickServiceModuleColl(production).ListManual(ctx, projectName, serviceName) + if err != nil { + return nil, err + } + out := make([]*models.Container, 0, len(records)) + for _, r := range records { + out = append(out, &models.Container{ + Name: r.Name, + Type: r.Type, + Image: r.Image, + ImageName: r.ImageName, + }) + } + return out, nil +} + +// DeleteAutoServiceModulesForRevision drops every auto record for one +// (service, revision). Used by Delete in this package when a specific +// revision is reaped. Manual records are untouched. +func DeleteAutoServiceModulesForRevision(ctx context.Context, projectName, serviceName string, production bool, revision int64) error { + if projectName == "" || serviceName == "" || revision == 0 { + return nil + } + return pickServiceModuleColl(production).DeleteAutoByRevision(ctx, projectName, serviceName, revision) +} + +// DeleteAllServiceModulesForService cascades: removes every record (manual +// and auto, all revisions) for one service. Called when a service template +// is fully deleted (DeleteServiceTemplate). +func DeleteAllServiceModulesForService(ctx context.Context, projectName, serviceName string, production bool) error { + if projectName == "" || serviceName == "" { + return nil + } + return pickServiceModuleColl(production).DeleteByService(ctx, projectName, serviceName) +} + +func pickServiceModuleColl(production bool) *mongodb.ServiceModuleColl { + if production { + return mongodb.NewProductionServiceModuleColl() + } + return mongodb.NewServiceModuleColl() +} + +func containersToAutoRecords(projectName, serviceName string, revision int64, containers []*models.Container) []*models.ServiceModule { + records := make([]*models.ServiceModule, 0, len(containers)) + for _, c := range containers { + if c == nil || c.Name == "" { + continue + } + records = append(records, &models.ServiceModule{ + ProjectName: projectName, + ServiceName: serviceName, + IsManual: false, + RevisionBound: revision, + Name: c.Name, + Type: c.Type, + Image: c.Image, + ImageName: normalizeImageName(c.ImageName, c.Name), + }) + } + return records +} + +// normalizeImageName mirrors the legacy GetImageNameFromContainerInfo +// fallback (empty ImageName falls back to Name) at write time. Stored data +// is therefore always non-empty for ImageName, so read paths don't need to +// keep the fallback around. Applies to both auto and manual records. +func normalizeImageName(imageName, name string) string { + if imageName != "" { + return imageName + } + return name +} + +// mergeServiceModules applies the first-come-first-served rule and returns +// the winning records mapped to Container shape. Input MUST be pre-sorted by +// CreateTime ascending then ObjectID — ListByServiceRevision already does so. +func mergeServiceModules(records []*models.ServiceModule) []*models.Container { + winners := make([]*models.Container, 0, len(records)) + seen := make(map[string]struct{}, len(records)) + for _, r := range records { + if _, ok := seen[r.Name]; ok { + continue + } + seen[r.Name] = struct{}{} + winners = append(winners, &models.Container{ + Name: r.Name, + Type: r.Type, + Image: r.Image, + ImageName: r.ImageName, + }) + } + return winners +} + +// TODO: convenience wrapper ResolveServiceModulesFor(ctx, svc, production) +// that infers project/service/revision from svc — deferred per discussion, +// callers will pass the explicit args until usage volume justifies it. + +// conflictsFromMerge walks the same pre-sorted slice and groups shadowed +// records under their winning entry. Returned slice is empty when there are +// no name collisions. +func conflictsFromMerge(records []*models.ServiceModule) []ModuleConflict { + byName := make(map[string][]*models.ServiceModule, len(records)) + order := make([]string, 0, len(records)) + for _, r := range records { + if _, ok := byName[r.Name]; !ok { + order = append(order, r.Name) + } + byName[r.Name] = append(byName[r.Name], r) + } + conflicts := make([]ModuleConflict, 0) + for _, name := range order { + group := byName[name] + if len(group) < 2 { + continue + } + conflicts = append(conflicts, ModuleConflict{ + Name: name, + Winner: group[0], + Shadowed: group[1:], + }) + } + return conflicts +} diff --git a/pkg/microservice/aslan/core/common/service/service.go b/pkg/microservice/aslan/core/common/service/service.go index 207da6640e..1310412db2 100644 --- a/pkg/microservice/aslan/core/common/service/service.go +++ b/pkg/microservice/aslan/core/common/service/service.go @@ -1107,11 +1107,13 @@ func BuildServiceInfoInEnv(productInfo *commonmodels.Product, templateSvcs []*co templateSvcMap[svc.ServiceName] = svc svcModulesMap[svc.ServiceName] = make(map[string]*commonmodels.Container) - for _, container := range svc.Containers { - // templateSvcs is default - if _, ok := svcModulesMap[svc.ServiceName]; !ok { - svcModulesMap[svc.ServiceName] = make(map[string]*commonmodels.Container) - } + // Service.Containers is no longer persisted — read modules from the + // service_module collection. + resolved, _, rerr := repository.ResolveServiceModules(context.Background(), svc.ProductName, svc.ServiceName, productInfo.Production, svc.Revision) + if rerr != nil { + return nil, e.ErrGetService.AddErr(errors.Wrapf(rerr, "failed to resolve modules for %s/%s rev %d", svc.ProductName, svc.ServiceName, svc.Revision)) + } + for _, container := range resolved { svcModulesMap[svc.ServiceName][container.Name] = container } } diff --git a/pkg/microservice/aslan/core/common/service/workflowcontroller/jobcontroller/job_deploy.go b/pkg/microservice/aslan/core/common/service/workflowcontroller/jobcontroller/job_deploy.go index 6b4cb9be2b..87c2f37ef7 100644 --- a/pkg/microservice/aslan/core/common/service/workflowcontroller/jobcontroller/job_deploy.go +++ b/pkg/microservice/aslan/core/common/service/workflowcontroller/jobcontroller/job_deploy.go @@ -794,6 +794,18 @@ func GetResourcesPodOwnerUID(kubeClient client.Client, namespace string, service func (c *DeployJobCtl) wait(ctx context.Context) { jobLogCtx := &joblog.JobLogContext{WorkflowCtx: c.workflowCtx, JobTask: c.job} + // No trackable native workloads (CRD-only / ConfigMap-only services + // produce an empty ReplaceResources because the kind switch upstream + // only tracks Deployment/StatefulSet/CronJob/Job). Native readiness + // doesn't apply to these — mark passed and skip the wait/check loops + // to avoid spinning forever on "Waiting for workloads to be created". + if len(c.jobTaskSpec.ReplaceResources) == 0 { + jobLogManager := joblog.NewJobLogManager(jobLogCtx) + jobLogManager.SaveJobLog("No native workload (Deployment / StatefulSet / CronJob / Job) in service yaml — skipping readiness check.") + c.job.Status = config.StatusPassed + return + } + timeout := time.After(time.Duration(c.timeout()) * time.Second) resources, err := GetResourcesPodOwnerUID(c.kubeClient, c.namespace, c.jobTaskSpec.ServiceAndImages, c.jobTaskSpec.DeployContents, c.jobTaskSpec.ReplaceResources, jobLogCtx) if err != nil { diff --git a/pkg/microservice/aslan/core/common/util/service.go b/pkg/microservice/aslan/core/common/util/service.go index 84be6be297..fc8f654882 100644 --- a/pkg/microservice/aslan/core/common/util/service.go +++ b/pkg/microservice/aslan/core/common/util/service.go @@ -45,11 +45,72 @@ var ( {setting.PathSearchComponentImage: "image"}, } - // Duplicated from kube.moduleImageRegex — kept here to avoid an import - // cycle (kube imports util). Keep both regexes in sync. moduleImagePlaceholderRegex = regexp.MustCompile(`^\$[A-Za-z0-9_-]+-image\$$`) ) +// IsModuleImagePlaceholder reports whether s is exactly a $-image$ +// placeholder string (no surrounding text). Used to avoid persisting an +// unresolved placeholder as a stored image URI. +func IsModuleImagePlaceholder(s string) bool { + return moduleImagePlaceholderRegex.MatchString(s) +} + +// FoldManualModulesInto merges user-declared manual modules into an already- +// populated auto-container slice in place. The intended call site is right +// after SetCurrentContainerImages on a service template: that call fills the +// slice with env-resolved auto entries (parsed from rendered YAML), and this +// helper grafts manual entries on top so the resulting slice represents the +// complete module set for the render context. +// +// Rules: +// - For an auto entry whose Image is empty or a $-image$ placeholder, +// a same-named manual entry's Image/ImageName replaces the auto values. +// (Covers the case where the YAML carries the placeholder literally and +// the manual record holds the real default image.) +// - Manual entries whose Name does not appear in the auto slice are +// appended at the end. +// - An auto entry with a real Image always wins over a same-named manual +// entry (auto reflects the env-resolved YAML, which is more current). +// +// Returns the resulting slice (may be the same backing array as autos, may be +// a longer one if manuals were appended). Caller should assign back. +func FoldManualModulesInto(autos []*commonmodels.Container, manuals []*commonmodels.Container) []*commonmodels.Container { + if len(manuals) == 0 { + return autos + } + manualByName := make(map[string]*commonmodels.Container, len(manuals)) + for _, m := range manuals { + if m == nil || m.Name == "" { + continue + } + manualByName[m.Name] = m + } + seen := make(map[string]struct{}, len(autos)) + for _, c := range autos { + if c == nil || c.Name == "" { + continue + } + seen[c.Name] = struct{}{} + if c.Image != "" && !IsModuleImagePlaceholder(c.Image) { + continue + } + if m, ok := manualByName[c.Name]; ok { + c.Image = m.Image + c.ImageName = m.ImageName + } + } + for _, m := range manuals { + if m == nil || m.Name == "" { + continue + } + if _, ok := seen[m.Name]; ok { + continue + } + autos = append(autos, m) + } + return autos +} + func GetServiceDeployStrategy(serviceName string, strategyMap map[string]setting.ServiceDeployStrategy) setting.ServiceDeployStrategy { if strategyMap == nil { return setting.ServiceDeployStrategyDeploy @@ -228,14 +289,14 @@ func SetCurrentContainerImages(args *commonmodels.Service) error { deduped := uniqueSlice(srvContainers) for _, c := range deduped { - if c == nil || !moduleImagePlaceholderRegex.MatchString(c.Image) { + if c == nil || !IsModuleImagePlaceholder(c.Image) { continue } // The rendered YAML still carries the placeholder for this module // (e.g. an unknown workload kind where ReplaceWorkloadImages can't // substitute by name+kind). Carry the previously-resolved image // forward so we never persist a placeholder as a stored image URI. - if prior, ok := priorByName[c.Name]; ok && prior.Image != "" && !moduleImagePlaceholderRegex.MatchString(prior.Image) { + if prior, ok := priorByName[c.Name]; ok && prior.Image != "" && !IsModuleImagePlaceholder(prior.Image) { c.Image = prior.Image c.ImageName = prior.ImageName } diff --git a/pkg/microservice/aslan/core/delivery/service/openapi.go b/pkg/microservice/aslan/core/delivery/service/openapi.go index 5263bf2a00..c39b821542 100644 --- a/pkg/microservice/aslan/core/delivery/service/openapi.go +++ b/pkg/microservice/aslan/core/delivery/service/openapi.go @@ -17,6 +17,7 @@ package service import ( + "context" "fmt" "go.mongodb.org/mongo-driver/bson/primitive" @@ -582,8 +583,14 @@ func OpenAPICreateHelmDeliveryVersion(openAPIReq *OpenAPICreateHelmDeliveryVersi return fmt.Errorf("can't find service %s in project %s", service.ServiceName, openAPIReq.ProjectKey) } + // Service.Containers no longer persisted — read modules from + // the service_module table. containerMap := map[string]*commonmodels.Container{} - for _, c := range svcTmpl.Containers { + resolved, _, rerr := repository.ResolveServiceModules(context.Background(), svcTmpl.ProductName, svcTmpl.ServiceName, openAPIReq.Production, svcTmpl.Revision) + if rerr != nil { + return fmt.Errorf("failed to resolve modules for %s/%s rev %d: %s", svcTmpl.ProductName, svcTmpl.ServiceName, svcTmpl.Revision, rerr) + } + for _, c := range resolved { containerMap[c.Name] = c } diff --git a/pkg/microservice/aslan/core/environment/service/environment.go b/pkg/microservice/aslan/core/environment/service/environment.go index 2b9fab9efb..0849b755c1 100644 --- a/pkg/microservice/aslan/core/environment/service/environment.go +++ b/pkg/microservice/aslan/core/environment/service/environment.go @@ -790,7 +790,9 @@ func updateProductImpl(updateRevisionSvcs []string, deployStrategy map[string]se service.Error = errFetchImage.Error() return } - service.Containers = containers + // Same fold-merge as k8s.go imported path — preserve + // manual modules that fetchWorkloadImages can't see. + service.Containers = commonutil.FoldManualModulesInto(containers, service.Containers) updateProd.ServiceDeployStrategy = deployStrategy err = commonutil.CreateEnvServiceVersion(updateProd, service, user, config.EnvOperationDefault, "", session, log) @@ -2992,7 +2994,15 @@ func upsertService(env *commonmodels.Product, newService *commonmodels.ProductSe if prevSvc == nil { fakeTemplateSvc := &commonmodels.Service{ServiceName: newService.ServiceName, ProductName: newService.ServiceName, KubeYamls: util.SplitYaml(parsedYaml)} commonutil.SetCurrentContainerImages(fakeTemplateSvc) - newService.Containers = fakeTemplateSvc.Containers + // Merge instead of overwrite. fakeTemplateSvc.Containers is the + // auto set re-parsed from the just-rendered YAML — authoritative + // for env-resolved image values on Deployment/StatefulSet/etc. + // newService.Containers came in from the caller already carrying + // manual modules (CRD/DaemonSet declarations), so blindly clobbering + // it would drop those manuals — and for ConfigMap/CRD-only services + // fakeTemplateSvc.Containers is empty, which would persist an empty + // Containers slice into ProductService and break later renders. + newService.Containers = commonutil.FoldManualModulesInto(fakeTemplateSvc.Containers, newService.Containers) } preResourceYaml := "" diff --git a/pkg/microservice/aslan/core/environment/service/k8s.go b/pkg/microservice/aslan/core/environment/service/k8s.go index 9b263c95d9..1e6faae2cb 100644 --- a/pkg/microservice/aslan/core/environment/service/k8s.go +++ b/pkg/microservice/aslan/core/environment/service/k8s.go @@ -76,7 +76,16 @@ func (k *K8sService) queryServiceStatus(serviceTmpl *commonmodels.Service, produ // queryWorkloadStatus query workload status // only supports Deployment and StatefulSet func (k *K8sService) queryWorkloadStatus(serviceTmpl *commonmodels.Service, productInfo *commonmodels.Product, informer informers.SharedInformerFactory) string { - if len(serviceTmpl.Containers) > 0 { + // Service.Containers no longer persisted — check workload count via the + // merged module list. Pure-CRD services (only manual modules) get an + // empty list here because GetServiceWorkloads only handles built-in + // workload kinds — intentional, CRDs have no native readiness signal. + resolved, _, rerr := repository.ResolveServiceModules(context.Background(), serviceTmpl.ProductName, serviceTmpl.ServiceName, productInfo.Production, serviceTmpl.Revision) + if rerr != nil { + k.log.Errorf("failed to resolve modules for %s/%s rev %d: %s", serviceTmpl.ProductName, serviceTmpl.ServiceName, serviceTmpl.Revision, rerr) + return setting.PodUnstable + } + if len(resolved) > 0 { workloads, err := GetServiceWorkloads(serviceTmpl, productInfo, informer, k.log) if err != nil { k.log.Errorf("failed to get service workloads, err: %s", err) @@ -143,7 +152,20 @@ func (k *K8sService) updateService(args *SvcOptArgs) error { if err != nil { curUsedSvc = nil } - newProductSvc.Containers = kube.CalculateContainer(currentProductSvc, curUsedSvc, latestSvcRevision.Containers, prodinfo) + // Phase 4: pull the merged (auto + manual) module list at the latest + // template revision so manual CRD/DaemonSet modules participate in + // the env override calc, not just legacy-field auto entries. + latestMerged, _, err := repository.ResolveServiceModules( + context.Background(), + latestSvcRevision.ProductName, + latestSvcRevision.ServiceName, + prodinfo.Production, + latestSvcRevision.Revision, + ) + if err != nil { + return e.ErrUpdateService.AddErr(fmt.Errorf("failed to resolve service modules: %s", err)) + } + newProductSvc.Containers = kube.CalculateContainer(currentProductSvc, curUsedSvc, latestMerged, prodinfo) } switch prodinfo.Status { @@ -683,7 +705,11 @@ func (k *K8sService) createGroup(username string, product *commonmodels.Product, if err != nil { return fmt.Errorf("failed to fetch related containers: %s", err) } - group[i].Containers = containers + // Fold-merge instead of overwrite: fetchWorkloadImages only knows + // Deployment/StatefulSet/etc — it does NOT see manual modules + // (CRDs/DaemonSets the user declared). A blind overwrite drops + // those manuals from the env snapshot. + group[i].Containers = commonutil.FoldManualModulesInto(containers, group[i].Containers) err = commonutil.CreateEnvServiceVersion(product, group[i], username, config.EnvOperationDefault, "", nil, k.log) if err != nil { diff --git a/pkg/microservice/aslan/core/environment/service/product.go b/pkg/microservice/aslan/core/environment/service/product.go index fbafe2f82e..5e98cd9987 100644 --- a/pkg/microservice/aslan/core/environment/service/product.go +++ b/pkg/microservice/aslan/core/environment/service/product.go @@ -181,7 +181,14 @@ func GetInitProduct(productTmplName string, envType types.EnvType, isBaseEnv boo } if serviceTmpl.Type == setting.K8SDeployType || serviceTmpl.Type == setting.HelmDeployType { serviceResp.Containers = make([]*commonmodels.Container, 0) - for _, c := range serviceTmpl.Containers { + // Service.Containers no longer persisted — read modules from + // the service_module table. + resolved, _, rerr := repository.ResolveServiceModules(context.Background(), serviceTmpl.ProductName, serviceTmpl.ServiceName, production, serviceTmpl.Revision) + if rerr != nil { + log.Errorf("failed to resolve modules for %s/%s rev %d: %s", serviceTmpl.ProductName, serviceTmpl.ServiceName, serviceTmpl.Revision, rerr) + continue + } + for _, c := range resolved { container := &commonmodels.Container{ Name: c.Name, Image: c.Image, diff --git a/pkg/microservice/aslan/core/environment/service/revision.go b/pkg/microservice/aslan/core/environment/service/revision.go index 88dfcf452e..f014bccad8 100644 --- a/pkg/microservice/aslan/core/environment/service/revision.go +++ b/pkg/microservice/aslan/core/environment/service/revision.go @@ -225,11 +225,19 @@ func compareGroupServicesRev(svcTmplNameList []string, productInfo *commonmodels return serviceRev, nil } -func containerImageChanged(svcTempl *commonmodels.Service, container *commonmodels.Container) bool { +func containerImageChanged(svcTempl *commonmodels.Service, container *commonmodels.Container, production bool) bool { if svcTempl == nil { return true } - for _, c := range svcTempl.Containers { + // Service.Containers is no longer persisted — resolve from the + // service_module table for the template's revision. Errors surface as + // "changed" so the caller falls back to using the env's current image, + // which preserves the conservative legacy semantic. + resolved, _, err := repository.ResolveServiceModules(context.Background(), svcTempl.ProductName, svcTempl.ServiceName, production, svcTempl.Revision) + if err != nil { + return true + } + for _, c := range resolved { if c.Name != container.Name { continue } @@ -285,7 +293,14 @@ func compareServicesRev(serviceTmplNames []string, productServices []*commonmode } if latestSvcTmpl.Type == setting.K8SDeployType { serviceRev.Containers = make([]*commonmodels.Container, 0) - for _, container := range latestSvcTmpl.Containers { + // Service.Containers no longer persisted — pull merged + // (auto + manual) module list for the latest revision. + resolved, _, rerr := repository.ResolveServiceModules(context.Background(), latestSvcTmpl.ProductName, latestSvcTmpl.ServiceName, productInfo.Production, latestSvcTmpl.Revision) + if rerr != nil { + log.Errorf("failed to resolve modules for %s/%s rev %d: %s", latestSvcTmpl.ProductName, latestSvcTmpl.ServiceName, latestSvcTmpl.Revision, rerr) + return serviceRevs, rerr + } + for _, container := range resolved { serviceRev.Containers = append(serviceRev.Containers, &commonmodels.Container{ Image: container.Image, Name: container.Name, @@ -344,17 +359,38 @@ func compareServicesRev(serviceTmplNames []string, productServices []*commonmode if productService.Type == setting.K8SDeployType { serviceRev.Containers = make([]*commonmodels.Container, 0) - for _, container := range latestServiceTmpl.Containers { + // Phase 4: pull the full module list (auto + manual) for the + // latest template revision from the service_module collection. + // Manual modules (CRD/DaemonSet declarations) flow through the + // bridge alongside parsed containers so the env carries them + // in ProductService.Containers after the next reconcile. + mergedContainers, _, err := repository.ResolveServiceModules( + context.Background(), + latestServiceTmpl.ProductName, + latestServiceTmpl.ServiceName, + productInfo.Production, + latestServiceTmpl.Revision, + ) + if err != nil { + log.Errorf("Failed to resolve service modules, %s:%s/%d, Error: %v", productInfo.ProductName, latestServiceTmpl.ServiceName, latestServiceTmpl.Revision, err) + return serviceRevs, err + } + + for _, container := range mergedContainers { c := &commonmodels.Container{ Image: container.Image, Name: container.Name, ImageName: util.GetImageNameFromContainerInfo(container.ImageName, container.Name), } - // reuse existed container image only if it has been changed since last deploy + // reuse existed container image only if it has been changed since last deploy. + // Note: for manual modules, curUsedSvc.Containers (legacy field) doesn't + // know about them, so containerImageChanged always returns true — meaning + // any env-deployed image for a manual module is preserved. That's the + // intended semantic. for _, exitedContainer := range productService.Containers { if exitedContainer.Name == container.Name { - if containerImageChanged(curUsedSvc, exitedContainer) { + if containerImageChanged(curUsedSvc, exitedContainer, productInfo.Production) { c.Image = exitedContainer.Image c.ImageName = exitedContainer.ImageName } diff --git a/pkg/microservice/aslan/core/project/service/openapi.go b/pkg/microservice/aslan/core/project/service/openapi.go index b96dc82e07..743f6b6fcf 100644 --- a/pkg/microservice/aslan/core/project/service/openapi.go +++ b/pkg/microservice/aslan/core/project/service/openapi.go @@ -17,6 +17,7 @@ limitations under the License. package service import ( + "context" "errors" "fmt" "time" @@ -29,6 +30,7 @@ import ( commonrepo "github.com/koderover/zadig/v2/pkg/microservice/aslan/core/common/repository/mongodb" templaterepo "github.com/koderover/zadig/v2/pkg/microservice/aslan/core/common/repository/mongodb/template" commonservice "github.com/koderover/zadig/v2/pkg/microservice/aslan/core/common/service" + "github.com/koderover/zadig/v2/pkg/microservice/aslan/core/common/service/repository" commontypes "github.com/koderover/zadig/v2/pkg/microservice/aslan/core/common/types" commonutil "github.com/koderover/zadig/v2/pkg/microservice/aslan/core/common/util" envService "github.com/koderover/zadig/v2/pkg/microservice/aslan/core/environment/service" @@ -213,7 +215,14 @@ func InitializeYAMLProject(userID, username, requestID string, args *OpenAPIInit } singleService.Containers = make([]*commonmodels.Container, 0) - for _, c := range serviceMap[serviceName].Containers { + // Service.Containers is no longer persisted — pull modules from + // service_module table. This OpenAPI flow is non-production. + tmpl := serviceMap[serviceName] + resolved, _, rerr := repository.ResolveServiceModules(context.Background(), tmpl.ProductName, tmpl.ServiceName, false, tmpl.Revision) + if rerr != nil { + return fmt.Errorf("failed to resolve modules for %s/%s rev %d: %s", tmpl.ProductName, tmpl.ServiceName, tmpl.Revision, rerr) + } + for _, c := range resolved { container := &commonmodels.Container{ Name: c.Name, Image: c.Image, @@ -221,8 +230,8 @@ func InitializeYAMLProject(userID, username, requestID string, args *OpenAPIInit ImageName: util.GetImageNameFromContainerInfo(c.ImageName, c.Name), } singleService.Containers = append(singleService.Containers, container) - singleService.VariableYaml = serviceMap[serviceName].VariableYaml - singleService.VariableKVs = commontypes.ServiceToRenderVariableKVs(serviceMap[serviceName].ServiceVariableKVs) + singleService.VariableYaml = tmpl.VariableYaml + singleService.VariableKVs = commontypes.ServiceToRenderVariableKVs(tmpl.ServiceVariableKVs) } serviceGroup = append(serviceGroup, singleService) } diff --git a/pkg/microservice/aslan/core/project/service/product.go b/pkg/microservice/aslan/core/project/service/product.go index b2a33de512..cd36f82c02 100644 --- a/pkg/microservice/aslan/core/project/service/product.go +++ b/pkg/microservice/aslan/core/project/service/product.go @@ -955,7 +955,15 @@ func ListTemplatesHierachy(userName string, log *zap.SugaredLogger) ([]*ProductI for _, svcTmpl := range services { sInfo := &ServiceInfo{Value: svcTmpl.ServiceName, Label: svcTmpl.ServiceName, ContainerInfo: make([]*ContainerInfo, 0)} - for _, c := range svcTmpl.Containers { + // Service.Containers is no longer persisted — read modules from the + // service_module collection so manual entries (CRD/DaemonSet) also + // show up in the project info response. + containers, _, err := repository.ResolveServiceModules(context.Background(), svcTmpl.ProductName, svcTmpl.ServiceName, false, svcTmpl.Revision) + if err != nil { + log.Errorf("failed to resolve modules for %s/%s rev %d: %s", svcTmpl.ProductName, svcTmpl.ServiceName, svcTmpl.Revision, err) + return nil, e.ErrGetProduct.AddDesc(err.Error()) + } + for _, c := range containers { sInfo.ContainerInfo = append(sInfo.ContainerInfo, &ContainerInfo{Value: c.Name, Label: c.Name}) } diff --git a/pkg/microservice/aslan/core/service/handler/manual_module.go b/pkg/microservice/aslan/core/service/handler/manual_module.go new file mode 100644 index 0000000000..c199281409 --- /dev/null +++ b/pkg/microservice/aslan/core/service/handler/manual_module.go @@ -0,0 +1,208 @@ +/* +Copyright 2026 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package handler + +import ( + "fmt" + + "github.com/gin-gonic/gin" + "github.com/koderover/zadig/v2/pkg/types" + + commonutil "github.com/koderover/zadig/v2/pkg/microservice/aslan/core/common/util" + svcservice "github.com/koderover/zadig/v2/pkg/microservice/aslan/core/service/service" + internalhandler "github.com/koderover/zadig/v2/pkg/shared/handler" +) + +// ListManualServiceModules lists the manual module declarations for a service. +// Query: ?projectName=...&production=true|false (default false) +// Path: /services/:name/manual-modules — :name is the service name. +func ListManualServiceModules(c *gin.Context) { + ctx, err := internalhandler.NewContextWithAuthorization(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + if err != nil { + ctx.RespErr = fmt.Errorf("authorization Info Generation failed: err %s", err) + ctx.UnAuthorized = true + return + } + + projectName := c.Query("projectName") + production := c.Query("production") == "true" + serviceName := c.Param("name") + + if !canViewService(ctx, projectName, production) { + ctx.UnAuthorized = true + return + } + if production { + if err := commonutil.CheckZadigProfessionalLicense(); err != nil { + ctx.RespErr = err + return + } + } + + ctx.Resp, ctx.RespErr = svcservice.ListManualServiceModules(projectName, serviceName, production, ctx.Logger) +} + +// CreateManualServiceModule declares a new manual module for a service. +func CreateManualServiceModule(c *gin.Context) { + ctx, err := internalhandler.NewContextWithAuthorization(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + if err != nil { + ctx.RespErr = fmt.Errorf("authorization Info Generation failed: err %s", err) + ctx.UnAuthorized = true + return + } + + projectName := c.Query("projectName") + production := c.Query("production") == "true" + + if !canEditService(ctx, projectName, production) { + ctx.UnAuthorized = true + return + } + if production { + if err := commonutil.CheckZadigProfessionalLicense(); err != nil { + ctx.RespErr = err + return + } + } + + args := new(svcservice.ManualModuleCreateReq) + if err := c.ShouldBindJSON(args); err != nil { + ctx.RespErr = err + return + } + + function := "项目管理-服务" + if production { + function = "项目管理-生产服务" + } + internalhandler.InsertOperationLog(c, ctx.UserName, projectName, "新增", function+"/手动模块", args.ServiceName+"/"+args.Name, args.Name, "", types.RequestBodyTypeJSON, ctx.Logger) + + ctx.Resp, ctx.RespErr = svcservice.CreateManualServiceModule(args, projectName, production, ctx.Logger) +} + +// UpdateManualServiceModule edits an existing manual module by id. +// Path: /services/manual-modules/:id +func UpdateManualServiceModule(c *gin.Context) { + ctx, err := internalhandler.NewContextWithAuthorization(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + if err != nil { + ctx.RespErr = fmt.Errorf("authorization Info Generation failed: err %s", err) + ctx.UnAuthorized = true + return + } + + projectName := c.Query("projectName") + production := c.Query("production") == "true" + + if !canEditService(ctx, projectName, production) { + ctx.UnAuthorized = true + return + } + if production { + if err := commonutil.CheckZadigProfessionalLicense(); err != nil { + ctx.RespErr = err + return + } + } + + args := new(svcservice.ManualModuleUpdateReq) + if err := c.ShouldBindJSON(args); err != nil { + ctx.RespErr = err + return + } + + id := c.Param("id") + function := "项目管理-服务" + if production { + function = "项目管理-生产服务" + } + internalhandler.InsertOperationLog(c, ctx.UserName, projectName, "修改", function+"/手动模块", id, "", "", types.RequestBodyTypeJSON, ctx.Logger) + + ctx.RespErr = svcservice.UpdateManualServiceModule(id, production, args, ctx.Logger) +} + +// DeleteManualServiceModule removes a manual module by id. +// Path: /services/manual-modules/:id +func DeleteManualServiceModule(c *gin.Context) { + ctx, err := internalhandler.NewContextWithAuthorization(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + if err != nil { + ctx.RespErr = fmt.Errorf("authorization Info Generation failed: err %s", err) + ctx.UnAuthorized = true + return + } + + projectName := c.Query("projectName") + production := c.Query("production") == "true" + + if !canEditService(ctx, projectName, production) { + ctx.UnAuthorized = true + return + } + if production { + if err := commonutil.CheckZadigProfessionalLicense(); err != nil { + ctx.RespErr = err + return + } + } + + id := c.Param("id") + function := "项目管理-服务" + if production { + function = "项目管理-生产服务" + } + internalhandler.InsertOperationLog(c, ctx.UserName, projectName, "删除", function+"/手动模块", id, "", "", types.RequestBodyTypeJSON, ctx.Logger) + + ctx.RespErr = svcservice.DeleteManualServiceModule(id, production, ctx.Logger) +} + +// canViewService mirrors the authorization scaffolding used by the existing +// service-template handlers — system admin / project admin / service view +// permission, branching on production. +func canViewService(ctx *internalhandler.Context, projectName string, production bool) bool { + if ctx.Resources.IsSystemAdmin { + return true + } + info, ok := ctx.Resources.ProjectAuthInfo[projectName] + if !ok { + return false + } + if production { + return info.IsProjectAdmin || info.ProductionService.View + } + return info.IsProjectAdmin || info.Service.View +} + +func canEditService(ctx *internalhandler.Context, projectName string, production bool) bool { + if ctx.Resources.IsSystemAdmin { + return true + } + info, ok := ctx.Resources.ProjectAuthInfo[projectName] + if !ok { + return false + } + if production { + return info.IsProjectAdmin || info.ProductionService.Edit + } + return info.IsProjectAdmin || info.Service.Edit +} diff --git a/pkg/microservice/aslan/core/service/handler/router.go b/pkg/microservice/aslan/core/service/handler/router.go index 47995a4c93..4e8fb11f67 100644 --- a/pkg/microservice/aslan/core/service/handler/router.go +++ b/pkg/microservice/aslan/core/service/handler/router.go @@ -59,6 +59,14 @@ func (*Router) Inject(router *gin.RouterGroup) { k8s.GET("/kube/workloads", GetKubeWorkloads) k8s.POST("/yaml", LoadKubeWorkloadsYaml) + + // Manual service modules — user-declared modules for CRD/DaemonSet + // workloads that the YAML parser can't auto-discover. Listed + // per-service; created/updated/deleted by id. + k8s.GET("/:name/manual-modules", ListManualServiceModules) + k8s.POST("/manual-modules", CreateManualServiceModule) + k8s.PUT("/manual-modules/:id", UpdateManualServiceModule) + k8s.DELETE("/manual-modules/:id", DeleteManualServiceModule) } labels := router.Group("labels") diff --git a/pkg/microservice/aslan/core/service/service/helm.go b/pkg/microservice/aslan/core/service/service/helm.go index b9dc023934..50ffd8ef08 100644 --- a/pkg/microservice/aslan/core/service/service/helm.go +++ b/pkg/microservice/aslan/core/service/service/helm.go @@ -18,6 +18,7 @@ package service import ( "bytes" + "context" "encoding/json" "fmt" "io/fs" @@ -216,7 +217,12 @@ func GetHelmServiceModule(serviceName, productName string, revision int64, isPro helmServiceModule := new(HelmServiceModule) serviceModules := make([]*ServiceModule, 0) - for _, container := range serviceTemplate.Containers { + // Service.Containers no longer persisted — read from service_module table. + resolvedContainers, _, rerr := repository.ResolveServiceModules(context.Background(), serviceTemplate.ProductName, serviceTemplate.ServiceName, isProduction, serviceTemplate.Revision) + if rerr != nil { + return nil, fmt.Errorf("failed to resolve modules for %s/%s rev %d: %s", serviceTemplate.ProductName, serviceTemplate.ServiceName, serviceTemplate.Revision, rerr) + } + for _, container := range resolvedContainers { serviceModule := new(ServiceModule) serviceModule.Container = container diff --git a/pkg/microservice/aslan/core/service/service/manual_module.go b/pkg/microservice/aslan/core/service/service/manual_module.go new file mode 100644 index 0000000000..7045f8cbe6 --- /dev/null +++ b/pkg/microservice/aslan/core/service/service/manual_module.go @@ -0,0 +1,282 @@ +/* +Copyright 2026 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package service + +import ( + "context" + "fmt" + "regexp" + + "go.mongodb.org/mongo-driver/bson/primitive" + "go.uber.org/zap" + + commonmodels "github.com/koderover/zadig/v2/pkg/microservice/aslan/core/common/repository/models" + commonrepo "github.com/koderover/zadig/v2/pkg/microservice/aslan/core/common/repository/mongodb" + "github.com/koderover/zadig/v2/pkg/microservice/aslan/core/common/service/repository" + "github.com/koderover/zadig/v2/pkg/setting" +) + +// moduleImagePlaceholderRegex matches `$-image$` anywhere in a string +// and captures the name. Mirrors moduleImageRegex in kube/parse.go — kept in +// a third copy here to avoid an import cycle (service/service → kube has +// other forward deps). +// +// TODO: consolidate the three copies (kube/parse.go, util/service.go, here) +// once we have a leaf package both can import safely. +var moduleImagePlaceholderRegex = regexp.MustCompile(`\$([A-Za-z0-9_-]+)-image\$`) + +// ManualModuleCreateReq is the API payload for declaring a new manual module. +// Manual modules attach to a K8s YAML service template and are version- +// agnostic — they remain visible across every revision of the service. +type ManualModuleCreateReq struct { + ServiceName string `json:"service_name"` + Name string `json:"name"` + Image string `json:"image"` + ImageName string `json:"image_name,omitempty"` +} + +// ManualModuleUpdateReq is the payload for editing an existing manual module. +// Name is immutable — change name by deleting and recreating. +type ManualModuleUpdateReq struct { + Image string `json:"image"` + ImageName string `json:"image_name,omitempty"` +} + +// ListManualServiceModules returns every manual module declared for one +// service. Returns an empty slice (not nil) when there are none. +func ListManualServiceModules(projectName, serviceName string, production bool, log *zap.SugaredLogger) ([]*commonmodels.ServiceModule, error) { + if projectName == "" || serviceName == "" { + return nil, fmt.Errorf("projectName and serviceName are required") + } + coll := pickModuleColl(production) + records, err := coll.ListManual(context.Background(), projectName, serviceName) + if err != nil { + log.Errorf("failed to list manual modules for %s/%s: %s", projectName, serviceName, err) + return nil, fmt.Errorf("failed to list manual modules: %s", err) + } + if records == nil { + records = []*commonmodels.ServiceModule{} + } + return records, nil +} + +// CreateManualServiceModule declares a new manual module. Rejects: +// - empty name or image (service-layer guard mirroring the UI required flag) +// - service template that does not exist +// - a duplicate Name within the same (project, service) +// +// ImageName defaults to Name when empty (also enforced at the storage layer +// as belt-and-suspenders). +func CreateManualServiceModule(args *ManualModuleCreateReq, projectName string, production bool, log *zap.SugaredLogger) (*commonmodels.ServiceModule, error) { + if args == nil { + return nil, fmt.Errorf("missing request body") + } + if projectName == "" || args.ServiceName == "" { + return nil, fmt.Errorf("projectName and service_name are required") + } + if args.Name == "" { + return nil, fmt.Errorf("module name is required") + } + if args.Image == "" { + return nil, fmt.Errorf("module image is required") + } + if !isValidModuleName(args.Name) { + return nil, fmt.Errorf("module name must match K8s container naming rules ([A-Za-z0-9_-]+)") + } + + if err := ensureServiceExists(projectName, args.ServiceName, production); err != nil { + return nil, err + } + + coll := pickModuleColl(production) + existing, err := coll.ListManual(context.Background(), projectName, args.ServiceName) + if err != nil { + return nil, fmt.Errorf("failed to look up existing manual modules: %s", err) + } + for _, m := range existing { + if m.Name == args.Name { + return nil, fmt.Errorf("manual module %q already exists for service %s", args.Name, args.ServiceName) + } + } + + record := &commonmodels.ServiceModule{ + ProjectName: projectName, + ServiceName: args.ServiceName, + Name: args.Name, + Image: args.Image, + ImageName: args.ImageName, + } + if err := coll.CreateManual(context.Background(), record); err != nil { + log.Errorf("failed to create manual module %s/%s/%s: %s", projectName, args.ServiceName, args.Name, err) + return nil, fmt.Errorf("failed to create manual module: %s", err) + } + return record, nil +} + +// UpdateManualServiceModule replaces Image and ImageName on an existing manual +// record. Name is not editable here — delete + recreate to rename, so the +// $-image$ placeholder references stay consistent. +// +// Both Image and ImageName are required at the API layer; the convenience +// fallback only applies on create. +func UpdateManualServiceModule(id string, production bool, args *ManualModuleUpdateReq, log *zap.SugaredLogger) error { + if args == nil { + return fmt.Errorf("missing request body") + } + if args.Image == "" { + return fmt.Errorf("module image is required") + } + if args.ImageName == "" { + return fmt.Errorf("module image_name is required") + } + objectID, err := primitive.ObjectIDFromHex(id) + if err != nil { + return fmt.Errorf("invalid module id: %s", err) + } + if err := pickModuleColl(production).UpdateManual(context.Background(), objectID, args.Image, args.ImageName); err != nil { + log.Errorf("failed to update manual module %s: %s", id, err) + return fmt.Errorf("failed to update manual module: %s", err) + } + return nil +} + +// DeleteManualServiceModule removes a single manual module by id. The +// $-image$ placeholder in any service YAML referring to this module +// will start failing to resolve on next deploy — surfaced as a warning by +// the save-time validator. +func DeleteManualServiceModule(id string, production bool, log *zap.SugaredLogger) error { + objectID, err := primitive.ObjectIDFromHex(id) + if err != nil { + return fmt.Errorf("invalid module id: %s", err) + } + if err := pickModuleColl(production).DeleteByID(context.Background(), objectID); err != nil { + log.Errorf("failed to delete manual module %s: %s", id, err) + return fmt.Errorf("failed to delete manual module: %s", err) + } + return nil +} + +// ResolveServiceModules is a convenience wrapper around the merge function +// that handler endpoints can call when they need the unified module list +// (auto + manual) plus the conflict report. +func ResolveServiceModules(projectName, serviceName string, production bool, revision int64) ([]*commonmodels.Container, []repository.ModuleConflict, error) { + return repository.ResolveServiceModules(context.Background(), projectName, serviceName, production, revision) +} + +// ValidatePlaceholderResolution scans the rendered YAML for $-image$ +// placeholders and returns the names that don't match any known module +// (parsed containers ∪ manual modules). The caller decides what to do — +// callers in this package log + include in the response, never reject the +// save (per the design rule that git-imported templates must not be +// blocked). +func ValidatePlaceholderResolution(svc *commonmodels.Service, production bool) []string { + if svc == nil || svc.Type != setting.K8SDeployType { + return nil + } + knownNames := make(map[string]struct{}, len(svc.Containers)) + for _, c := range svc.Containers { + if c != nil && c.Name != "" { + knownNames[c.Name] = struct{}{} + } + } + manuals, err := pickModuleColl(production).ListManual(context.Background(), svc.ProductName, svc.ServiceName) + if err == nil { + for _, m := range manuals { + knownNames[m.Name] = struct{}{} + } + } + + placeholders := extractPlaceholderNames(svc.Yaml) + if len(placeholders) == 0 { + // Also scan the rendered yaml if Yaml itself doesn't contain + // placeholders (variable substitution may inject them). + placeholders = extractPlaceholderNames(svc.RenderedYaml) + } + unresolved := make([]string, 0) + seen := make(map[string]struct{}) + for _, name := range placeholders { + if _, ok := knownNames[name]; ok { + continue + } + if _, dup := seen[name]; dup { + continue + } + seen[name] = struct{}{} + unresolved = append(unresolved, name) + } + return unresolved +} + +func extractPlaceholderNames(yaml string) []string { + if yaml == "" { + return nil + } + matches := moduleImagePlaceholderRegex.FindAllStringSubmatch(yaml, -1) + out := make([]string, 0, len(matches)) + for _, m := range matches { + if len(m) >= 2 { + out = append(out, m[1]) + } + } + return out +} + +func pickModuleColl(production bool) *commonrepo.ServiceModuleColl { + if production { + return commonrepo.NewProductionServiceModuleColl() + } + return commonrepo.NewServiceModuleColl() +} + +func ensureServiceExists(projectName, serviceName string, production bool) error { + opt := &commonrepo.ServiceFindOption{ + ProductName: projectName, + ServiceName: serviceName, + ExcludeStatus: setting.ProductStatusDeleting, + } + var err error + if production { + _, err = commonrepo.NewProductionServiceColl().Find(opt) + } else { + _, err = commonrepo.NewServiceColl().Find(opt) + } + if err != nil { + return fmt.Errorf("service %q not found in project %q: %s", serviceName, projectName, err) + } + return nil +} + +// isValidModuleName accepts the same character set as parseContainer accepts +// for K8s container names (the broader [A-Za-z0-9_-]+ form, matching what +// the moduleImageRegex in kube/parse.go expects). The regex anchors the +// whole string. +func isValidModuleName(name string) bool { + if name == "" { + return false + } + for _, r := range name { + switch { + case r >= 'a' && r <= 'z': + case r >= 'A' && r <= 'Z': + case r >= '0' && r <= '9': + case r == '_' || r == '-': + default: + return false + } + } + return true +} diff --git a/pkg/microservice/aslan/core/service/service/service.go b/pkg/microservice/aslan/core/service/service/service.go index a0dd57faf3..c64d16f6f5 100644 --- a/pkg/microservice/aslan/core/service/service/service.go +++ b/pkg/microservice/aslan/core/service/service/service.go @@ -121,7 +121,7 @@ func GetServiceTemplateOption(serviceName, productName string, revision int64, p return nil, err } - serviceOption, err := GetServiceOption(service, log) + serviceOption, err := GetServiceOption(service, production, log) if serviceOption != nil { serviceOption.Service = service } @@ -129,7 +129,7 @@ func GetServiceTemplateOption(serviceName, productName string, revision int64, p return serviceOption, err } -func GetServiceOption(args *commonmodels.Service, log *zap.SugaredLogger) (*ServiceOption, error) { +func GetServiceOption(args *commonmodels.Service, production bool, log *zap.SugaredLogger) (*ServiceOption, error) { serviceOption := new(ServiceOption) serviceModules := make([]*ServiceModule, 0) @@ -150,9 +150,18 @@ func GetServiceOption(args *commonmodels.Service, log *zap.SugaredLogger) (*Serv serviceModules = append(serviceModules, serviceModule) } else { - for _, container := range args.Containers { + // Phase 4: module list comes from service_module table (merged auto + + // manual). Manual modules for CRD/DaemonSet workloads now show up in + // the build-config dropdown alongside parsed containers. + containers, _, err := repository.ResolveServiceModules(context.Background(), args.ProductName, args.ServiceName, production, args.Revision) + if err != nil { + return nil, fmt.Errorf("failed to resolve service modules for %s/%s rev %d: %s", args.ProductName, args.ServiceName, args.Revision, err) + } + for _, container := range containers { serviceModule := new(ServiceModule) serviceModule.Container = container + // ImageName was normalized at write time; the fallback here is + // belt-and-suspenders for legacy data not yet migrated. serviceModule.ImageName = util.GetImageNameFromContainerInfo(container.ImageName, container.Name) buildObjs, err := commonrepo.NewBuildColl().List(&commonrepo.BuildListOption{ProductName: args.ProductName, ServiceName: args.ServiceName, Targets: []string{container.Name}}) if err != nil { @@ -509,6 +518,9 @@ func CreateWorkloadTemplate(args *commonmodels.Service, production bool, session log.Errorf("Failed tosetCurrentContainerImages %s, err: %s", args.ProductName, err) return nil, err } + if unresolved := ValidatePlaceholderResolution(args, production); len(unresolved) > 0 { + log.Warnf("service %s/%s has unresolved $-image$ placeholder(s): %v", args.ProductName, args.ServiceName, unresolved) + } opt := &commonrepo.ServiceFindOption{ ServiceName: args.ServiceName, ProductName: args.ProductName, @@ -649,7 +661,7 @@ func CreateServiceTemplate(userName string, args *commonmodels.Service, force bo return nil, e.ErrCreateTemplate.AddErr(err) } - return GetServiceOption(args, log) + return GetServiceOption(args, production, log) } func UpdateServiceEnvStatus(args *commonservice.ServiceTmplObject) error { @@ -1064,6 +1076,14 @@ func DeleteServiceTemplate(serviceName, serviceType, productName string, product } } commonservice.DeleteServiceWebhookByName(serviceName, productName, production, log) + + // Cascade service_module records (manual + auto, all revisions). The + // service template itself is soft-deleted (status=Deleting) above, but + // from the module perspective the service is gone — a future re-create + // will repopulate auto records and the user can re-add manual ones. + if err := repository.DeleteAllServiceModulesForService(context.Background(), productName, serviceName, production); err != nil { + log.Warnf("service_module: failed to cascade-delete records for %s/%s: %s", productName, serviceName, err) + } return nil } @@ -1148,6 +1168,9 @@ func ensureServiceTmpl(userName string, args *commonmodels.Service, log *zap.Sug log.Errorf("failed to ser set container images, err: %s", err) //return err } + if unresolved := ValidatePlaceholderResolution(args, args.Production); len(unresolved) > 0 { + log.Warnf("service %s/%s has unresolved $-image$ placeholder(s): %v", args.ProductName, args.ServiceName, unresolved) + } log.Infof("find %d containers in service %s", len(args.Containers), args.ServiceName) } diff --git a/pkg/microservice/aslan/core/workflow/service/webhook/gerrit.go b/pkg/microservice/aslan/core/workflow/service/webhook/gerrit.go index 7c60eecbf3..d3dbfa033f 100644 --- a/pkg/microservice/aslan/core/workflow/service/webhook/gerrit.go +++ b/pkg/microservice/aslan/core/workflow/service/webhook/gerrit.go @@ -451,6 +451,9 @@ func ensureServiceTmpl(userName string, args *commonmodels.Service, log *zap.Sug if err := commonutil.SetCurrentContainerImages(args); err != nil { return err } + if unresolved := service.ValidatePlaceholderResolution(args, args.Production); len(unresolved) > 0 { + log.Warnf("service %s/%s has unresolved $-image$ placeholder(s): %v", args.ProductName, args.ServiceName, unresolved) + } log.Infof("find %d containers in service %s", len(args.Containers), args.ServiceName) } diff --git a/pkg/microservice/aslan/core/workflow/service/workflow/controller/job/job_build.go b/pkg/microservice/aslan/core/workflow/service/workflow/controller/job/job_build.go index 848cea549b..0b7614bc87 100644 --- a/pkg/microservice/aslan/core/workflow/service/workflow/controller/job/job_build.go +++ b/pkg/microservice/aslan/core/workflow/service/workflow/controller/job/job_build.go @@ -17,6 +17,7 @@ limitations under the License. package job import ( + "context" "fmt" "net/url" "path" @@ -205,10 +206,15 @@ func (j BuildJobController) Update(useUserInput bool, ticket *commonmodels.Appro Repos: applyRepos(buildInfo.Repos, configuredBuild.Repos), } - for _, container := range service.Containers { - if container.Name == configuredBuild.ServiceModule { - item.ImageName = container.ImageName - break + // Service.Containers no longer persisted — pull module's ImageName + // from the service_module table. + buildContainers, _, rerr := repository.ResolveServiceModules(context.Background(), service.ProductName, service.ServiceName, false, service.Revision) + if rerr == nil { + for _, container := range buildContainers { + if container.Name == configuredBuild.ServiceModule { + item.ImageName = container.ImageName + break + } } } @@ -1161,10 +1167,15 @@ func (j BuildJobController) getReferredJobTargets(jobName string, refRepos bool) if service == nil { return nil, fmt.Errorf("service %s not found", build.ServiceName) } - for _, container := range service.Containers { - if container.Name == build.ServiceModule { - target.ImageName = container.ImageName - break + // Service.Containers no longer persisted — pull module's ImageName + // from the service_module table. + targetContainers, _, rerr := repository.ResolveServiceModules(context.Background(), service.ProductName, service.ServiceName, false, service.Revision) + if rerr == nil { + for _, container := range targetContainers { + if container.Name == build.ServiceModule { + target.ImageName = container.ImageName + break + } } } diff --git a/pkg/microservice/aslan/core/workflow/service/workflow/controller/job/job_deploy.go b/pkg/microservice/aslan/core/workflow/service/workflow/controller/job/job_deploy.go index f1d3da344b..bf6d270d94 100644 --- a/pkg/microservice/aslan/core/workflow/service/workflow/controller/job/job_deploy.go +++ b/pkg/microservice/aslan/core/workflow/service/workflow/controller/job/job_deploy.go @@ -17,6 +17,7 @@ limitations under the License. package job import ( + "context" "fmt" "strings" @@ -618,6 +619,15 @@ func (j DeployJobController) ToTask(taskID int64) ([]*commonmodels.JobTask, erro VariableYaml: varsYaml, VariableKVs: varKVs, Containers: containers, + // Workflow task creation may run before the build that + // produces a module's image — `containers` then carries + // "{{ NOT BE RENDERED }}" sentinel images. For + // $-image$ placeholders targeting such modules, + // substitution is intentionally skipped (see parse.go), + // so the rendered yaml retains the literal placeholder. + // Tolerate that here; the real image is filled in at task + // execution time. + AllowUnresolvedModuleImages: true, } updatedYaml, _, _, err := kube.GenerateRenderedYaml(option) @@ -982,7 +992,9 @@ func generateDeployInfoForEnv(env, project string, production bool, configuredSe if projectInfo.IsHostProduct() { for _, service := range envServiceMap { modules := make([]*commonmodels.DeployModuleInfo, 0) + modulesMap := make(map[string]struct{}) for _, module := range service.Containers { + modulesMap[module.Name] = struct{}{} if approvalTicket.IsAllowedService(project, service.ServiceName, module.Name) { modules = append(modules, &commonmodels.DeployModuleInfo{ ServiceModule: module.Name, @@ -992,6 +1004,24 @@ func generateDeployInfoForEnv(env, project string, production bool, configuredSe } } + // Phase 4: append manual modules for host-product env case. + manualMods, mErr := repository.ListManualServiceModules(context.Background(), project, service.ServiceName, production) + if mErr != nil { + return nil, fmt.Errorf("failed to list manual service modules for %s: %v", service.ServiceName, mErr) + } + for _, m := range manualMods { + if _, exists := modulesMap[m.Name]; exists { + continue + } + if approvalTicket.IsAllowedService(project, service.ServiceName, m.Name) { + modules = append(modules, &commonmodels.DeployModuleInfo{ + ServiceModule: m.Name, + Image: m.Image, + ImageName: util.ExtractImageName(m.Image), + }) + } + } + svcBasicInfo := commonmodels.DeployBasicInfo{ ServiceName: service.ServiceName, Modules: modules, @@ -1074,6 +1104,28 @@ func generateDeployInfoForEnv(env, project string, production bool, configuredSe } } + // Phase 4: append manual modules (user-declared at the service level + // for CRD/DaemonSet workloads) and dedup against modulesMap so we + // don't double-count when a manual record shares a name with a + // container already pulled from ProductService via the env reconcile. + manualMods, mErr := repository.ListManualServiceModules(context.Background(), project, service.ServiceName, production) + if mErr != nil { + return nil, fmt.Errorf("failed to list manual service modules for %s: %v", service.ServiceName, mErr) + } + for _, m := range manualMods { + if _, exists := modulesMap[m.Name]; exists { + continue + } + modulesMap[m.Name] = m.Image + if approvalTicket.IsAllowedService(project, service.ServiceName, m.Name) { + modules = append(modules, &commonmodels.DeployModuleInfo{ + ServiceModule: m.Name, + Image: m.Image, + ImageName: util.ExtractImageName(m.Image), + }) + } + } + currentReleaseName := "" if service.Type == setting.HelmDeployType { envService, err := repositoryCache.QueryTemplateServiceWithCache(&commonrepo.ServiceFindOption{ @@ -1090,7 +1142,13 @@ func generateDeployInfoForEnv(env, project string, production bool, configuredSe latestReleaseName := "" if serviceDef, ok := serviceDefinitionMap[service.ServiceName]; ok { - for _, module := range serviceDef.Containers { + // Service.Containers no longer persisted — pull merged modules + // for the latest template revision. + defContainers, _, dErr := repository.ResolveServiceModules(context.Background(), serviceDef.ProductName, serviceDef.ServiceName, production, serviceDef.Revision) + if dErr != nil { + return nil, fmt.Errorf("failed to resolve modules for %s/%s rev %d: %s", serviceDef.ProductName, serviceDef.ServiceName, serviceDef.Revision, dErr) + } + for _, module := range defContainers { // if a container is newly created in the service, add it to the module list if _, ok := modulesMap[module.Name]; !ok { modules = append(modules, &commonmodels.DeployModuleInfo{ @@ -1157,7 +1215,9 @@ func generateDeployInfoForEnv(env, project string, production bool, configuredSe } modules := make([]*commonmodels.DeployModuleInfo, 0) + modulesMap := make(map[string]struct{}) for _, module := range service.Containers { + modulesMap[module.Name] = struct{}{} if approvalTicket.IsAllowedService(project, service.ServiceName, module.Name) { modules = append(modules, &commonmodels.DeployModuleInfo{ ServiceModule: module.Name, @@ -1167,6 +1227,24 @@ func generateDeployInfoForEnv(env, project string, production bool, configuredSe } } + // Phase 4: append manual modules for services not yet deployed in env. + manualMods, mErr := repository.ListManualServiceModules(context.Background(), project, service.ServiceName, production) + if mErr != nil { + return nil, fmt.Errorf("failed to list manual service modules for %s: %v", service.ServiceName, mErr) + } + for _, m := range manualMods { + if _, exists := modulesMap[m.Name]; exists { + continue + } + if approvalTicket.IsAllowedService(project, service.ServiceName, m.Name) { + modules = append(modules, &commonmodels.DeployModuleInfo{ + ServiceModule: m.Name, + Image: m.Image, + ImageName: util.ExtractImageName(m.Image), + }) + } + } + svcBasicInfo := commonmodels.DeployBasicInfo{ ServiceName: service.ServiceName, Modules: modules, diff --git a/pkg/microservice/aslan/core/workflow/service/workflow/controller/job/job_distribute_image.go b/pkg/microservice/aslan/core/workflow/service/workflow/controller/job/job_distribute_image.go index f3952bd6fc..e392e8f9bd 100644 --- a/pkg/microservice/aslan/core/workflow/service/workflow/controller/job/job_distribute_image.go +++ b/pkg/microservice/aslan/core/workflow/service/workflow/controller/job/job_distribute_image.go @@ -17,6 +17,7 @@ limitations under the License. package job import ( + "context" "fmt" "strings" @@ -141,7 +142,13 @@ func (j DistributeImageJobController) SetOptions(ticket *commonmodels.ApprovalTi options := make([]*commonmodels.DistributeTarget, 0) for _, svc := range servicesMap { - for _, module := range svc.Containers { + // Service.Containers no longer persisted — list modules from the + // service_module table for this revision. + modules, _, rerr := repository.ResolveServiceModules(context.Background(), svc.ProductName, svc.ServiceName, false, svc.Revision) + if rerr != nil { + return fmt.Errorf("failed to resolve modules for %s/%s rev %d: %s", svc.ProductName, svc.ServiceName, svc.Revision, rerr) + } + for _, module := range modules { if ticket.IsAllowedService(j.workflow.Project, svc.ServiceName, module.Name) { options = append(options, &commonmodels.DistributeTarget{ ServiceName: svc.ServiceName, diff --git a/pkg/microservice/aslan/core/workflow/service/workflow/workflow_task_v4.go b/pkg/microservice/aslan/core/workflow/service/workflow/workflow_task_v4.go index 19b3ae813d..d4821ee470 100644 --- a/pkg/microservice/aslan/core/workflow/service/workflow/workflow_task_v4.go +++ b/pkg/microservice/aslan/core/workflow/service/workflow/workflow_task_v4.go @@ -43,6 +43,7 @@ import ( templaterepo "github.com/koderover/zadig/v2/pkg/microservice/aslan/core/common/repository/mongodb/template" "github.com/koderover/zadig/v2/pkg/microservice/aslan/core/common/service" commonservice "github.com/koderover/zadig/v2/pkg/microservice/aslan/core/common/service" + "github.com/koderover/zadig/v2/pkg/microservice/aslan/core/common/service/repository" "github.com/koderover/zadig/v2/pkg/microservice/aslan/core/common/service/dingtalk" "github.com/koderover/zadig/v2/pkg/microservice/aslan/core/common/service/instantmessage" "github.com/koderover/zadig/v2/pkg/microservice/aslan/core/common/service/lark" @@ -3248,7 +3249,13 @@ func ListWorkflowFilterInfo(project, workflow, typeName string, jobName string, services := make([]string, 0) serviceList, _ := commonrepo.NewServiceColl().ListMaxRevisions(&commonrepo.ServiceListOption{ProductName: project}) for _, service := range serviceList { - for _, container := range service.Containers { + // Service.Containers no longer persisted — fetch modules from the + // service_module table. Non-production filter list. + resolved, _, rerr := repository.ResolveServiceModules(context.Background(), service.ProductName, service.ServiceName, false, service.Revision) + if rerr != nil { + continue + } + for _, container := range resolved { if !utils.Contains(services, container.Name) { services = append(services, container.Name) }