Skip to content

🐛 OCPBUGS-123707: prevent ClusterCatalog catalog rollbacks #2965

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
tmshort wants to merge 1 commit into
base: main
Choose a base branch
from
Open

🐛 OCPBUGS-123707: prevent ClusterCatalog catalog rollbacks #2965

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 6 additions & 4 deletions cmd/catalogd/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -332,6 +332,7 @@ func run(ctx context.Context) error {

imageCache := imageutil.CatalogCache(unpackCacheBasePath)
imagePuller := &imageutil.ContainersImagePuller{
CatalogRollbackProtectionEnabled: true,
SourceCtxFunc: func(ctx context.Context) (*types.SystemContext, error) {
logger := log.FromContext(ctx)
srcContext := &types.SystemContext{
Expand Down Expand Up @@ -403,10 +404,11 @@ func run(ctx context.Context) error {
}

if err = (&corecontrollers.ClusterCatalogReconciler{
Client: mgr.GetClient(),
ImageCache: imageCache,
ImagePuller: imagePuller,
Storage: localStorage,
Client: mgr.GetClient(),
ImageCache: imageCache,
ImagePuller: imagePuller,
EventRecorder: mgr.GetEventRecorder("clustercatalog-controller"),
Storage: localStorage,
}).SetupWithManager(mgr); err != nil {
setupLog.Error(err, "unable to create controller", "controller", "ClusterCatalog")
return err
Expand Down
32 changes: 26 additions & 6 deletions internal/catalogd/controllers/core/clustercatalog_controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,11 +25,13 @@ import (
"time"

"go.podman.io/image/v5/docker/reference"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/equality"
"k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/client-go/tools/events"
"k8s.io/utils/ptr"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
Expand All @@ -56,8 +58,9 @@ const (
type ClusterCatalogReconciler struct {
client.Client

ImageCache imageutil.Cache
ImagePuller imageutil.Puller
ImageCache imageutil.Cache
ImagePuller imageutil.Puller
EventRecorder events.EventRecorder

Storage storage.Instance

Expand Down Expand Up @@ -255,6 +258,18 @@ func (r *ClusterCatalogReconciler) reconcile(ctx context.Context, catalog *ocv1.

fsys, canonicalRef, unpackTime, err := r.ImagePuller.Pull(ctx, catalog.Name, catalog.Spec.Source.Image.Ref, r.ImageCache)
if err != nil {
var rollbackErr *imageutil.CatalogRollbackError
if errors.As(err, &rollbackErr) {
if r.EventRecorder != nil {
r.EventRecorder.Eventf(catalog, nil, corev1.EventTypeWarning, "CatalogRollbackPrevented", "CatalogRollbackPrevented", "%s", rollbackErr)
}
lastSuccessfulPoll := time.Now()
r.storedCatalogsMu.Lock()
storedCatalog.lastSuccessfulPoll = lastSuccessfulPoll
r.storedCatalogs[catalog.Name] = storedCatalog
r.storedCatalogsMu.Unlock()
return nextPollResult(lastSuccessfulPoll, catalog), nil
}
unpackErr := fmt.Errorf("source catalog content: %w", err)
updateStatusProgressing(&catalog.Status, catalog.GetGeneration(), unpackErr)
return ctrl.Result{}, unpackErr
Expand Down Expand Up @@ -350,10 +365,8 @@ func updateStatusProgressing(status *ocv1.ClusterCatalogStatus, generation int64

func updateStatusServing(status *ocv1.ClusterCatalogStatus, ref reference.Canonical, modTime time.Time, baseURL string, generation int64) {
status.ResolvedSource = &ocv1.ResolvedCatalogSource{
Type: ocv1.SourceTypeImage,
Image: &ocv1.ResolvedImageSource{
Ref: ref.String(),
},
Type: ocv1.SourceTypeImage,
Image: &ocv1.ResolvedImageSource{Ref: ref.String()},
}
status.URLs = &ocv1.ClusterCatalogURLs{
Base: baseURL,
Expand Down Expand Up @@ -398,6 +411,13 @@ func updateStatusProgressingUserSpecifiedUnavailable(status *ocv1.ClusterCatalog

func updateStatusNotServing(status *ocv1.ClusterCatalogStatus, generation int64) {
status.ResolvedSource = nil
updateStatusNotServingWithResolvedSource(status, generation)
}

// updateStatusNotServingWithResolvedSource marks content unavailable while retaining the last accepted
// source. In particular, this preserves a catalog publication version so a rejected rollback cannot
// become acceptable on the next reconcile merely because served content is missing.
func updateStatusNotServingWithResolvedSource(status *ocv1.ClusterCatalogStatus, generation int64) {
status.URLs = nil
status.LastUnpacked = nil
meta.SetStatusCondition(&status.Conditions, metav1.Condition{
Expand Down
48 changes: 48 additions & 0 deletions internal/catalogd/controllers/core/clustercatalog_controller_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ import (
"go.uber.org/mock/gomock"
"k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/utils/ptr"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/reconcile"
Expand All @@ -40,6 +41,14 @@ func newMockStore(ctrl *gomock.Controller, shouldError bool) *mockstorage.MockIn
return m
}

type fakeEventRecorder struct {
events chan string
}

func (f *fakeEventRecorder) Eventf(_ runtime.Object, _ runtime.Object, _ string, reason, action, note string, args ...interface{}) {
f.events <- fmt.Sprintf("%s:%s:%s", reason, action, fmt.Sprintf(note, args...))
}

func TestCatalogdControllerReconcile(t *testing.T) {
mockCtrl := gomock.NewController(t)
for _, tt := range []struct {
Expand Down Expand Up @@ -1140,3 +1149,42 @@ func mustRef(t *testing.T, ref string) reference.Canonical {
}
return p.(reference.Canonical)
}

func TestReconcileCatalogRollbackKeepsServingCatalog(t *testing.T) {
mockCtrl := gomock.NewController(t)
acceptedRef := mustRef(t, "my.org/catalog@sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855")
catalog := &ocv1.ClusterCatalog{
ObjectMeta: metav1.ObjectMeta{Name: "catalog", Finalizers: []string{fbcDeletionFinalizer}},
Spec: ocv1.ClusterCatalogSpec{
Source: ocv1.CatalogSource{Type: ocv1.SourceTypeImage, Image: &ocv1.ImageSource{Ref: "my.org/catalog:latest", PollIntervalMinutes: ptr.To(5)}},
AvailabilityMode: ocv1.AvailabilityModeAvailable,
},
Status: ocv1.ClusterCatalogStatus{
Conditions: []metav1.Condition{
{Type: ocv1.TypeServing, Status: metav1.ConditionTrue, Reason: ocv1.ReasonAvailable},
{Type: ocv1.TypeProgressing, Status: metav1.ConditionTrue, Reason: ocv1.ReasonSucceeded},
},
ResolvedSource: &ocv1.ResolvedCatalogSource{Type: ocv1.SourceTypeImage, Image: &ocv1.ResolvedImageSource{Ref: acceptedRef.String()}},
URLs: &ocv1.ClusterCatalogURLs{Base: "URL"},
LastUnpacked: ptr.To(metav1.Now()),
},
}
originalStatus := catalog.Status.DeepCopy()
events := &fakeEventRecorder{events: make(chan string, 1)}
reconciler := &ClusterCatalogReconciler{
ImagePuller: &imageutil.FakePuller{Error: &imageutil.CatalogRollbackError{
CandidateDigest: "sha256:candidate", CandidateCreated: time.Date(2026, time.January, 1, 0, 0, 0, 0, time.UTC),
AcceptedDigest: "sha256:accepted", AcceptedCreated: time.Date(2026, time.January, 2, 0, 0, 0, 0, time.UTC),
}},
ImageCache: &imageutil.FakeCache{},
Storage: newMockStore(mockCtrl, false),
EventRecorder: events,
storedCatalogs: map[string]storedCatalogData{catalog.Name: {ref: acceptedRef, lastUnpack: catalog.Status.LastUnpacked.Time, lastSuccessfulPoll: time.Now().Add(-6 * time.Minute)}},
}
require.NoError(t, reconciler.setupFinalizers())
result, err := reconciler.reconcile(context.Background(), catalog)
require.NoError(t, err)
assert.Positive(t, result.RequeueAfter)
assert.Equal(t, *originalStatus, catalog.Status)
assert.Contains(t, <-events.events, "CatalogRollbackPrevented")
}
Loading
Loading