From 6a18ee911ece95b2a9128da17e4e4f5eb0069d82 Mon Sep 17 00:00:00 2001 From: Victor Moura Date: Fri, 21 Aug 2020 15:13:47 -0300 Subject: [PATCH] Adds scopeUID config to enable multiple instances of Watchtower (#511) * Adds scopeUID config to enable multiple instances of Watchtower * Adds tests for multiple instance support with scopeuid * Adds docs on scope monitoring and multiple instance support * Adds multiple instances docs to mkdocs config file * Changes multiple instances check and refactors naming for scope feature * Applies linter suggestions * Fixes documentation on Watchtower monitoring scope --- cmd/root.go | 30 ++++++++++++---------- docs/arguments.md | 10 ++++++++ docs/container-selection.md | 6 +++++ docs/running-multiple-instances.md | 27 +++++++++++++++++++ internal/actions/actions_suite_test.go | 10 ++++---- internal/actions/check.go | 7 ++--- internal/flags/flags.go | 6 +++++ mkdocs.yml | 1 + pkg/container/container.go | 11 ++++++++ pkg/container/metadata.go | 1 + pkg/container/mocks/FilterableContainer.go | 24 +++++++++++++++++ pkg/filters/filters.go | 23 ++++++++++++++++- pkg/filters/filters_test.go | 27 +++++++++++++++++-- pkg/types/filterable_container.go | 1 + 14 files changed, 160 insertions(+), 24 deletions(-) create mode 100644 docs/running-multiple-instances.md diff --git a/cmd/root.go b/cmd/root.go index 6546933..a7985e9 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -30,6 +30,7 @@ var ( notifier *notifications.Notifier timeout time.Duration lifecycleHooks bool + scope string ) var rootCmd = &cobra.Command{ @@ -90,6 +91,9 @@ func PreRun(cmd *cobra.Command, args []string) { enableLabel, _ = f.GetBool("label-enable") lifecycleHooks, _ = f.GetBool("enable-lifecycle-hooks") + scope, _ = f.GetString("scope") + + log.Debug(scope) // configure environment vars for client err := flags.EnvConfig(cmd) @@ -118,21 +122,10 @@ func PreRun(cmd *cobra.Command, args []string) { // Run is the main execution flow of the command func Run(c *cobra.Command, names []string) { - filter := filters.BuildFilter(names, enableLabel) + filter := filters.BuildFilter(names, enableLabel, scope) runOnce, _ := c.PersistentFlags().GetBool("run-once") httpAPI, _ := c.PersistentFlags().GetBool("http-api") - if httpAPI { - apiToken, _ := c.PersistentFlags().GetString("http-api-token") - - if err := api.SetupHTTPUpdates(apiToken, func() { runUpdatesWithNotifications(filter) }); err != nil { - log.Fatal(err) - os.Exit(1) - } - - api.WaitForHTTPUpdates() - } - if runOnce { if noStartupMessage, _ := c.PersistentFlags().GetBool("no-startup-message"); !noStartupMessage { log.Info("Running a one time update.") @@ -143,10 +136,21 @@ func Run(c *cobra.Command, names []string) { return } - if err := actions.CheckForMultipleWatchtowerInstances(client, cleanup); err != nil { + if err := actions.CheckForMultipleWatchtowerInstances(client, cleanup, scope); err != nil { log.Fatal(err) } + if httpAPI { + apiToken, _ := c.PersistentFlags().GetString("http-api-token") + + if err := api.SetupHTTPUpdates(apiToken, func() { runUpdatesWithNotifications(filter) }); err != nil { + log.Fatal(err) + os.Exit(1) + } + + api.WaitForHTTPUpdates() + } + if err := runUpgradesOnSchedule(c, filter); err != nil { log.Error(err) } diff --git a/docs/arguments.md b/docs/arguments.md index b22aa79..0f8f8cd 100644 --- a/docs/arguments.md +++ b/docs/arguments.md @@ -228,6 +228,16 @@ Environment Variable: WATCHTOWER_HTTP_API_TOKEN Default: - ``` +## Filter by scope +Update containers that have a `com.centurylinklabs.watchtower.scope` label set with the same value as the given argument. This enables [running multiple instances](https://containrrr.github.io/watchtower/running-multiple-instances). + +``` + Argument: --scope +Environment Variable: WATCHTOWER_SCOPE + Type: String + Default: - +``` + ## Scheduling [Cron expression](https://pkg.go.dev/github.com/robfig/cron@v1.2.0?tab=doc#hdr-CRON_Expression_Format) in 6 fields (rather than the traditional 5) which defines when and how often to check for new images. Either `--interval` or the schedule expression can be defined, but not both. An example: `--schedule "0 0 4 * * *"` diff --git a/docs/container-selection.md b/docs/container-selection.md index eea0c03..0aeb897 100644 --- a/docs/container-selection.md +++ b/docs/container-selection.md @@ -23,3 +23,9 @@ Or, it can be specified as part of the `docker run` command line: ```bash docker run -d --label=com.centurylinklabs.watchtower.enable=true someimage ``` + +If you wish to create a monitoring scope, you will need to [run multiple instances and set a scope for each of them](https://containrrr.github.io/watchtower/running-multiple-instances). + +Watchtower filters running containers by testing them against each configured criteria. A container is monitored if all criteria are met. For example: +- If a container's name is on the monitoring name list (not empty `--name` argument) but it is not enabled (_centurylinklabs.watchtower.enable=false_), it won't be monitored; +- If a container's name is not on the monitoring name list (not empty `--name` argument), even if it is enabled (_centurylinklabs.watchtower.enable=true_ and `--label-enable` flag is set), it won't be monitored; \ No newline at end of file diff --git a/docs/running-multiple-instances.md b/docs/running-multiple-instances.md new file mode 100644 index 0000000..82cd955 --- /dev/null +++ b/docs/running-multiple-instances.md @@ -0,0 +1,27 @@ +By default, Watchtower will clean up other instances and won't allow multiple instances running on the same Docker host or swarm. It is possible to override this behavior by defining a [scope](https://containrrr.github.io/watchtower/arguments/#filter_by_scope) to each running instance. + +Notice that: +- Multiple instances can't run with the same scope; +- An instance without a scope will clean up other running instances, even if they have a defined scope; + +To define an instance monitoring scope, use the `--scope` argument or the `WATCHTOWER_SCOPE` environment variable on startup and set the _com.centurylinklabs.watchtower.scope_ label with the same value for the containers you want to include in this instance's scope (including the instance itself). + +For example, in a Docker Compose config file: + +```json +version: '3' + +services: + app-monitored-by-watchtower: + image: myapps/monitored-by-watchtower + labels: + - "com.centurylinklabs.watchtower.scope=myscope" + + watchtower: + image: containrrr/watchtower + volumes: + - /var/run/docker.sock:/var/run/docker.sock + command: --interval 30 --scope myscope + labels: + - "com.centurylinklabs.watchtower.scope=myscope" +``` \ No newline at end of file diff --git a/internal/actions/actions_suite_test.go b/internal/actions/actions_suite_test.go index 3a51bfa..7cbd71b 100644 --- a/internal/actions/actions_suite_test.go +++ b/internal/actions/actions_suite_test.go @@ -46,7 +46,7 @@ var _ = Describe("the actions package", func() { When("given an empty array", func() { It("should not do anything", func() { client.TestData.Containers = []container.Container{} - err := actions.CheckForMultipleWatchtowerInstances(client, false) + err := actions.CheckForMultipleWatchtowerInstances(client, false, "") Expect(err).NotTo(HaveOccurred()) }) }) @@ -59,7 +59,7 @@ var _ = Describe("the actions package", func() { "watchtower", time.Now()), } - err := actions.CheckForMultipleWatchtowerInstances(client, false) + err := actions.CheckForMultipleWatchtowerInstances(client, false, "") Expect(err).NotTo(HaveOccurred()) }) }) @@ -90,7 +90,7 @@ var _ = Describe("the actions package", func() { }) It("should stop all but the latest one", func() { - err := actions.CheckForMultipleWatchtowerInstances(client, false) + err := actions.CheckForMultipleWatchtowerInstances(client, false, "") Expect(err).NotTo(HaveOccurred()) }) }) @@ -120,12 +120,12 @@ var _ = Describe("the actions package", func() { ) }) It("should try to delete the image if the cleanup flag is true", func() { - err := actions.CheckForMultipleWatchtowerInstances(client, true) + err := actions.CheckForMultipleWatchtowerInstances(client, true, "") Expect(err).NotTo(HaveOccurred()) Expect(client.TestData.TriedToRemoveImage()).To(BeTrue()) }) It("should not try to delete the image if the cleanup flag is false", func() { - err := actions.CheckForMultipleWatchtowerInstances(client, false) + err := actions.CheckForMultipleWatchtowerInstances(client, false, "") Expect(err).NotTo(HaveOccurred()) Expect(client.TestData.TriedToRemoveImage()).To(BeFalse()) }) diff --git a/internal/actions/check.go b/internal/actions/check.go index c6d5c12..56a9fc4 100644 --- a/internal/actions/check.go +++ b/internal/actions/check.go @@ -19,10 +19,11 @@ import ( // CheckForMultipleWatchtowerInstances will ensure that there are not multiple instances of the // watchtower running simultaneously. If multiple watchtower containers are detected, this function -// will stop and remove all but the most recently started container. -func CheckForMultipleWatchtowerInstances(client container.Client, cleanup bool) error { +// will stop and remove all but the most recently started container. This behaviour can be bypassed +// if a scope UID is defined. +func CheckForMultipleWatchtowerInstances(client container.Client, cleanup bool, scope string) error { awaitDockerClient() - containers, err := client.ListContainers(filters.WatchtowerContainersFilter) + containers, err := client.ListContainers(filters.FilterByScope(scope, filters.WatchtowerContainersFilter)) if err != nil { log.Fatal(err) diff --git a/internal/flags/flags.go b/internal/flags/flags.go index 8d8d2ab..299ab38 100644 --- a/internal/flags/flags.go +++ b/internal/flags/flags.go @@ -134,6 +134,12 @@ func RegisterSystemFlags(rootCmd *cobra.Command) { "", viper.GetString("WATCHTOWER_HTTP_API_TOKEN"), "Sets an authentication token to HTTP API requests.") + + flags.StringP( + "scope", + "", + viper.GetString("WATCHTOWER_SCOPE"), + "Defines a monitoring scope for the Watchtower instance.") } // RegisterNotificationFlags that are used by watchtower to send notifications diff --git a/mkdocs.yml b/mkdocs.yml index 645c1cc..696f87d 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -20,5 +20,6 @@ nav: - 'Secure connections': 'secure-connections.md' - 'Stop signals': 'stop-signals.md' - 'Lifecycle hooks': 'lifecycle-hooks.md' + - 'Running multiple instances': 'running-multiple-instances.md' plugins: - search diff --git a/pkg/container/container.go b/pkg/container/container.go index 50d3bb4..dc105d7 100644 --- a/pkg/container/container.go +++ b/pkg/container/container.go @@ -90,6 +90,17 @@ func (c Container) Enabled() (bool, bool) { return parsedBool, true } +// Scope returns the value of the scope UID label and if the label +// was set. +func (c Container) Scope() (string, bool) { + rawString, ok := c.getLabelValue(scope) + if !ok { + return "", false + } + + return rawString, true +} + // Links returns a list containing the names of all the containers to which // this container is linked. func (c Container) Links() []string { diff --git a/pkg/container/metadata.go b/pkg/container/metadata.go index 2c1b933..f86317c 100644 --- a/pkg/container/metadata.go +++ b/pkg/container/metadata.go @@ -6,6 +6,7 @@ const ( enableLabel = "com.centurylinklabs.watchtower.enable" dependsOnLabel = "com.centurylinklabs.watchtower.depends-on" zodiacLabel = "com.centurylinklabs.zodiac.original-image" + scope = "com.centurylinklabs.watchtower.scope" preCheckLabel = "com.centurylinklabs.watchtower.lifecycle.pre-check" postCheckLabel = "com.centurylinklabs.watchtower.lifecycle.post-check" preUpdateLabel = "com.centurylinklabs.watchtower.lifecycle.pre-update" diff --git a/pkg/container/mocks/FilterableContainer.go b/pkg/container/mocks/FilterableContainer.go index 508bd7c..107ca1b 100644 --- a/pkg/container/mocks/FilterableContainer.go +++ b/pkg/container/mocks/FilterableContainer.go @@ -55,3 +55,27 @@ func (_m *FilterableContainer) Name() string { return r0 } + +// Scope provides a mock function with given fields: +func (_m *FilterableContainer) Scope() (string, bool) { + ret := _m.Called() + + var r0 string + + if rf, ok := ret.Get(0).(func() string); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(string) + } + + var r1 bool + + if rf, ok := ret.Get(1).(func() bool); ok { + r1 = rf() + } else { + r1 = ret.Get(1).(bool) + } + + return r0, r1 +} + diff --git a/pkg/filters/filters.go b/pkg/filters/filters.go index b923745..bc6faa2 100644 --- a/pkg/filters/filters.go +++ b/pkg/filters/filters.go @@ -51,8 +51,24 @@ func FilterByDisabledLabel(baseFilter t.Filter) t.Filter { } } +// FilterByScope returns all containers that belongs to a specific scope +func FilterByScope(scope string, baseFilter t.Filter) t.Filter { + if scope == "" { + return baseFilter + } + + return func(c t.FilterableContainer) bool { + containerScope, ok := c.Scope() + if ok && containerScope == scope { + return baseFilter(c) + } + + return false + } +} + // BuildFilter creates the needed filter of containers -func BuildFilter(names []string, enableLabel bool) t.Filter { +func BuildFilter(names []string, enableLabel bool, scope string) t.Filter { filter := NoFilter filter = FilterByNames(names, filter) if enableLabel { @@ -60,6 +76,11 @@ func BuildFilter(names []string, enableLabel bool) t.Filter { // if the label is specifically set. filter = FilterByEnableLabel(filter) } + if scope != "" { + // If a scope has been defined, containers should only be considered + // if the scope is specifically set. + filter = FilterByScope(scope, filter) + } filter = FilterByDisabledLabel(filter) return filter } diff --git a/pkg/filters/filters_test.go b/pkg/filters/filters_test.go index d24b186..5766b64 100644 --- a/pkg/filters/filters_test.go +++ b/pkg/filters/filters_test.go @@ -67,6 +67,29 @@ func TestFilterByEnableLabel(t *testing.T) { container.AssertExpectations(t) } +func TestFilterByScope(t *testing.T) { + var scope string + scope = "testscope" + + filter := FilterByScope(scope, NoFilter) + assert.NotNil(t, filter) + + container := new(mocks.FilterableContainer) + container.On("Scope").Return("testscope", true) + assert.True(t, filter(container)) + container.AssertExpectations(t) + + container = new(mocks.FilterableContainer) + container.On("Scope").Return("nottestscope", true) + assert.False(t, filter(container)) + container.AssertExpectations(t) + + container = new(mocks.FilterableContainer) + container.On("Scope").Return("", false) + assert.False(t, filter(container)) + container.AssertExpectations(t) +} + func TestFilterByDisabledLabel(t *testing.T) { filter := FilterByDisabledLabel(NoFilter) assert.NotNil(t, filter) @@ -91,7 +114,7 @@ func TestBuildFilter(t *testing.T) { var names []string names = append(names, "test") - filter := BuildFilter(names, false) + filter := BuildFilter(names, false, "") container := new(mocks.FilterableContainer) container.On("Name").Return("Invalid") @@ -127,7 +150,7 @@ func TestBuildFilterEnableLabel(t *testing.T) { var names []string names = append(names, "test") - filter := BuildFilter(names, true) + filter := BuildFilter(names, true, "") container := new(mocks.FilterableContainer) container.On("Enabled").Return(false, false) diff --git a/pkg/types/filterable_container.go b/pkg/types/filterable_container.go index d89b910..3c46295 100644 --- a/pkg/types/filterable_container.go +++ b/pkg/types/filterable_container.go @@ -6,4 +6,5 @@ type FilterableContainer interface { Name() string IsWatchtower() bool Enabled() (bool, bool) + Scope() (string, bool) }