From 402eb87681fbb379f3593416396f081221ffd464 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 25 May 2023 19:14:58 +0000 Subject: [PATCH 1/7] deps(go): bump github.com/Azure/go-autorest/autorest (#2074) Bumps [github.com/Azure/go-autorest/autorest](https://github.com/Azure/go-autorest) from 0.11.28 to 0.11.29. - [Release notes](https://github.com/Azure/go-autorest/releases) - [Changelog](https://github.com/Azure/go-autorest/blob/main/CHANGELOG.md) - [Commits](https://github.com/Azure/go-autorest/compare/autorest/v0.11.28...autorest/v0.11.29) --- updated-dependencies: - dependency-name: github.com/Azure/go-autorest/autorest dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 96e5809b3e2..a3797e5c204 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ replace ( // Direct and indirect dependencies are grouped together require ( github.com/Azure/azure-sdk-for-go v68.0.0+incompatible - github.com/Azure/go-autorest/autorest v0.11.28 + github.com/Azure/go-autorest/autorest v0.11.29 github.com/Azure/go-autorest/autorest/adal v0.9.23 github.com/Azure/go-autorest/autorest/azure/auth v0.5.12 github.com/Azure/go-autorest/autorest/to v0.4.0 diff --git a/go.sum b/go.sum index 7589fc60c6e..070c55c2967 100644 --- a/go.sum +++ b/go.sum @@ -41,11 +41,12 @@ github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSW github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI= github.com/Azure/go-autorest/autorest v0.9.6/go.mod h1:/FALq9T/kS7b5J5qsQ+RSTUdAmGFqi0vUdVNNx8q630= github.com/Azure/go-autorest/autorest v0.11.24/go.mod h1:G6kyRlFnTuSbEYkQGawPfsCswgme4iYf6rfSKUDzbCc= -github.com/Azure/go-autorest/autorest v0.11.28 h1:ndAExarwr5Y+GaHE6VCaY1kyS/HwwGGyuimVhWsHOEM= -github.com/Azure/go-autorest/autorest v0.11.28/go.mod h1:MrkzG3Y3AH668QyF9KRk5neJnGgmhQ6krbhR8Q5eMvA= +github.com/Azure/go-autorest/autorest v0.11.29 h1:I4+HL/JDvErx2LjyzaVxllw2lRDB5/BT2Bm4g20iqYw= +github.com/Azure/go-autorest/autorest v0.11.29/go.mod h1:ZtEzC4Jy2JDrZLxvWs8LrBWEBycl1hbT1eknI8MtfAs= github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0= github.com/Azure/go-autorest/autorest/adal v0.8.2/go.mod h1:ZjhuQClTqx435SRJ2iMlOxPYt3d2C/T/7TiQCVZSn3Q= github.com/Azure/go-autorest/autorest/adal v0.9.18/go.mod h1:XVVeme+LZwABT8K5Lc3hA4nAe8LDBVle26gTrguhhPQ= +github.com/Azure/go-autorest/autorest/adal v0.9.22/go.mod h1:XuAbAEUv2Tta//+voMI038TrJBqjKam0me7qR+L8Cmk= github.com/Azure/go-autorest/autorest/adal v0.9.23 h1:Yepx8CvFxwNKpH6ja7RZ+sKX+DWYNldbLiALMC3BTz8= github.com/Azure/go-autorest/autorest/adal v0.9.23/go.mod h1:5pcMqFkdPhviJdlEy3kC/v1ZLnQl0MH6XA5YCcMhy4c= github.com/Azure/go-autorest/autorest/azure/auth v0.5.12 h1:wkAZRgT/pn8HhFyzfe9UnqOjJYqlembgCTi72Bm/xKk= From 0552645e04caea6f67af748157a6fb14ec7ae99b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 25 May 2023 20:43:04 +0000 Subject: [PATCH 2/7] deps(go): bump github.com/prometheus/client_golang from 1.15.0 to 1.15.1 (#2057) Bumps [github.com/prometheus/client_golang](https://github.com/prometheus/client_golang) from 1.15.0 to 1.15.1. - [Release notes](https://github.com/prometheus/client_golang/releases) - [Changelog](https://github.com/prometheus/client_golang/blob/main/CHANGELOG.md) - [Commits](https://github.com/prometheus/client_golang/compare/v1.15.0...v1.15.1) --- updated-dependencies: - dependency-name: github.com/prometheus/client_golang dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index a3797e5c204..02ee5c27f1c 100644 --- a/go.mod +++ b/go.mod @@ -41,7 +41,7 @@ require ( github.com/openshift/api v0.0.0-20230406152840-ce21e3fe5da2 github.com/openshift/client-go v0.0.0-20230324103026-3f1513df25e0 github.com/pkg/errors v0.9.1 - github.com/prometheus/client_golang v1.15.0 + github.com/prometheus/client_golang v1.15.1 github.com/sirupsen/logrus v1.9.0 github.com/spf13/cobra v1.7.0 github.com/vmware/govmomi v0.30.4 diff --git a/go.sum b/go.sum index 070c55c2967..6a1fc33099a 100644 --- a/go.sum +++ b/go.sum @@ -441,8 +441,8 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_golang v1.15.0 h1:5fCgGYogn0hFdhyhLbw7hEsWxufKtY9klyvdNfFlFhM= -github.com/prometheus/client_golang v1.15.0/go.mod h1:e9yaBhRPU2pPNsZwE+JdQl0KEt1N9XgF6zxWmaC0xOk= +github.com/prometheus/client_golang v1.15.1 h1:8tXpTmJbyH5lydzFPoxSIJ0J46jdh3tylbvM1xCv0LI= +github.com/prometheus/client_golang v1.15.1/go.mod h1:e9yaBhRPU2pPNsZwE+JdQl0KEt1N9XgF6zxWmaC0xOk= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4= github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= From f88b9faf27f2bb0fc333da88828dd1c298adffac Mon Sep 17 00:00:00 2001 From: kale-amruta <41624751+kale-amruta@users.noreply.github.com> Date: Fri, 26 May 2023 04:38:04 +0530 Subject: [PATCH 3/7] Make repository cache settings configurable using repository server CR (#1918) * Remove RepositoryServerResource from CreateCustomResources list since it's used by Kanister controller * Add a tag for repositoryserver.yaml from customresource files in pkg/customresource/embed.go * Add new webinar link to README (#1643) Signed-off-by: Ivan Sim Signed-off-by: Ivan Sim Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> * Remove noStatus local tag from RepositoryServer in types.go to generate UpdateStatus code for it * Add Status subresource to RepositoryServer CRD * Add the 'Ref' suffix to SecretReference CRD fields, update NetworkPolicy labelselector field and replace some omitempty CRD fields to required * Remove kubebuilder specific annotations from CRD YAML * Add Handler and few utility functions to the RepositoryServer controller * Update CRD type and deepcopy function to fix the pod and namespace LabelSelector error * Update controller to use podSelector and namespaceSelector from CR to define ingress traffic for repo server pod * Added Kubebuilder skeleton code * Revert go.sum * Fix vet and fmt target in Kubebuilder.mk * Merge Kubebuilder.mk and Makefile * Remove TODO in Makefile to update the 'test' target name * Rename Kubebuilder build, test, run, deploy and undeploy targets in Makefile * Add isReady field to the CR * Add extra line at the EOF in the YAML * Remove the clientset, listers and informers for RepositoryServer CR * Update RepositoryServer CRD with 'isReady' status field * Regenerate deepcopy methods for the CRs using 'make generate' target * Correct the 'isReady' field in CRD * Remove the previous controller using typed client * Move sample RepositoryServer CR YAML to a different file * Add the new controller implemented using kubebuilder framework * Update the kanister tools image in pod options * Populate pod object using podOptions in a function separate from CreatePod * Update the controller with owned resources and reconciliation for update CR event * Fix the call to GetPodObjectFromPodOptions function in controller handler * Add 'ServerPending' state of RepositoryServer CR in types.go * Set the 'IsReady' field in RepositoryServer CR * Change 'isReady' field to 'Progress' in RepositoryServer CR status * Update the change from 'IsReady' field to 'Progress' field in controller implementation * Add actual status update call to the CR in cluster after updating the CR /status 'Progress' field * Rename podOverrideSpecForCACertificate utility method * Fix infinite error loopback on Update Status * Set the credentials data from secret in the pod as env vars * remove network policy lifecyccle from repository controller * remove network policy lifecycle from the controller * udpate mysql-sidecar dockerfile * address comments * remove prometheus config * repository server start,repository create, add users * remove network policy configuration from CRD * move the server access username to CR and get the hostname and password from secret * move the server access username to CR and get the hostname and password from secret * add server access username changes to repository server CR * Added the manifests for CRD and example CR along with a basic CR controller to log outputs * Run the triage workflow on primary repo only (#1642) Signed-off-by: Ivan Sim * Bump github.com/vmware/govmomi from 0.28.0 to 0.29.0 (#1599) Bumps [github.com/vmware/govmomi](https://github.com/vmware/govmomi) from 0.28.0 to 0.29.0. - [Release notes](https://github.com/vmware/govmomi/releases) - [Changelog](https://github.com/vmware/govmomi/blob/master/CHANGELOG.md) - [Commits](https://github.com/vmware/govmomi/compare/v0.28.0...v0.29.0) --- updated-dependencies: - dependency-name: github.com/vmware/govmomi dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Update the command paths in pkg/kopia/repositoryserver/README.md * Update the README.md * Update the command paths in pkg/kopia/repositoryserver/README.md * Remove go.mod and go.sum files * Remove pkg/kopia/repositoryserver/bin * Removed RepositoryServer CRD manifest and it's client/apis/controller packages from pkg/kopia/repositoryserver * Added RepositoryServer CRD manifest, apis code and client code to pkg/customresource, pkg/apis and pkg/client respectively * Added the symlink to pkg/customresource/repositoryserver.yaml in helm/kanister-operator/crds * Revised the CRD fields * Remove RepositoryServerResource from CreateCustomResources list since it's used by Kanister controller * Add a tag for repositoryserver.yaml from customresource files in pkg/customresource/embed.go * Remove noStatus local tag from RepositoryServer in types.go to generate UpdateStatus code for it * Add Status subresource to RepositoryServer CRD * Add the 'Ref' suffix to SecretReference CRD fields, update NetworkPolicy labelselector field and replace some omitempty CRD fields to required * Remove kubebuilder specific annotations from CRD YAML * Update CRD type and deepcopy function to fix the pod and namespace LabelSelector error * Added Kubebuilder skeleton code * Revert go.sum * Fix vet and fmt target in Kubebuilder.mk * Merge Kubebuilder.mk and Makefile * Remove TODO in Makefile to update the 'test' target name * Rename Kubebuilder build, test, run, deploy and undeploy targets in Makefile * Add isReady field to the CR * Add extra line at the EOF in the YAML * Remove the clientset, listers and informers for RepositoryServer CR * Update RepositoryServer CRD with 'isReady' status field * Regenerate deepcopy methods for the CRs using 'make generate' target * Correct the 'isReady' field in CRD * Move sample RepositoryServer CR YAML to a different file * Populate pod object using podOptions in a function separate from CreatePod * Add 'ServerPending' state of RepositoryServer CR in types.go * Change 'isReady' field to 'Progress' in RepositoryServer CR status * remove network policy lifecycle from the controller * address comments * remove prometheus config * remove network policy configuration from CRD * add server access username changes to repository server CR * move the server access username to CR and get the hostname and password from secret * add server access username changes to repository server CR * add repo server controller deployment * go mod tidy * add repository server container in the kanister deployment * remove unwanted kustomization files * sync the GetPodObjectFromPodOptions to master branch * sync the GetPodObjectFromPodOptions to master branch * remove unncessary kustomization.yaml * rebase with crd PR * fix lint issues * remove kustomization files * move context from struct to function variables * move context from struct to function variables * make server start timeout configurable * make repository server start timeout configurable * make server start timeout configurable * go mod tidy * modify copyright information * adding client set for repository server CR * address review comment * address review comments * address review comments * address review comments * move the Makefiles to repository server deployment PR #1871 * add makefile for repository server controller * move the Makefiles to repository server deployment PR #1871 * go mod tidy * address review comments * address review comments * change coyright information * add event filter to skip status update events * make log level configurable, move pod security context to pod options * add event recorder * make repository cache settings configurable using repository server CR * address review comments * remove kubernetes authors from repositoryserver_types.go * test * add comments to the repository server types * add comments to the repository server types * add comments to the repository server types * address review comments related to inconsistent error/log messages * log messages at debug level * address review comments * address review comments * go mod tidy * address review comment * go mod tidy * add log level in values * testing * retrying after sometime * hide repository server controller beyond a flag * remove unnecessary commits * add default enabled flag in values.yaml * fix typo * resolve conflicts * address review comments --------- Signed-off-by: Ivan Sim Signed-off-by: dependabot[bot] Co-authored-by: shlokchaudhari9 Co-authored-by: Ivan Sim <1330522+ihcsim@users.noreply.github.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Pavan Navarathna <6504783+pavannd1@users.noreply.github.com> --- .../cr/v1alpha1/repositoryserver_types.go | 8 +++++++ .../repositoryserver/repository.go | 24 ++++++++++++++++++- pkg/customresource/repositoryserver.yaml | 12 ++++++++++ 3 files changed, 43 insertions(+), 1 deletion(-) diff --git a/pkg/apis/cr/v1alpha1/repositoryserver_types.go b/pkg/apis/cr/v1alpha1/repositoryserver_types.go index 1a2e9623607..1bfc4e682f9 100644 --- a/pkg/apis/cr/v1alpha1/repositoryserver_types.go +++ b/pkg/apis/cr/v1alpha1/repositoryserver_types.go @@ -76,6 +76,14 @@ type Repository struct { Hostname string `json:"hostname"` // PasswordSecretRef has the password required to connect to kopia repository PasswordSecretRef corev1.SecretReference `json:"passwordSecretRef"` + CacheSizeSettings CacheSizeSettings `json:"cacheSizeSettings,omitempty"` +} + +// CacheSettings are the metadata/content cache size details +// that can be used while establishing connection to the kopia repository +type CacheSizeSettings struct { + Metadata string `json:"metadata"` + Content string `json:"content"` } // Server details required for starting the repository proxy server and initializing the repository client users diff --git a/pkg/controllers/repositoryserver/repository.go b/pkg/controllers/repositoryserver/repository.go index 81f2e5cb338..07840e70ca6 100644 --- a/pkg/controllers/repositoryserver/repository.go +++ b/pkg/controllers/repositoryserver/repository.go @@ -15,6 +15,8 @@ package repositoryserver import ( + "strconv" + "github.com/kanisterio/kanister/pkg/kopia/command" "github.com/kanisterio/kanister/pkg/kopia/repository" ) @@ -27,7 +29,10 @@ const ( ) func (h *RepoServerHandler) connectToKopiaRepository() error { - contentCacheMB, metadataCacheMB := command.GetGeneralCacheSizeSettings() + contentCacheMB, metadataCacheMB, err := h.getRepositoryCacheSettings() + if err != nil { + return err + } args := command.RepositoryCommandArgs{ CommandArgs: &command.CommandArgs{ RepoPassword: string(h.RepositoryServerSecrets.repositoryPassword.Data[repoPasswordKey]), @@ -52,3 +57,20 @@ func (h *RepoServerHandler) connectToKopiaRepository() error { args, ) } + +func (h *RepoServerHandler) getRepositoryCacheSettings() (contentCacheMB, metadataCacheMB int, err error) { + contentCacheMB, metadataCacheMB = command.GetGeneralCacheSizeSettings() + if h.RepositoryServer.Spec.Repository.CacheSizeSettings.Content != "" { + contentCacheMB, err = strconv.Atoi(h.RepositoryServer.Spec.Repository.CacheSizeSettings.Content) + if err != nil { + return + } + } + if h.RepositoryServer.Spec.Repository.CacheSizeSettings.Metadata != "" { + metadataCacheMB, err = strconv.Atoi(h.RepositoryServer.Spec.Repository.CacheSizeSettings.Metadata) + if err != nil { + return + } + } + return +} diff --git a/pkg/customresource/repositoryserver.yaml b/pkg/customresource/repositoryserver.yaml index ea2477057a5..3fa2cf55358 100644 --- a/pkg/customresource/repositoryserver.yaml +++ b/pkg/customresource/repositoryserver.yaml @@ -37,6 +37,18 @@ spec: description: Repository has the details required by the repository server to connect to kopia repository properties: + cacheSizeSettings: + description: CacheSettings are the metadata/content cache size + details that can be used for establishing connection to the kopia repository + properties: + content: + type: string + metadata: + type: string + required: + - content + - metadata + type: object hostname: description: If specified, these values will be used by the controller to override default hostname when connecting to the repository From fd4624ace448aa188e6db1de5be3844da90a099c Mon Sep 17 00:00:00 2001 From: Rajat Gupta <37516416+r4rajat@users.noreply.github.com> Date: Mon, 29 May 2023 10:42:30 +0530 Subject: [PATCH 4/7] Add functionality to create RepositoryServer CR using kanctl command line (#2032) * Add repository-server in kanctl create Signed-off-by: Rajat Gupta * Add docs Signed-off-by: Rajat Gupta * Change Flag values Signed-off-by: Rajat Gupta * Update docs Signed-off-by: Rajat Gupta * Add s3compliant storage option for kopia repository server Signed-off-by: Rajat Gupta * Address Comments Signed-off-by: Rajat Gupta * Remove support for using namespace/name Signed-off-by: Rajat Gupta * Addressed Comments Signed-off-by: Rajat Gupta * Addressed Comments Signed-off-by: Rajat Gupta * Updated Documentation Signed-off-by: Rajat Gupta * Address Comments Signed-off-by: Rajat Gupta * Error handling for clients initialization Signed-off-by: Rajat Gupta * Update pkg/kanctl/repositoryserver.go Co-authored-by: Ankit Jain * Update documentation Signed-off-by: Rajat Gupta * Address comments Signed-off-by: Rajat Gupta * Add wait feature Signed-off-by: Rajat Gupta * Dynamic kanister namespace Signed-off-by: Rajat Gupta * Update pkg/kanctl/repositoryserver.go Co-authored-by: Vivek Singh * Update pkg/kanctl/repositoryserver.go Co-authored-by: Vivek Singh * Update pkg/kanctl/repositoryserver.go Co-authored-by: Vivek Singh * Update documentation Signed-off-by: Rajat Gupta * Fix Lint Issue Signed-off-by: Rajat Gupta * Address Comment Signed-off-by: Rajat Gupta * Add descriptive output in case repo server creation fails with --wait Signed-off-by: Rajat Gupta * Fix Lint Issues Signed-off-by: Rajat Gupta * Update flags Signed-off-by: Rajat Gupta * Update flags Signed-off-by: Rajat Gupta * Update Documentation Signed-off-by: Rajat Gupta * Revert go.sum Signed-off-by: Rajat Gupta * Revert go.sum Signed-off-by: Rajat Gupta * Add kopia in flags Signed-off-by: Rajat Gupta --------- Signed-off-by: Rajat Gupta Co-authored-by: Ankit Jain Co-authored-by: Vivek Singh --- docs/tooling.rst | 32 ++++ pkg/kanctl/create.go | 1 + pkg/kanctl/repositoryserver.go | 307 +++++++++++++++++++++++++++++++++ 3 files changed, 340 insertions(+) create mode 100644 pkg/kanctl/repositoryserver.go diff --git a/docs/tooling.rst b/docs/tooling.rst index 9f3f1bdea2b..2967cc42740 100644 --- a/docs/tooling.rst +++ b/docs/tooling.rst @@ -40,6 +40,7 @@ kanctl create Available Commands: actionset Create a new ActionSet or override a ActionSet profile Create a new profile + repository-server Create a new kopia repository server Flags: --dry-run if set, resource YAML will be printed but not created @@ -229,6 +230,37 @@ A new S3Compliant profile can be created using the s3compliant subcommand secret 's3-secret-chst2' created profile 's3-profile-5mmkj' created + +Kopia Repository Server resource creation using ``kanctl create`` + +.. code-block:: bash + + $ kanctl create repository-server --help + Create a new RepositoryServer + + Usage: + kanctl create repository-server [flags] + + Flags: + -a, --admin-user-access-secret string name of the secret having admin credentials to connect to connect to kopia repository server + -r, --kopia-repository-password-secret string name of the secret containing password for the kopia repository + -k, --kopia-repository-user string name of the user for accessing the kopia repository + -c, --location-creds-secret string name of the secret containing kopia repository storage credentials + -l, --location-secret string name of the secret containing kopia repository storage location details + -p, --prefix string prefix to be set in kopia repository + -t, --tls-secret string name of the tls secret needed for secure kopia client and kopia repository server communication + -u, --user string name of the user to be created for the kopia repository server + -s, --user-access-secret string name of the secret having access credentials of the users that can connect to kopia repository server + -w, --wait wait for the kopia repository server to be in ready state after creation + -h, --help help for repository-server + + Global Flags: + --dry-run if set, resource YAML will be printed but not created + -n, --namespace string Override namespace obtained from kubectl context + --skip-validation if set, resource is not validated before creation + --verbose Display verbose output + + .. _kanctlvalidate: kanctl validate diff --git a/pkg/kanctl/create.go b/pkg/kanctl/create.go index 011cf190c6c..5d901d17dd7 100644 --- a/pkg/kanctl/create.go +++ b/pkg/kanctl/create.go @@ -30,6 +30,7 @@ func newCreateCommand() *cobra.Command { } cmd.AddCommand(newActionSetCmd()) cmd.AddCommand(newProfileCommand()) + cmd.AddCommand(newRepositoryServerCommand()) cmd.PersistentFlags().Bool(dryRunFlag, false, "if set, resource YAML will be printed but not created") cmd.PersistentFlags().Bool(skipValidationFlag, false, "if set, resource is not validated before creation") return cmd diff --git a/pkg/kanctl/repositoryserver.go b/pkg/kanctl/repositoryserver.go new file mode 100644 index 00000000000..c16c80ca466 --- /dev/null +++ b/pkg/kanctl/repositoryserver.go @@ -0,0 +1,307 @@ +// Copyright 2023 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package kanctl + +import ( + "context" + "fmt" + "strings" + "time" + + "github.com/pkg/errors" + "github.com/spf13/cobra" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + + "github.com/kanisterio/kanister/pkg/apis/cr/v1alpha1" + "github.com/kanisterio/kanister/pkg/client/clientset/versioned" + "github.com/kanisterio/kanister/pkg/kube" + "github.com/kanisterio/kanister/pkg/poll" +) + +const ( + tlsSecretFlag = "tls-secret" + repoServerUserFlag = "user" + repoServerUserAccessSecretFlag = "user-access-secret" + repoServerAdminUserAccessSecretFlag = "admin-user-access-secret" + kopiaRepoPasswordSecretFlag = "kopia-repository-password-secret" + kopiaRepoUserFlag = "kopia-repository-user" + locationCredsSecretFlag = "location-creds-secret" + locationSecretFlag = "location-secret" + defaultRepositoryServerHost = "localhost" + waitFlag = "wait" + contextWaitTimeout = 10 * time.Minute +) + +type repositoryServerParams struct { + tls string + repositoryServerUser string + repositoryServerUserAccess string + repositoryServerAdminUserAccess string + repositoryUser string + repositoryPassword string + prefix string + location string + locationCreds string + namespace string +} + +func newRepositoryServerCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "repository-server", + Short: "Create a new RepositoryServer", + Args: cobra.ExactArgs(0), + RunE: func(cmd *cobra.Command, args []string) error { + return createNewRepositoryServer(cmd, args) + }, + } + + cmd.PersistentFlags().StringP(tlsSecretFlag, "t", "", "name of the tls secret needed for secure kopia client and kopia repository server communication") + cmd.PersistentFlags().StringP(repoServerUserFlag, "u", "", "name of the user to be created for the kopia repository server") + cmd.PersistentFlags().StringP(repoServerUserAccessSecretFlag, "s", "", "name of the secret having access credentials of the users that can connect to kopia repository server") + cmd.PersistentFlags().StringP(repoServerAdminUserAccessSecretFlag, "a", "", "name of the secret having admin credentials to connect to connect to kopia repository server") + cmd.PersistentFlags().StringP(kopiaRepoPasswordSecretFlag, "r", "", "name of the secret containing password for the kopia repository") + cmd.PersistentFlags().StringP(prefixFlag, "p", "", "prefix to be set in kopia repository") + cmd.PersistentFlags().StringP(kopiaRepoUserFlag, "k", "", "name of the user for accessing the kopia repository") + cmd.PersistentFlags().StringP(locationSecretFlag, "l", "", "name of the secret containing kopia repository storage location details") + cmd.PersistentFlags().StringP(locationCredsSecretFlag, "c", "", "name of the secret containing kopia repository storage credentials") + cmd.PersistentFlags().BoolP(waitFlag, "w", false, "wait for the kopia repository server to be in ready state after creation") + + _ = cmd.MarkFlagRequired(tlsSecretFlag) + _ = cmd.MarkFlagRequired(repoServerUserFlag) + _ = cmd.MarkFlagRequired(repoServerUserAccessSecretFlag) + _ = cmd.MarkFlagRequired(repoServerAdminUserAccessSecretFlag) + _ = cmd.MarkFlagRequired(kopiaRepoPasswordSecretFlag) + _ = cmd.MarkFlagRequired(prefixFlag) + _ = cmd.MarkFlagRequired(locationSecretFlag) + _ = cmd.MarkFlagRequired(locationCredsSecretFlag) + return cmd +} + +func createNewRepositoryServer(cmd *cobra.Command, args []string) error { + cmd.SilenceUsage = true + if len(args) > 0 { + return newArgsLengthError("expected 0 args. Got %#v", args) + } + + rsParams, err := generateRepositoryServerParams(cmd) + if err != nil { + return err + } + + repositoryServer, err := validateSecretsAndConstructRepositoryServer(rsParams) + if err != nil { + return err + } + + config, err := kube.LoadConfig() + if err != nil { + return err + } + cli, err := kubernetes.NewForConfig(config) + if err != nil { + return errors.Wrap(err, "could not get the kubernetes client") + } + crCli, err := versioned.NewForConfig(config) + if err != nil { + return errors.Wrap(err, "could not get the CRD client") + } + + ctx := context.Background() + rs, err := crCli.CrV1alpha1().RepositoryServers(rsParams.namespace).Create(ctx, repositoryServer, metav1.CreateOptions{}) + if err != nil { + return err + } + fmt.Printf("repositoryservers.cr.kanister.io/%s created\n", rs.GetName()) + + waitFlag, _ := cmd.Flags().GetBool(waitFlag) + if waitFlag { + fmt.Print("Waiting for the kopia repository server CR to be in ready state...\n") + err = waitForRepositoryServerReady(ctx, cli, crCli, rs) + if err != nil { + return err + } + fmt.Printf("repositoryservers.cr.kanister.io/%s is ready.\n", rs.GetName()) + } + return nil +} + +func generateRepositoryServerParams(cmd *cobra.Command) (*repositoryServerParams, error) { + // Fetch values of the flags + tlsSecret, _ := cmd.Flags().GetString(tlsSecretFlag) + if strings.Contains(tlsSecret, "/") { + return nil, errors.Errorf("Invalid secret name %s, it should not be of the form namespace/name )", tlsSecret) + } + + repositoryServerUser, _ := cmd.Flags().GetString(repoServerUserFlag) + + repositoryServerUserAccessSecret, _ := cmd.Flags().GetString(repoServerUserAccessSecretFlag) + if strings.Contains(repositoryServerUserAccessSecret, "/") { + return nil, errors.Errorf("Invalid secret name %s, it should not be of the form namespace/name )", repositoryServerUserAccessSecret) + } + + repositoryServerAdminUserAccessSecret, _ := cmd.Flags().GetString(repoServerAdminUserAccessSecretFlag) + if strings.Contains(repositoryServerAdminUserAccessSecret, "/") { + return nil, errors.Errorf("Invalid secret name %s, it should not be of the form namespace/name )", repositoryServerAdminUserAccessSecret) + } + + repositoryUser, _ := cmd.Flags().GetString(kopiaRepoUserFlag) + + repositoryPassword, _ := cmd.Flags().GetString(kopiaRepoPasswordSecretFlag) + if strings.Contains(repositoryPassword, "/") { + return nil, errors.Errorf("Invalid secret name %s, it should not be of the form namespace/name )", repositoryPassword) + } + + prefix, _ := cmd.Flags().GetString(prefixFlag) + + location, _ := cmd.Flags().GetString(locationSecretFlag) + if strings.Contains(location, "/") { + return nil, errors.Errorf("Invalid secret name %s, it should not be of the form namespace/name )", location) + } + + locationCreds, _ := cmd.Flags().GetString(locationCredsSecretFlag) + if strings.Contains(locationCreds, "/") { + return nil, errors.Errorf("Invalid secret name %s, it should not be of the form namespace/name )", locationCreds) + } + + ns, err := resolveNamespace(cmd) + if err != nil { + return nil, err + } + + return &repositoryServerParams{ + tls: tlsSecret, + repositoryServerUser: repositoryServerUser, + repositoryServerUserAccess: repositoryServerUserAccessSecret, + repositoryUser: repositoryUser, + repositoryServerAdminUserAccess: repositoryServerAdminUserAccessSecret, + repositoryPassword: repositoryPassword, + prefix: prefix, + location: location, + locationCreds: locationCreds, + namespace: ns, + }, nil +} + +func validateSecretsAndConstructRepositoryServer(rsParams *repositoryServerParams) (*v1alpha1.RepositoryServer, error) { + // Fetch and Validate Secrets + ctx := context.Background() + config, err := kube.LoadConfig() + if err != nil { + return nil, err + } + cli, err := kubernetes.NewForConfig(config) + if err != nil { + return nil, errors.Wrap(err, "could not get the kubernetes client") + } + tlsSecret, err := cli.CoreV1().Secrets(rsParams.namespace).Get(ctx, rsParams.tls, metav1.GetOptions{}) + if err != nil { + return nil, err + } + repositoryServerUserAccessSecret, err := cli.CoreV1().Secrets(rsParams.namespace).Get(ctx, rsParams.repositoryServerUserAccess, metav1.GetOptions{}) + if err != nil { + return nil, err + } + repositoryServerAdminUserAccessSecret, err := cli.CoreV1().Secrets(rsParams.namespace).Get(ctx, rsParams.repositoryServerAdminUserAccess, metav1.GetOptions{}) + if err != nil { + return nil, err + } + repositoryPasswordSecret, err := cli.CoreV1().Secrets(rsParams.namespace).Get(ctx, rsParams.repositoryPassword, metav1.GetOptions{}) + if err != nil { + return nil, err + } + locationSecret, err := cli.CoreV1().Secrets(rsParams.namespace).Get(ctx, rsParams.location, metav1.GetOptions{}) + if err != nil { + return nil, err + } + locationCredsSecret, err := cli.CoreV1().Secrets(rsParams.namespace).Get(ctx, rsParams.locationCreds, metav1.GetOptions{}) + if err != nil { + return nil, err + } + + return &v1alpha1.RepositoryServer{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: `kopia-repo-server-`, + }, + Spec: v1alpha1.RepositoryServerSpec{ + Storage: v1alpha1.Storage{ + SecretRef: corev1.SecretReference{ + Name: locationSecret.GetName(), + Namespace: locationSecret.GetNamespace(), + }, + CredentialSecretRef: corev1.SecretReference{ + Name: locationCredsSecret.GetName(), + Namespace: locationCredsSecret.GetNamespace(), + }, + }, + Repository: v1alpha1.Repository{ + RootPath: rsParams.prefix, + Username: rsParams.repositoryUser, + Hostname: defaultRepositoryServerHost, + PasswordSecretRef: corev1.SecretReference{ + Name: repositoryPasswordSecret.GetName(), + Namespace: repositoryPasswordSecret.GetNamespace(), + }, + }, + Server: v1alpha1.Server{ + UserAccess: v1alpha1.UserAccess{ + UserAccessSecretRef: corev1.SecretReference{ + Name: repositoryServerUserAccessSecret.GetName(), + Namespace: repositoryServerUserAccessSecret.GetNamespace(), + }, + Username: rsParams.repositoryServerUser, + }, + AdminSecretRef: corev1.SecretReference{ + Name: repositoryServerAdminUserAccessSecret.GetName(), + Namespace: repositoryServerAdminUserAccessSecret.GetNamespace(), + }, + TLSSecretRef: corev1.SecretReference{ + Name: tlsSecret.GetName(), + Namespace: tlsSecret.GetNamespace(), + }, + }, + }, + }, nil +} + +func waitForRepositoryServerReady(ctx context.Context, cli *kubernetes.Clientset, crCli *versioned.Clientset, rs *v1alpha1.RepositoryServer) error { + timeoutCtx, waitCancel := context.WithTimeout(ctx, contextWaitTimeout) + defer waitCancel() + pollErr := poll.Wait(timeoutCtx, func(ctx context.Context) (bool, error) { + repositoryServer, err := crCli.CrV1alpha1().RepositoryServers(rs.GetNamespace()).Get(ctx, rs.GetName(), metav1.GetOptions{}) + if repositoryServer.Status.Progress == v1alpha1.ServerReady && err == nil { + return true, nil + } + return false, err + }) + if pollErr != nil { + repositoryServer, err := crCli.CrV1alpha1().RepositoryServers(rs.GetNamespace()).Get(ctx, rs.GetName(), metav1.GetOptions{}) + if err != nil { + return errors.Wrapf(err, "Error Getting repository server %s", repositoryServer.GetName()) + } + + opts := metav1.ListOptions{ + FieldSelector: fmt.Sprintf("involvedObject.name=%s", repositoryServer.GetName()), + } + events, err := cli.CoreV1().Events(repositoryServer.GetNamespace()).List(ctx, opts) + if err != nil { + return err + } + + return errors.Wrapf(pollErr, "Repository Server is not ready.\nCurrent Status: %s\nReason: %s\n", repositoryServer.Status.Progress, events.Items[0].Message) + } + return nil +} From 6169a0f98402d881c9785dd5e6be4dd6b6580b08 Mon Sep 17 00:00:00 2001 From: kale-amruta <41624751+kale-amruta@users.noreply.github.com> Date: Tue, 30 May 2023 12:25:17 +0530 Subject: [PATCH 5/7] Move kopia storage consts to a secrets package (#2059) * move the consts to a secrets package so that they can be reused * move location key constants to repositoryserver package * refactor imports --- .../repositoryserver/repository.go | 9 +- pkg/controllers/repositoryserver/server.go | 2 +- pkg/kopia/command/storage/azure_args_test.go | 6 +- .../command/storage/filesystem_args_test.go | 4 +- pkg/kopia/command/storage/gcs_args_test.go | 10 +- pkg/kopia/command/storage/s3_args_test.go | 22 ++-- pkg/kopia/command/storage/secret_utils.go | 49 +++------ .../command/storage/secret_utils_test.go | 101 +++++++++--------- pkg/kopia/command/storage/storage_args.go | 9 +- .../command/storage/storage_args_test.go | 30 +++--- pkg/secrets/repositoryserver/const.go | 50 +++++++++ 11 files changed, 168 insertions(+), 124 deletions(-) create mode 100644 pkg/secrets/repositoryserver/const.go diff --git a/pkg/controllers/repositoryserver/repository.go b/pkg/controllers/repositoryserver/repository.go index 07840e70ca6..91174be75c5 100644 --- a/pkg/controllers/repositoryserver/repository.go +++ b/pkg/controllers/repositoryserver/repository.go @@ -22,10 +22,7 @@ import ( ) const ( - repoPasswordKey = "repo-password" - defaultRepoConfigFilePath = "/tmp/config.file" - defaultRepoLogDirectory = "/tmp/log.dir" - defaultCacheDirectory = "/tmp/cache.dir" + repoPasswordKey = "repo-password" ) func (h *RepoServerHandler) connectToKopiaRepository() error { @@ -37,9 +34,9 @@ func (h *RepoServerHandler) connectToKopiaRepository() error { CommandArgs: &command.CommandArgs{ RepoPassword: string(h.RepositoryServerSecrets.repositoryPassword.Data[repoPasswordKey]), ConfigFilePath: command.DefaultConfigFilePath, - LogDirectory: command.DefaultCacheDirectory, + LogDirectory: command.DefaultLogDirectory, }, - CacheDirectory: defaultCacheDirectory, + CacheDirectory: command.DefaultCacheDirectory, Hostname: h.RepositoryServer.Spec.Repository.Hostname, ContentCacheMB: contentCacheMB, MetadataCacheMB: metadataCacheMB, diff --git a/pkg/controllers/repositoryserver/server.go b/pkg/controllers/repositoryserver/server.go index 84981dc1dc4..b22bfe67b89 100644 --- a/pkg/controllers/repositoryserver/server.go +++ b/pkg/controllers/repositoryserver/server.go @@ -54,7 +54,7 @@ func (h *RepoServerHandler) startRepoProxyServer(ctx context.Context) (err error CommandArgs: &command.CommandArgs{ RepoPassword: "", ConfigFilePath: command.DefaultConfigFilePath, - LogDirectory: command.DefaultCacheDirectory, + LogDirectory: command.DefaultLogDirectory, }, ServerAddress: repoServerAddress, TLSCertFile: tlsCertPath, diff --git a/pkg/kopia/command/storage/azure_args_test.go b/pkg/kopia/command/storage/azure_args_test.go index ba5d64d25b6..db8166a58d0 100644 --- a/pkg/kopia/command/storage/azure_args_test.go +++ b/pkg/kopia/command/storage/azure_args_test.go @@ -18,6 +18,8 @@ import ( "fmt" "gopkg.in/check.v1" + + "github.com/kanisterio/kanister/pkg/secrets/repositoryserver" ) func (s *StorageUtilsSuite) TestAzureArgsUtil(c *check.C) { @@ -28,8 +30,8 @@ func (s *StorageUtilsSuite) TestAzureArgsUtil(c *check.C) { }{ { location: map[string][]byte{ - bucketKey: []byte("test-bucket"), - prefixKey: []byte("test-prefix"), + repositoryserver.BucketKey: []byte("test-bucket"), + repositoryserver.PrefixKey: []byte("test-prefix"), }, expectedCommand: fmt.Sprint(azureSubCommand, fmt.Sprintf(" %s=%s ", azureContainerFlag, "test-bucket"), diff --git a/pkg/kopia/command/storage/filesystem_args_test.go b/pkg/kopia/command/storage/filesystem_args_test.go index bf9300d3645..e77ca07785e 100644 --- a/pkg/kopia/command/storage/filesystem_args_test.go +++ b/pkg/kopia/command/storage/filesystem_args_test.go @@ -18,6 +18,8 @@ import ( "fmt" "gopkg.in/check.v1" + + "github.com/kanisterio/kanister/pkg/secrets/repositoryserver" ) func (s *StorageUtilsSuite) TestFilesystemArgsUtil(c *check.C) { @@ -38,7 +40,7 @@ func (s *StorageUtilsSuite) TestFilesystemArgsUtil(c *check.C) { }, } { sec := map[string][]byte{ - prefixKey: []byte(tc.prefix), + repositoryserver.PrefixKey: []byte(tc.prefix), } args := filesystemArgs(sec, tc.repoPathPrefix) expectedValue := fmt.Sprint( diff --git a/pkg/kopia/command/storage/gcs_args_test.go b/pkg/kopia/command/storage/gcs_args_test.go index 5cbb8d64368..24a26bc1a2f 100644 --- a/pkg/kopia/command/storage/gcs_args_test.go +++ b/pkg/kopia/command/storage/gcs_args_test.go @@ -18,19 +18,21 @@ import ( "fmt" "gopkg.in/check.v1" + + "github.com/kanisterio/kanister/pkg/secrets/repositoryserver" ) func (s *StorageUtilsSuite) TestGCSArgsUtil(c *check.C) { locSecret := map[string][]byte{ - prefixKey: []byte("test-prefix"), - bucketKey: []byte("test-bucket"), + repositoryserver.PrefixKey: []byte("test-prefix"), + repositoryserver.BucketKey: []byte("test-bucket"), } repoPathPrefix := "dir/sub-dir" cmd := gcsArgs(locSecret, repoPathPrefix) c.Assert(cmd.String(), check.Equals, fmt.Sprint( gcsSubCommand, - fmt.Sprintf(" --%s=%s", bucketKey, locSecret[bucketKey]), + fmt.Sprintf(" --%s=%s", repositoryserver.BucketKey, locSecret[repositoryserver.BucketKey]), fmt.Sprintf(" %s=/tmp/creds.txt", credentialsFileFlag), - fmt.Sprintf(" --%s=%s/%s/", prefixKey, locSecret[prefixKey], repoPathPrefix), + fmt.Sprintf(" --%s=%s/%s/", repositoryserver.PrefixKey, locSecret[repositoryserver.PrefixKey], repoPathPrefix), )) } diff --git a/pkg/kopia/command/storage/s3_args_test.go b/pkg/kopia/command/storage/s3_args_test.go index e72b9479fc9..9531b378d58 100644 --- a/pkg/kopia/command/storage/s3_args_test.go +++ b/pkg/kopia/command/storage/s3_args_test.go @@ -18,6 +18,8 @@ import ( "fmt" "gopkg.in/check.v1" + + "github.com/kanisterio/kanister/pkg/secrets/repositoryserver" ) func (s *StorageUtilsSuite) TestS3ArgsUtil(c *check.C) { @@ -28,10 +30,10 @@ func (s *StorageUtilsSuite) TestS3ArgsUtil(c *check.C) { }{ { location: map[string][]byte{ - bucketKey: []byte("test-bucket"), - prefixKey: []byte("test-prefix"), - regionKey: []byte("test-region"), - skipSSLVerifyKey: []byte("true"), + repositoryserver.BucketKey: []byte("test-bucket"), + repositoryserver.PrefixKey: []byte("test-prefix"), + repositoryserver.RegionKey: []byte("test-region"), + repositoryserver.SkipSSLVerifyKey: []byte("true"), }, expectedCommand: fmt.Sprint(s3SubCommand, fmt.Sprintf(" %s=%s", bucketFlag, "test-bucket"), @@ -42,9 +44,9 @@ func (s *StorageUtilsSuite) TestS3ArgsUtil(c *check.C) { }, { location: map[string][]byte{ - bucketKey: []byte("test-bucket"), - prefixKey: []byte("test-prefix"), - endpointKey: []byte("https://test.test:9000/"), + repositoryserver.BucketKey: []byte("test-bucket"), + repositoryserver.PrefixKey: []byte("test-prefix"), + repositoryserver.EndpointKey: []byte("https://test.test:9000/"), }, expectedCommand: fmt.Sprint(s3SubCommand, fmt.Sprintf(" %s=%s", bucketFlag, "test-bucket"), @@ -53,9 +55,9 @@ func (s *StorageUtilsSuite) TestS3ArgsUtil(c *check.C) { }, { location: map[string][]byte{ - bucketKey: []byte("test-bucket"), - prefixKey: []byte("test-prefix"), - endpointKey: []byte("http://test.test:9000"), + repositoryserver.BucketKey: []byte("test-bucket"), + repositoryserver.PrefixKey: []byte("test-prefix"), + repositoryserver.EndpointKey: []byte("http://test.test:9000"), }, expectedCommand: fmt.Sprint(s3SubCommand, fmt.Sprintf(" %s=%s", bucketFlag, "test-bucket"), diff --git a/pkg/kopia/command/storage/secret_utils.go b/pkg/kopia/command/storage/secret_utils.go index 0c36f3d3a1f..1935f7354be 100644 --- a/pkg/kopia/command/storage/secret_utils.go +++ b/pkg/kopia/command/storage/secret_utils.go @@ -25,25 +25,10 @@ import ( "github.com/kanisterio/kanister/pkg/apis/cr/v1alpha1" "github.com/kanisterio/kanister/pkg/aws" "github.com/kanisterio/kanister/pkg/secrets" + "github.com/kanisterio/kanister/pkg/secrets/repositoryserver" ) -type LocType string - const ( - // Location secret keys - bucketKey = "bucket" - endpointKey = "endpoint" - prefixKey = "prefix" - regionKey = "region" - skipSSLVerifyKey = "skipSSLVerify" - typeKey = "type" - - // Location types - LocTypeS3 LocType = "s3" - LocTypeGCS LocType = "gcs" - LocTypeAzure LocType = "azure" - LocTypeFilestore LocType = "filestore" - // Azure location related environment variables azureStorageAccountEnv = "AZURE_STORAGE_ACCOUNT" azureStorageKeyEnv = "AZURE_STORAGE_KEY" @@ -51,28 +36,28 @@ const ( ) func getBucketNameFromMap(m map[string][]byte) string { - return string(m[bucketKey]) + return string(m[repositoryserver.BucketKey]) } func getEndpointFromMap(m map[string][]byte) string { - return string(m[endpointKey]) + return string(m[repositoryserver.EndpointKey]) } func getPrefixFromMap(m map[string][]byte) string { - return string(m[prefixKey]) + return string(m[repositoryserver.PrefixKey]) } func getRegionFromMap(m map[string][]byte) string { - return string(m[regionKey]) + return string(m[repositoryserver.RegionKey]) } func checkSkipSSLVerifyFromMap(m map[string][]byte) bool { - v := string(m[skipSSLVerifyKey]) + v := string(m[repositoryserver.SkipSSLVerifyKey]) return v == "true" } -func locationType(m map[string][]byte) LocType { - return LocType(m[typeKey]) +func locationType(m map[string][]byte) repositoryserver.LocType { + return repositoryserver.LocType(m[repositoryserver.TypeKey]) } // GenerateEnvSpecFromCredentialSecret parses the secret and returns @@ -159,7 +144,7 @@ func getEnvVar(varName, value string) v1.EnvVar { // GetMapForLocationValues return a map with valid keys // for different location values func GetMapForLocationValues( - locType LocType, + locType repositoryserver.LocType, prefix, region, bucket, @@ -168,24 +153,24 @@ func GetMapForLocationValues( ) map[string][]byte { m := map[string][]byte{} if bucket != "" { - m[bucketKey] = []byte(bucket) + m[repositoryserver.BucketKey] = []byte(bucket) } if endpoint != "" { - m[endpointKey] = []byte(endpoint) + m[repositoryserver.EndpointKey] = []byte(endpoint) } if prefix != "" { - m[prefixKey] = []byte(prefix) + m[repositoryserver.PrefixKey] = []byte(prefix) } if region != "" { - m[regionKey] = []byte(region) + m[repositoryserver.RegionKey] = []byte(region) } if skipSSLVerify != "" { - m[skipSSLVerifyKey] = []byte(skipSSLVerify) + m[repositoryserver.SkipSSLVerifyKey] = []byte(skipSSLVerify) } if locType != "" { - m[typeKey] = []byte(locType) - if locType == LocType(v1alpha1.LocationTypeS3Compliant) { - m[typeKey] = []byte(LocTypeS3) + m[repositoryserver.TypeKey] = []byte(locType) + if locType == repositoryserver.LocType(v1alpha1.LocationTypeS3Compliant) { + m[repositoryserver.TypeKey] = []byte(repositoryserver.LocTypeS3) } } return m diff --git a/pkg/kopia/command/storage/secret_utils_test.go b/pkg/kopia/command/storage/secret_utils_test.go index eb0c4e1879b..edeeaa4b7f5 100644 --- a/pkg/kopia/command/storage/secret_utils_test.go +++ b/pkg/kopia/command/storage/secret_utils_test.go @@ -25,6 +25,7 @@ import ( "github.com/kanisterio/kanister/pkg/apis/cr/v1alpha1" "github.com/kanisterio/kanister/pkg/aws" "github.com/kanisterio/kanister/pkg/secrets" + "github.com/kanisterio/kanister/pkg/secrets/repositoryserver" ) func Test(t *testing.T) { check.TestingT(t) } @@ -35,43 +36,43 @@ var _ = check.Suite(&StorageUtilsSuite{}) func (s *StorageUtilsSuite) TestLocationUtils(c *check.C) { loc := map[string][]byte{ - bucketKey: []byte("test-key"), - endpointKey: []byte("test-endpoint"), - prefixKey: []byte("test-prefix"), - regionKey: []byte("test-region"), - skipSSLVerifyKey: []byte("true"), + repositoryserver.BucketKey: []byte("test-key"), + repositoryserver.EndpointKey: []byte("test-endpoint"), + repositoryserver.PrefixKey: []byte("test-prefix"), + repositoryserver.RegionKey: []byte("test-region"), + repositoryserver.SkipSSLVerifyKey: []byte("true"), } for _, tc := range []struct { LocType string - expectedLocType LocType + expectedLocType repositoryserver.LocType skipSSLVerify string expectedSkipSSLVerifyValue bool }{ { LocType: "s3", - expectedLocType: LocTypeS3, + expectedLocType: repositoryserver.LocTypeS3, skipSSLVerify: "true", expectedSkipSSLVerifyValue: true, }, { LocType: "gcs", - expectedLocType: LocTypeGCS, + expectedLocType: repositoryserver.LocTypeGCS, skipSSLVerify: "false", expectedSkipSSLVerifyValue: false, }, { LocType: "azure", - expectedLocType: LocTypeAzure, + expectedLocType: repositoryserver.LocTypeAzure, skipSSLVerify: "true", expectedSkipSSLVerifyValue: true, }, } { - loc[typeKey] = []byte(tc.LocType) - loc[skipSSLVerifyKey] = []byte(tc.skipSSLVerify) - c.Assert(getBucketNameFromMap(loc), check.Equals, string(loc[bucketKey])) - c.Assert(getEndpointFromMap(loc), check.Equals, string(loc[endpointKey])) - c.Assert(getPrefixFromMap(loc), check.Equals, string(loc[prefixKey])) - c.Assert(getRegionFromMap(loc), check.Equals, string(loc[regionKey])) + loc[repositoryserver.TypeKey] = []byte(tc.LocType) + loc[repositoryserver.SkipSSLVerifyKey] = []byte(tc.skipSSLVerify) + c.Assert(getBucketNameFromMap(loc), check.Equals, string(loc[repositoryserver.BucketKey])) + c.Assert(getEndpointFromMap(loc), check.Equals, string(loc[repositoryserver.EndpointKey])) + c.Assert(getPrefixFromMap(loc), check.Equals, string(loc[repositoryserver.PrefixKey])) + c.Assert(getRegionFromMap(loc), check.Equals, string(loc[repositoryserver.RegionKey])) c.Assert(checkSkipSSLVerifyFromMap(loc), check.Equals, tc.expectedSkipSSLVerifyValue) c.Assert(locationType(loc), check.Equals, tc.expectedLocType) } @@ -195,7 +196,7 @@ func (s *StorageUtilsSuite) TestGetMapForLocationValues(c *check.C) { endpointValue := "test-endpoint" skipSSLVerifyValue := "true" for _, tc := range []struct { - locType LocType + locType repositoryserver.LocType prefix string region string bucket string @@ -204,85 +205,85 @@ func (s *StorageUtilsSuite) TestGetMapForLocationValues(c *check.C) { expectedOutput map[string][]byte }{ { - locType: LocTypeS3, + locType: repositoryserver.LocTypeS3, expectedOutput: map[string][]byte{ - typeKey: []byte(LocTypeS3), + repositoryserver.TypeKey: []byte(repositoryserver.LocTypeS3), }, }, { - locType: LocTypeS3, + locType: repositoryserver.LocTypeS3, prefix: prefixValue, expectedOutput: map[string][]byte{ - typeKey: []byte(LocTypeS3), - prefixKey: []byte(prefixValue), + repositoryserver.TypeKey: []byte(repositoryserver.LocTypeS3), + repositoryserver.PrefixKey: []byte(prefixValue), }, }, { - locType: LocTypeS3, + locType: repositoryserver.LocTypeS3, prefix: prefixValue, region: regionValue, expectedOutput: map[string][]byte{ - typeKey: []byte(LocTypeS3), - prefixKey: []byte(prefixValue), - regionKey: []byte(regionValue), + repositoryserver.TypeKey: []byte(repositoryserver.LocTypeS3), + repositoryserver.PrefixKey: []byte(prefixValue), + repositoryserver.RegionKey: []byte(regionValue), }, }, { - locType: LocTypeS3, + locType: repositoryserver.LocTypeS3, prefix: prefixValue, region: regionValue, bucket: bucketValue, expectedOutput: map[string][]byte{ - typeKey: []byte(LocTypeS3), - prefixKey: []byte(prefixValue), - regionKey: []byte(regionValue), - bucketKey: []byte(bucketValue), + repositoryserver.TypeKey: []byte(repositoryserver.LocTypeS3), + repositoryserver.PrefixKey: []byte(prefixValue), + repositoryserver.RegionKey: []byte(regionValue), + repositoryserver.BucketKey: []byte(bucketValue), }, }, { - locType: LocTypeS3, + locType: repositoryserver.LocTypeS3, prefix: prefixValue, region: regionValue, bucket: bucketValue, endpoint: endpointValue, expectedOutput: map[string][]byte{ - typeKey: []byte(LocTypeS3), - prefixKey: []byte(prefixValue), - regionKey: []byte(regionValue), - bucketKey: []byte(bucketValue), - endpointKey: []byte(endpointValue), + repositoryserver.TypeKey: []byte(repositoryserver.LocTypeS3), + repositoryserver.PrefixKey: []byte(prefixValue), + repositoryserver.RegionKey: []byte(regionValue), + repositoryserver.BucketKey: []byte(bucketValue), + repositoryserver.EndpointKey: []byte(endpointValue), }, }, { - locType: LocTypeS3, + locType: repositoryserver.LocTypeS3, prefix: prefixValue, region: regionValue, bucket: bucketValue, endpoint: endpointValue, skipSSLVerify: skipSSLVerifyValue, expectedOutput: map[string][]byte{ - typeKey: []byte(LocTypeS3), - prefixKey: []byte(prefixValue), - regionKey: []byte(regionValue), - bucketKey: []byte(bucketValue), - endpointKey: []byte(endpointValue), - skipSSLVerifyKey: []byte(skipSSLVerifyValue), + repositoryserver.TypeKey: []byte(repositoryserver.LocTypeS3), + repositoryserver.PrefixKey: []byte(prefixValue), + repositoryserver.RegionKey: []byte(regionValue), + repositoryserver.BucketKey: []byte(bucketValue), + repositoryserver.EndpointKey: []byte(endpointValue), + repositoryserver.SkipSSLVerifyKey: []byte(skipSSLVerifyValue), }, }, { - locType: LocType(v1alpha1.LocationTypeS3Compliant), + locType: repositoryserver.LocType(v1alpha1.LocationTypeS3Compliant), prefix: prefixValue, region: regionValue, bucket: bucketValue, endpoint: endpointValue, skipSSLVerify: skipSSLVerifyValue, expectedOutput: map[string][]byte{ - typeKey: []byte(LocTypeS3), - prefixKey: []byte(prefixValue), - regionKey: []byte(regionValue), - bucketKey: []byte(bucketValue), - endpointKey: []byte(endpointValue), - skipSSLVerifyKey: []byte(skipSSLVerifyValue), + repositoryserver.TypeKey: []byte(repositoryserver.LocTypeS3), + repositoryserver.PrefixKey: []byte(prefixValue), + repositoryserver.RegionKey: []byte(regionValue), + repositoryserver.BucketKey: []byte(bucketValue), + repositoryserver.EndpointKey: []byte(endpointValue), + repositoryserver.SkipSSLVerifyKey: []byte(skipSSLVerifyValue), }, }, } { diff --git a/pkg/kopia/command/storage/storage_args.go b/pkg/kopia/command/storage/storage_args.go index 5f624936abd..faf97762abd 100644 --- a/pkg/kopia/command/storage/storage_args.go +++ b/pkg/kopia/command/storage/storage_args.go @@ -18,6 +18,7 @@ import ( "fmt" "github.com/kanisterio/kanister/pkg/logsafe" + "github.com/kanisterio/kanister/pkg/secrets/repositoryserver" ) const ( @@ -35,13 +36,13 @@ type StorageCommandParams struct { func KopiaStorageArgs(params *StorageCommandParams) (logsafe.Cmd, error) { LocType := locationType(params.Location) switch locationType(params.Location) { - case LocTypeFilestore: + case repositoryserver.LocTypeFilestore: return filesystemArgs(params.Location, params.RepoPathPrefix), nil - case LocTypeS3: + case repositoryserver.LocTypeS3: return s3Args(params.Location, params.RepoPathPrefix), nil - case LocTypeGCS: + case repositoryserver.LocTypeGCS: return gcsArgs(params.Location, params.RepoPathPrefix), nil - case LocTypeAzure: + case repositoryserver.LocTypeAzure: return azureArgs(params.Location, params.RepoPathPrefix), nil default: return nil, fmt.Errorf("unsupported type for the location: %s", LocType) diff --git a/pkg/kopia/command/storage/storage_args_test.go b/pkg/kopia/command/storage/storage_args_test.go index bc6fa371fed..cd5835a8ad7 100644 --- a/pkg/kopia/command/storage/storage_args_test.go +++ b/pkg/kopia/command/storage/storage_args_test.go @@ -18,6 +18,8 @@ import ( "fmt" "gopkg.in/check.v1" + + "github.com/kanisterio/kanister/pkg/secrets/repositoryserver" ) func (s *StorageUtilsSuite) TestStorageArgsUtil(c *check.C) { @@ -29,11 +31,11 @@ func (s *StorageUtilsSuite) TestStorageArgsUtil(c *check.C) { { params: &StorageCommandParams{ Location: map[string][]byte{ - bucketKey: []byte("test-bucket"), - prefixKey: []byte("test-prefix"), - regionKey: []byte("test-region"), - skipSSLVerifyKey: []byte("true"), - typeKey: []byte("s3"), + repositoryserver.BucketKey: []byte("test-bucket"), + repositoryserver.PrefixKey: []byte("test-prefix"), + repositoryserver.RegionKey: []byte("test-region"), + repositoryserver.SkipSSLVerifyKey: []byte("true"), + repositoryserver.TypeKey: []byte("s3"), }, RepoPathPrefix: "dir/subdir/", }, @@ -48,8 +50,8 @@ func (s *StorageUtilsSuite) TestStorageArgsUtil(c *check.C) { { params: &StorageCommandParams{ Location: map[string][]byte{ - prefixKey: []byte("test-prefix"), - typeKey: []byte("filestore"), + repositoryserver.PrefixKey: []byte("test-prefix"), + repositoryserver.TypeKey: []byte("filestore"), }, RepoPathPrefix: "dir/subdir", }, @@ -62,9 +64,9 @@ func (s *StorageUtilsSuite) TestStorageArgsUtil(c *check.C) { { params: &StorageCommandParams{ Location: map[string][]byte{ - prefixKey: []byte("test-prefix"), - bucketKey: []byte("test-bucket"), - typeKey: []byte("gcs"), + repositoryserver.PrefixKey: []byte("test-prefix"), + repositoryserver.BucketKey: []byte("test-bucket"), + repositoryserver.TypeKey: []byte("gcs"), }, RepoPathPrefix: "dir/subdir", }, @@ -79,9 +81,9 @@ func (s *StorageUtilsSuite) TestStorageArgsUtil(c *check.C) { { params: &StorageCommandParams{ Location: map[string][]byte{ - bucketKey: []byte("test-bucket"), - prefixKey: []byte("test-prefix"), - typeKey: []byte("azure"), + repositoryserver.BucketKey: []byte("test-bucket"), + repositoryserver.PrefixKey: []byte("test-prefix"), + repositoryserver.TypeKey: []byte("azure"), }, RepoPathPrefix: "dir/subdir", }, @@ -95,7 +97,7 @@ func (s *StorageUtilsSuite) TestStorageArgsUtil(c *check.C) { { params: &StorageCommandParams{ Location: map[string][]byte{ - typeKey: []byte("random-type"), + repositoryserver.TypeKey: []byte("random-type"), }, }, Checker: check.NotNil, diff --git a/pkg/secrets/repositoryserver/const.go b/pkg/secrets/repositoryserver/const.go new file mode 100644 index 00000000000..d5a4b38373e --- /dev/null +++ b/pkg/secrets/repositoryserver/const.go @@ -0,0 +1,50 @@ +// Copyright 2023 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package repositoryserver + +import ( + corev1 "k8s.io/api/core/v1" +) + +type LocType string + +const ( + LocTypeS3 LocType = "s3" + LocTypeGCS LocType = "gcs" + LocTypeAzure LocType = "azure" + LocTypeFilestore LocType = "filestore" + + // Location represents the storage location secret type for kopia repository server + Location corev1.SecretType = "secrets.kanister.io/storage-location" + // RepositoryPasswordSecretType represents the kopia repository passowrd secret type + RepositoryPassword corev1.SecretType = "secrets.kanister.io/kopia-repository/password" + // RepositoryServerAdminCredentialsSecretType represents the kopia server admin credentials secret type + RepositoryServerAdminCredentials corev1.SecretType = "secrets.kanister.io/kopia-repository/serveradmin" + // LocationTypeKey represents the key used to define the location type in + // the kopia repository server location secret + LocationTypeKey string = "type" +) + +const ( + // Location secret keys + BucketKey = "bucket" + EndpointKey = "endpoint" + PrefixKey = "prefix" + RegionKey = "region" + SkipSSLVerifyKey = "skipSSLVerify" + TypeKey = "type" + // Kopia Repository password secret key + RepoPasswordKey = "repo-password" +) From 91208aed4f7625fb2d7c5b460bf76aa3f36f9df6 Mon Sep 17 00:00:00 2001 From: kale-amruta <41624751+kale-amruta@users.noreply.github.com> Date: Tue, 30 May 2023 14:40:44 +0530 Subject: [PATCH 6/7] fix generate CRDs script (#2079) --- build/generate_crds.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/generate_crds.sh b/build/generate_crds.sh index cf52f113b5c..7cc55b12ea1 100755 --- a/build/generate_crds.sh +++ b/build/generate_crds.sh @@ -27,5 +27,5 @@ CONTROLLER_GEN=${LOCALBIN}/controller-gen ## Tool Versions CONTROLLER_TOOLS_VERSION=${1} -test -s ${CONTROLLER_GEN} || GOBIN=${LOCALBIN} go install sigs.k8s.io/controller-tools/cmd/controller-gen@$(CONTROLLER_TOOLS_VERSION) +test -s ${CONTROLLER_GEN} || GOBIN=${LOCALBIN} go install sigs.k8s.io/controller-tools/cmd/controller-gen@${CONTROLLER_TOOLS_VERSION} ${CONTROLLER_GEN} crd webhook paths="github.com/kanisterio/kanister/pkg/apis/cr/v1alpha1" output:crd:artifacts:config=pkg/customresource From 3a0b3cd61184d4166347dcd577df388569a55136 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 31 May 2023 02:48:19 +0000 Subject: [PATCH 7/7] deps(go): bump github.com/sirupsen/logrus from 1.9.0 to 1.9.2 (#2076) Bumps [github.com/sirupsen/logrus](https://github.com/sirupsen/logrus) from 1.9.0 to 1.9.2. - [Release notes](https://github.com/sirupsen/logrus/releases) - [Changelog](https://github.com/sirupsen/logrus/blob/master/CHANGELOG.md) - [Commits](https://github.com/sirupsen/logrus/compare/v1.9.0...v1.9.2) --- updated-dependencies: - dependency-name: github.com/sirupsen/logrus dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 02ee5c27f1c..a4b5e574111 100644 --- a/go.mod +++ b/go.mod @@ -42,7 +42,7 @@ require ( github.com/openshift/client-go v0.0.0-20230324103026-3f1513df25e0 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.15.1 - github.com/sirupsen/logrus v1.9.0 + github.com/sirupsen/logrus v1.9.2 github.com/spf13/cobra v1.7.0 github.com/vmware/govmomi v0.30.4 go.uber.org/zap v1.24.0 diff --git a/go.sum b/go.sum index 6a1fc33099a..8060c7ad921 100644 --- a/go.sum +++ b/go.sum @@ -461,8 +461,8 @@ github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0= -github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= -github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/sirupsen/logrus v1.9.2 h1:oxx1eChJGI6Uks2ZC4W1zpLlVgqB8ner4EuQwV4Ik1Y= +github.com/sirupsen/logrus v1.9.2/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/softlayer/softlayer-go v0.0.0-20190615201252-ba6e7f295217 h1:MFHQI+AYM6otrSP+l3dLhE2DjrSr5HXfV4mt4M6pjPs= github.com/softlayer/softlayer-go v0.0.0-20190615201252-ba6e7f295217/go.mod h1:Cw4GTlQccdRGSEf6KiMju767x0NEHE0YIVPJSaXjlsw= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=