message: add service per pod functionality

diff:
@@ -36,6 +36,18 @@ spec:
size: 3
image: percona/percona-xtradb-cluster:8.0.22-13.1
autoRecovery: true
+# expose:
+# enabled: true
+# type: LoadBalancer
+# trafficPolicy: Local
+# loadBalancerSourceRanges:
+# - 10.0.0.0/8
+# annotations:
+# networking.gke.io/load-balancer-type: "Internal"
+# replicationChannels:
+# - name: pxc1_to_pxc2
+# isSource: true
+# secretName: my-cluster-name-replication
# schedulerName: mycustom-scheduler
# readinessDelaySec: 15
# livenessDelaySec: 600

diff:
@@ -46,9 +46,25 @@ type PerconaXtraDBClusterSpec struct {
type PXCSpec struct {
AutoRecovery *bool `json:"autoRecovery,omitempty"`
+ ReplicationChannels []ReplicationChannel `json:"replicationChannels,omitempty"`
+ Expose ServiceExpose `json:"expose,omitempty"`
*PodSpec
}
+type ServiceExpose struct {
+ Enabled bool `json:"enabled,omitempty"`
+ Type corev1.ServiceType `json:"type,omitempty"`
+ LoadBalancerSourceRanges []string `json:"loadBalancerSourceRanges,omitempty"`
+ Annotations map[string]string `json:"annotations,omitempty"`
+ TrafficPolicy corev1.ServiceExternalTrafficPolicyType `json:"trafficPolicy,omitempty"`
+}
+
+type ReplicationChannel struct {
+ Name string `json:"name,omitempty"`
+ IsSource bool `json:"isSource,omitempty"`
+ SecretName string `json:"secretName,omitempty"`
+}
+
type TLSSpec struct {
SANs []string `json:"SANs,omitempty"`
IssuerConf *cmmeta.ObjectReference `json:"issuerConf,omitempty"`
@@ -173,6 +189,15 @@ func (cr *PerconaXtraDBCluster) Validate() error {
return errors.New("pxc.Image can't be empty")
}
+ for _, v := range c.PXC.ReplicationChannels {
+ if v.Name == "" {
+ return errors.New("pxc.replicationChannels.Name can't be empty")
+ }
+ if v.SecretName == "" {
+ return errors.New("pxc.replicationChannels.SecretName can't be empty")
+ }
+ }
+
if c.PMM != nil && c.PMM.Enabled {
if c.PMM.Image == "" {
return errors.New("pmm.Image can't be empty")

diff:
@@ -353,6 +353,18 @@ func (r *ReconcilePerconaXtraDBCluster) Reconcile(request reconcile.Request) (re
}
}
+ if o.Spec.PXC.Expose.Enabled {
+ err = r.ensurePxcPodServices(o)
+ if err != nil {
+ return rr, errors.Wrap(err, "create replication services")
+ }
+ } else {
+ err = r.removePxcPodServices(o)
+ if err != nil {
+ return rr, errors.Wrap(err, "remove pxc pod services")
+ }
+ }
+
if o.Spec.HAProxy != nil && o.Spec.HAProxy.Enabled {
err = r.updatePod(statefulset.NewHAProxy(o), o.Spec.HAProxy, o, nil)
if err != nil {
@@ -1234,6 +1246,9 @@ func (r *ReconcilePerconaXtraDBCluster) createOrUpdate(obj runtime.Object) error
switch object := obj.(type) {
case *corev1.Service:
object.Spec.ClusterIP = oldObject.(*corev1.Service).Spec.ClusterIP
+ if object.Spec.Type == corev1.ServiceTypeLoadBalancer {
+ object.Spec.HealthCheckNodePort = oldObject.(*corev1.Service).Spec.HealthCheckNodePort
+ }
}
return r.client.Update(context.TODO(), obj)

diff:
+package pxc
+
+import (
+ "context"
+ "fmt"
+
+ api "github.com/percona/percona-xtradb-cluster-operator/pkg/apis/pxc/v1"
+ "github.com/pkg/errors"
+ corev1 "k8s.io/api/core/v1"
+ k8serrors "k8s.io/apimachinery/pkg/api/errors"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/apimachinery/pkg/labels"
+ "sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+func (r *ReconcilePerconaXtraDBCluster) ensurePxcPodServices(cr *api.PerconaXtraDBCluster) error {
+ if cr.Spec.Pause {
+ return nil
+ }
+
+ isBackupRunning, err := r.isBackupRunning(cr)
+ if err != nil {
+ return errors.Wrap(err, "failed to check if backup is running")
+ }
+
+ if isBackupRunning {
+ return nil
+ }
+
+ isRestoreRunning, err := r.isRestoreRunning(cr.Name, cr.Namespace)
+ if err != nil {
+ return errors.Wrap(err, "failed to check if restore is running")
+ }
+
+ if isRestoreRunning {
+ return nil
+ }
+
+ for i := 0; i < int(cr.Spec.PXC.Size); i++ {
+ svcName := fmt.Sprintf("%s-pxc-%d", cr.Name, i)
+ svc := NewExposedPXCService(svcName, cr)
+
+ err := setControllerReference(cr, svc, r.scheme)
+ if err != nil {
+ return errors.Wrap(err, "failed to set owner to external service")
+ }
+
+ err = r.createOrUpdate(svc)
+ if err != nil {
+ return errors.Wrap(err, "failed to ensure pxc service")
+ }
+ }
+ return r.removeOutdatedServices(cr)
+}
+
+func (r *ReconcilePerconaXtraDBCluster) removeOutdatedServices(cr *api.PerconaXtraDBCluster) error {
+ //needed for labels
+ svc := NewExposedPXCService("", cr)
+
+ svcNames := make(map[string]struct{}, cr.Spec.PXC.Size)
+ for i := 0; i < int(cr.Spec.PXC.Size); i++ {
+ svcNames[fmt.Sprintf("%s-pxc-%d", cr.Name, i)] = struct{}{}
+ }
+
+ svcList := &corev1.ServiceList{}
+ err := r.client.List(context.TODO(),
+ svcList,
+ &client.ListOptions{
+ Namespace: cr.Namespace,
+ LabelSelector: labels.SelectorFromSet(svc.Labels),
+ },
+ )
+
+ if err != nil {
+ return errors.Wrap(err, "failed to list external services")
+ }
+
+ for _, service := range svcList.Items {
+ if _, ok := svcNames[service.Name]; !ok {
+ err = r.client.Delete(context.TODO(), &service)
+ if err != nil {
+ return errors.Wrapf(err, "failed to delete service %s", service.Name)
+ }
+ }
+ }
+ return nil
+}
+
+func (r *ReconcilePerconaXtraDBCluster) removePxcPodServices(cr *api.PerconaXtraDBCluster) error {
+ if cr.Spec.Pause {
+ return nil
+ }
+
+ //needed for labels
+ svc := NewExposedPXCService("", cr)
+
+ svcList := &corev1.ServiceList{}
+ err := r.client.List(context.TODO(),
+ svcList,
+ &client.ListOptions{
+ Namespace: cr.Namespace,
+ LabelSelector: labels.SelectorFromSet(svc.Labels),
+ },
+ )
+ if k8serrors.IsNotFound(err) {
+ return nil
+ }
+
+ if err != nil {
+ return errors.Wrap(err, "failed to list external services")
+ }
+
+ for _, service := range svcList.Items {
+ err = r.client.Delete(context.TODO(), &service)
+ if err != nil {
+ return errors.Wrap(err, "failed to delete external service")
+ }
+ }
+ return nil
+}
+
+func NewExposedPXCService(svcName string, cr *api.PerconaXtraDBCluster) *corev1.Service {
+ svc := &corev1.Service{
+ TypeMeta: metav1.TypeMeta{
+ APIVersion: "v1",
+ Kind: "Service",
+ },
+ ObjectMeta: metav1.ObjectMeta{
+ Name: svcName,
+ Namespace: cr.Namespace,
+ Labels: map[string]string{
+ "app.kubernetes.io/name": "percona-xtradb-cluster",
+ "app.kubernetes.io/instance": cr.Name,
+ "app.kubernetes.io/component": "external-service",
+ },
+ Annotations: cr.Spec.PXC.Expose.Annotations,
+ },
+ Spec: corev1.ServiceSpec{
+ Ports: []corev1.ServicePort{
+ {
+ Port: 3306,
+ Name: "mysql",
+ },
+ },
+ LoadBalancerSourceRanges: cr.Spec.PXC.Expose.LoadBalancerSourceRanges,
+ Selector: map[string]string{
+ "statefulset.kubernetes.io/pod-name": svcName,
+ },
+ },
+ }
+
+ switch cr.Spec.PXC.Expose.Type {
+ case corev1.ServiceTypeNodePort:
+ svc.Spec.Type = corev1.ServiceTypeNodePort
+ svc.Spec.ExternalTrafficPolicy = "Local"
+ case corev1.ServiceTypeLoadBalancer:
+ svc.Spec.Type = corev1.ServiceTypeLoadBalancer
+ svc.Spec.ExternalTrafficPolicy = "Cluster"
+ default:
+ svc.Spec.Type = corev1.ServiceTypeClusterIP
+ }
+
+ return svc
+}

diff:
@@ -536,6 +536,10 @@ func (r *ReconcilePerconaXtraDBCluster) isBackupRunning(cr *api.PerconaXtraDBClu
}
for _, bcp := range bcpList.Items {
+ if bcp.Spec.PXCCluster != cr.Name {
+ continue
+ }
+
if bcp.Status.State == api.BackupRunning || bcp.Status.State == api.BackupStarting {
return true, nil
}
@@ -544,6 +548,30 @@ func (r *ReconcilePerconaXtraDBCluster) isBackupRunning(cr *api.PerconaXtraDBClu
return false, nil
}
+func (r *ReconcilePerconaXtraDBCluster) isRestoreRunning(clusterName, namespace string) (bool, error) {
+ restoreList := api.PerconaXtraDBClusterRestoreList{}
+
+ err := r.client.List(context.TODO(), &restoreList, &client.ListOptions{
+ Namespace: namespace,
+ })
+ if err != nil {
+ return false, errors.Wrap(err, "failed to get restore list")
+ }
+
+ for _, v := range restoreList.Items {
+ if v.Spec.PXCCluster != clusterName {
+ continue
+ }
+
+ switch v.Status.State {
+ case api.RestoreStarting, api.RestoreStopCluster, api.RestoreRestore,
+ api.RestoreStartCluster, api.RestorePITR:
+ return true, nil
+ }
+ }
+ return false, nil
+}
+
func (r *ReconcilePerconaXtraDBCluster) getConfigHash(cr *api.PerconaXtraDBCluster, sfs api.StatefulApp) string {
configString := cr.Spec.PXC.Configuration
if sfs.Labels()["app.kubernetes.io/component"] == "haproxy" {

original_message: K8SPXC-308: add service per pod functionality (#891)


message: get new sfs revision before upgrade

change_type: MODIFY

diff:
@@ -217,7 +217,19 @@ func (r *ReconcilePerconaXtraDBCluster) smartUpdate(sfs api.StatefulApp, cr *api
return nil
}
- if sfs.StatefulSet().Status.UpdatedReplicas >= sfs.StatefulSet().Status.Replicas {
+ // sleep to get new sfs revision
+ time.Sleep(time.Second)
+
+ currentSet := sfs.StatefulSet()
+ err := r.client.Get(context.TODO(), types.NamespacedName{
+ Name: currentSet.Name,
+ Namespace: currentSet.Namespace,
+ }, currentSet)
+ if err != nil {
+ return errors.Wrap(err, "failed to get current sfs")
+ }
+
+ if currentSet.Status.UpdatedReplicas >= currentSet.Status.Replicas {
return nil
}
@@ -235,7 +247,7 @@ func (r *ReconcilePerconaXtraDBCluster) smartUpdate(sfs api.StatefulApp, cr *api
return nil
}
- if sfs.StatefulSet().Status.ReadyReplicas < sfs.StatefulSet().Status.Replicas {
+ if currentSet.Status.ReadyReplicas < currentSet.Status.Replicas {
logger.Info("can't start/continue 'SmartUpdate': waiting for all replicas are ready")
return nil
}
@@ -244,7 +256,7 @@ func (r *ReconcilePerconaXtraDBCluster) smartUpdate(sfs api.StatefulApp, cr *api
if err := r.client.List(context.TODO(),
&list,
&client.ListOptions{
- Namespace: sfs.StatefulSet().Namespace,
+ Namespace: currentSet.Namespace,
LabelSelector: labels.SelectorFromSet(sfs.Labels()),
},
); err != nil {
@@ -257,7 +269,7 @@ func (r *ReconcilePerconaXtraDBCluster) smartUpdate(sfs api.StatefulApp, cr *api
}
for _, pod := range list.Items {
if pod.Status.PodIP == primary || pod.Name == primary {
- primary = fmt.Sprintf("%s.%s.%s", pod.Name, sfs.StatefulSet().Name, sfs.StatefulSet().Namespace)
+ primary = fmt.Sprintf("%s.%s.%s", pod.Name, currentSet.Name, currentSet.Namespace)
break
}
}
@@ -276,18 +288,18 @@ func (r *ReconcilePerconaXtraDBCluster) smartUpdate(sfs api.StatefulApp, cr *api
var primaryPod corev1.Pod
for _, pod := range list.Items {
pod := pod
- if strings.HasPrefix(primary, fmt.Sprintf("%s.%s.%s", pod.Name, sfs.StatefulSet().Name, sfs.StatefulSet().Namespace)) {
+ if strings.HasPrefix(primary, fmt.Sprintf("%s.%s.%s", pod.Name, currentSet.Name, currentSet.Namespace)) {
primaryPod = pod
} else {
logger.Info("apply changes to secondary pod", "pod name", pod.Name)
- if err := r.applyNWait(cr, sfs.StatefulSet(), &pod, waitLimit); err != nil {
+ if err := r.applyNWait(cr, currentSet, &pod, waitLimit); err != nil {
return errors.Wrap(err, "failed to apply changes")
}
}
}
logger.Info("apply changes to primary pod", "pod name", primaryPod.Name)
- if err := r.applyNWait(cr, sfs.StatefulSet(), &primaryPod, waitLimit); err != nil {
+ if err := r.applyNWait(cr, currentSet, &primaryPod, waitLimit); err != nil {
return errors.Wrap(err, "failed to apply changes")
}

license: Apache License 2.0

repo: percona/percona-xtradb-cluster-operator

original_message: get new sfs revision before upgrade


message: implement pxc replication

diff:
@@ -39,7 +39,8 @@ function mysql_root_exec() {
NODE_IP=$(hostname -I | awk ' { print $1 } ')
CLUSTER_NAME="$(hostname -f | cut -d'.' -f2)"
-SERVER_ID=${HOSTNAME/$CLUSTER_NAME-}
+SERVER_NUM=${HOSTNAME/$CLUSTER_NAME-/}
+SERVER_ID=${CLUSTER_HASH}${SERVER_NUM}
NODE_NAME=$(hostname -f)
NODE_PORT=3306
@@ -80,7 +81,7 @@ fi
grep -E -q "^[#]?wsrep_sst_donor" "$CFG" || sed '/^\[mysqld\]/a wsrep_sst_donor=\n' ${CFG} 1<>${CFG}
grep -E -q "^[#]?wsrep_node_incoming_address" "$CFG" || sed '/^\[mysqld\]/a wsrep_node_incoming_address=\n' ${CFG} 1<>${CFG}
grep -E -q "^[#]?wsrep_provider_options" "$CFG" || sed '/^\[mysqld\]/a wsrep_provider_options="pc.weight=10"\n' ${CFG} 1<>${CFG}
-sed -r "s|^[#]?server_id=.*$|server_id=1${SERVER_ID}|" ${CFG} 1<>${CFG}
+sed -r "s|^[#]?server_id=.*$|server_id=${SERVER_ID}|" ${CFG} 1<>${CFG}
sed -r "s|^[#]?coredumper$|coredumper|" ${CFG} 1<>${CFG}
sed -r "s|^[#]?wsrep_node_address=.*$|wsrep_node_address=${NODE_IP}|" ${CFG} 1<>${CFG}
sed -r "s|^[#]?wsrep_cluster_name=.*$|wsrep_cluster_name=${CLUSTER_NAME}|" ${CFG} 1<>${CFG}

diff:
@@ -148,6 +148,7 @@ function join {
}
MYSQL_VERSION=$(mysqld -V | awk '{print $3}' | awk -F'.' '{print $1"."$2}')
+MYSQL_PATCH_VERSION=$(mysqld -V | awk '{print $3}' | awk -F'.' '{print $3}' | awk -F'-' '{print $1}')
# if vault secret file exists we assume we need to turn on encryption
vault_secret="/etc/mysql/vault-keyring-secret/keyring_vault.conf"
@@ -179,6 +180,13 @@ fi
grep -q "^progress=" $CFG && sed -i "s|^progress=.*|progress=1|" $CFG
grep -q "^\[sst\]" "$CFG" || printf '[sst]\n' >>"$CFG"
grep -q "^cpat=" "$CFG" || sed '/^\[sst\]/a cpat=.*\\.pem$\\|.*init\\.ok$\\|.*galera\\.cache$\\|.*wsrep_recovery_verbose\\.log$\\|.*readiness-check\\.sh$\\|.*liveness-check\\.sh$\\|.*sst_in_progress$\\|.*sst-xb-tmpdir$\\|.*\\.sst$\\|.*gvwstate\\.dat$\\|.*grastate\\.dat$\\|.*\\.err$\\|.*\\.log$\\|.*RPM_UPGRADE_MARKER$\\|.*RPM_UPGRADE_HISTORY$\\|.*pxc-entrypoint\\.sh$\\|.*unsafe-bootstrap\\.sh$\\|.*pxc-configure-pxc\\.sh\\|.*peer-list$' "$CFG" 1<>"$CFG"
+if [[ "$MYSQL_VERSION" == '8.0' ]]; then
+ if [[ $MYSQL_PATCH_VERSION -ge 26 ]]; then
+ grep -q "^skip_replica_start=ON" "$CFG" || sed -i "/\[mysqld\]/a skip_replica_start=ON" $CFG
+ else
+ grep -q "^skip_slave_start=ON" "$CFG" || sed -i "/\[mysqld\]/a skip_slave_start=ON" $CFG
+ fi
+fi
file_env 'XTRABACKUP_PASSWORD' 'xtrabackup' 'xtrabackup'
file_env 'CLUSTERCHECK_PASSWORD' '' 'clustercheck'
@@ -302,8 +310,13 @@ if [ -z "$CLUSTER_JOIN" ] && [ "$1" = 'mysqld' -a -z "$wantHelp" ]; then
fi
if [ -z "$MYSQL_INITDB_SKIP_TZINFO" ]; then
+ (
+ echo "set wsrep_on=0;"
+ echo "SET @@SESSION.SQL_LOG_BIN = off;"
# sed is for https://bugs.mysql.com/bug.php?id=20545
- mysql_tzinfo_to_sql /usr/share/zoneinfo | sed 's/Local time zone must be set--see zic manual page/FCTY/' | "${mysql[@]}" mysql
+ mysql_tzinfo_to_sql /usr/share/zoneinfo | sed 's/Local time zone must be set--see zic manual page/FCTY/'
+ echo "set wsrep_on=1;"
+ ) | "${mysql[@]}" mysql
fi
{ set +x; } 2>/dev/null

diff:
@@ -34,7 +34,7 @@ spec:
schedule: "0 4 * * *"
pxc:
size: 3
- image: percona/percona-xtradb-cluster:8.0.22-13.1
+ image: percona/percona-xtradb-cluster:8.0.23-14.1
autoRecovery: true
# expose:
# enabled: true
@@ -47,7 +47,12 @@ spec:
# replicationChannels:
# - name: pxc1_to_pxc2
# isSource: true
-# secretName: my-cluster-name-replication
+# - name: pxc2_to_pxc1
+# isSource: false
+# sourcesList:
+# - host: 10.95.251.101
+# port: 3306
+# weight: 100
# schedulerName: mycustom-scheduler
# readinessDelaySec: 15
# livenessDelaySec: 600

diff:
@@ -319,6 +319,7 @@ compare_kubectl() {
| yq d - '**.namespace' \
| yq d - '**.uid' \
| yq d - 'metadata.resourceVersion' \
+ | yq d - '**.env.(name==CLUSTER_HASH)' \
| yq d - 'metadata.selfLink' \
| yq d - 'metadata.deletionTimestamp' \
| yq d - 'metadata.annotations."k8s.v1.cni.cncf.io*"' \

diff:
@@ -62,7 +62,13 @@ type ServiceExpose struct {
type ReplicationChannel struct {
Name string `json:"name,omitempty"`
IsSource bool `json:"isSource,omitempty"`
- SecretName string `json:"secretName,omitempty"`
+ SourcesList []ReplicationSource `json:"sourcesList,omitempty"`
+}
+
+type ReplicationSource struct {
+ Host string `json:"host,omitempty"`
+ Port int `json:"port,omitempty"`
+ Weight int `json:"weight,omitempty"`
}
type TLSSpec struct {
@@ -194,12 +200,36 @@ func (cr *PerconaXtraDBCluster) Validate() error {
return errors.New("pxc.Image can't be empty")
}
- for _, v := range c.PXC.ReplicationChannels {
- if v.Name == "" {
- return errors.New("pxc.replicationChannels.Name can't be empty")
+ if len(c.PXC.ReplicationChannels) > 0 {
+ // since we do not allow multimaster
+ // isSource field should be equal everywhere
+ isSrc := c.PXC.ReplicationChannels[0].IsSource
+ for _, channel := range c.PXC.ReplicationChannels {
+ // this restrictions coming from mysql itself
+ if len(channel.Name) > 64 || channel.Name == "" || channel.Name == "group_replication_applier" || channel.Name == "group_replication_recovery" {
+ return errors.Errorf("invalid replication channel name %s, please see channel naming conventions", channel.Name)
+ }
+
+ if isSrc != channel.IsSource {
+ return errors.New("you can specify only one type of replication please specify equal values for isSource field")
+ }
+
+ if channel.IsSource {
+ continue
+ }
+
+ if len(channel.SourcesList) == 0 {
+ return errors.Errorf("sources list for replication channel %s should be empty, because it's replica", channel.Name)
+ }
+
+ for _, src := range channel.SourcesList {
+ if src.Weight == 0 {
+ src.Weight = 100
+ }
+ if src.Port == 0 {
+ src.Port = 3306
+ }
}
- if v.SecretName == "" {
- return errors.New("pxc.replicationChannels.SecretName can't be empty")
}
}

diff:
@@ -236,7 +236,7 @@ func (r *ReconcilePerconaXtraDBCluster) Reconcile(request reconcile.Request) (re
if o.CompareVersionWith("1.7.0") >= 0 && *o.Spec.PXC.AutoRecovery {
err = r.recoverFullClusterCrashIfNeeded(o)
if err != nil {
- reqLogger.Error(err, "Failed to check if cluster needs to recover")
+ reqLogger.Info("Failed to check if cluster needs to recover", "err", err.Error())
}
}
@@ -510,6 +510,13 @@ func (r *ReconcilePerconaXtraDBCluster) Reconcile(request reconcile.Request) (re
}
}
+ if o.CompareVersionWith("1.9.0") >= 0 {
+ err = r.reconcileReplication(o)
+ if err != nil {
+ reqLogger.Info("reconcile replication error", "err", err.Error())
+ }
+ }
+
err = r.reconcileBackups(o)
if err != nil {
return reconcile.Result{}, err

diff:
@@ -3,16 +3,26 @@ package pxc
import (
"context"
"fmt"
+ "strings"
+ "github.com/go-logr/logr"
+ "github.com/hashicorp/go-version"
api "github.com/percona/percona-xtradb-cluster-operator/pkg/apis/pxc/v1"
+ "github.com/percona/percona-xtradb-cluster-operator/pkg/pxc/app/statefulset"
+ "github.com/percona/percona-xtradb-cluster-operator/pkg/pxc/queries"
"github.com/pkg/errors"
corev1 "k8s.io/api/core/v1"
k8serrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
+ "k8s.io/apimachinery/pkg/types"
"sigs.k8s.io/controller-runtime/pkg/client"
)
+const replicationPodLabel = "percona.com/replicationPod"
+
+var minReplicationVersion = version.Must(version.NewVersion("8.0.23"))
+
func (r *ReconcilePerconaXtraDBCluster) ensurePxcPodServices(cr *api.PerconaXtraDBCluster) error {
if cr.Spec.Pause {
return nil
@@ -86,6 +96,325 @@ func (r *ReconcilePerconaXtraDBCluster) removeOutdatedServices(cr *api.PerconaXt
return nil
}
+func (r *ReconcilePerconaXtraDBCluster) reconcileReplication(cr *api.PerconaXtraDBCluster) error {
+ if cr.Status.PXC.Ready < 1 || cr.Spec.Pause {
+ return nil
+ }
+
+ logger := r.logger(cr.Name, cr.Namespace)
+
+ sfs := statefulset.NewNode(cr)
+
+ listRaw := corev1.PodList{}
+ err := r.client.List(context.TODO(),
+ &listRaw,
+ &client.ListOptions{
+ Namespace: cr.Namespace,
+ LabelSelector: labels.SelectorFromSet(sfs.Labels()),
+ },
+ )
+ if k8serrors.IsNotFound(err) {
+ return nil
+ } else if err != nil {
+ return errors.Wrap(err, "get pod list")
+ }
+
+ // we need only running pods, because we unable to
+ // connect to failed/pending pods
+ podList := make([]corev1.Pod, 0)
+ for _, pod := range listRaw.Items {
+ if isPodReady(pod) {
+ podList = append(podList, pod)
+ }
+ }
+
+ primary, err := r.getPrimaryPod(cr)
+ if err != nil {
+ return errors.Wrap(err, "get primary pxc pod")
+ }
+
+ var primaryPod *corev1.Pod
+ for _, pod := range podList {
+ if pod.Status.PodIP == primary || pod.Name == primary || strings.HasPrefix(primary, fmt.Sprintf("%s.%s.%s", pod.Name, sfs.StatefulSet().Name, cr.Namespace)) {
+ primaryPod = &pod
+ break
+ }
+ }
+
+ if primaryPod == nil {
+ logger.Info("Unable to find primary pod for replication. No pod with name or ip like this", "primary name", primary)
+ return nil
+ }
+
+ user := "operator"
+ port := int32(33062)
+
+ primaryDB, err := queries.New(r.client, cr.Namespace, internalPrefix+cr.Name, user, primaryPod.Name+"."+cr.Name+"-pxc."+cr.Namespace, port)
+ if err != nil {
+ return errors.Wrapf(err, "failed to connect to pod %s", primaryPod.Name)
+ }
+
+ defer primaryDB.Close()
+
+ dbVer, err := primaryDB.Version()
+ if err != nil {
+ return errors.Wrap(err, "failed to get current db version")
+ }
+
+ if minReplicationVersion.Compare(version.Must(version.NewVersion(dbVer))) < 0 {
+ logger.Info("Failed to reconcile replication. Unsupported mysql version", "minimum version", minReplicationVersion.String())
+ return nil
+ }
+
+ err = removeOutdatedChannels(primaryDB, cr.Spec.PXC.ReplicationChannels)
+ if err != nil {
+ return errors.Wrap(err, "remove outdated replication channels")
+ }
+
+ err = checkReadonlyStatus(cr.Spec.PXC.ReplicationChannels, podList, cr, r.client)
+ if err != nil {
+ return errors.Wrap(err, "failed to ensure cluster readonly status")
+ }
+
+ if len(cr.Spec.PXC.ReplicationChannels) == 0 {
+ return deleteReplicaLabels(r.client, podList)
+ }
+
+ if cr.Spec.PXC.ReplicationChannels[0].IsSource {
+ return deleteReplicaLabels(r.client, podList)
+ }
+
+ // if primary pod is not a replica, we need to make it as replica, and stop replication on other pods
+ _, ok := primaryPod.Labels[replicationPodLabel]
+ if !ok {
+ for _, pod := range podList {
+ if pod.Name == primaryPod.Name {
+ continue
+ }
+ if _, ok := pod.Labels[replicationPodLabel]; ok {
+ r.logger(cr.Name, cr.Namespace).Info("Replication pod has changed", "new replication pod", pod.Name)
+ db, err := queries.New(r.client, cr.Namespace, internalPrefix+cr.Name, user, pod.Name+"."+cr.Name+"-pxc."+cr.Namespace, port)
+ if err != nil {
+ return errors.Wrapf(err, "failed to connect to pod %s", pod.Name)
+ }
+ err = db.StopAllReplication()
+ db.Close()
+ if err != nil {
+ return errors.Wrapf(err, "stop replication on pod %s", pod.Name)
+ }
+ delete(pod.Labels, replicationPodLabel)
+ err = r.client.Update(context.TODO(), &pod)
+ if err != nil {
+ return errors.Wrap(err, "failed to remove primary label from secondary pod")
+ }
+ }
+ }
+ primaryPod.Labels[replicationPodLabel] = "true"
+ err = r.client.Update(context.TODO(), primaryPod)
+ if err != nil {
+ return errors.Wrap(err, "add label to main replica pod")
+ }
+ }
+
+ sysUsersSecretObj := corev1.Secret{}
+ err = r.client.Get(context.TODO(),
+ types.NamespacedName{
+ Namespace: cr.Namespace,
+ Name: internalPrefix + cr.Name,
+ },
+ &sysUsersSecretObj,
+ )
+ if err != nil {
+ return errors.Wrap(err, "get secrets")
+ }
+
+ for _, channels := range cr.Spec.PXC.ReplicationChannels {
+ if channels.IsSource {
+ continue
+ }
+ err = manageReplicationChannel(r.log, primaryDB, channels, string(sysUsersSecretObj.Data["replication"]))
+ if err != nil {
+ return errors.Wrapf(err, "manage replication channel %s", channels.Name)
+ }
+ }
+
+ return nil
+}
+
+func checkReadonlyStatus(channels []api.ReplicationChannel, pods []corev1.Pod, cr *api.PerconaXtraDBCluster, client client.Client) error {
+ isReplica := false
+ if len(channels) > 0 {
+ isReplica = !channels[0].IsSource
+ }
+
+ for _, pod := range pods {
+ db, err := queries.New(client, cr.Namespace, internalPrefix+cr.Name, "operator", pod.Name+"."+cr.Name+"-pxc."+cr.Namespace, 33062)
+ if err != nil {
+ return errors.Wrapf(err, "connect to pod %s", pod.Name)
+ }
+ defer db.Close()
+ readonly, err := db.IsReadonly()
+ if err != nil {
+ return errors.Wrap(err, "check readonly status")
+ }
+
+ if isReplica && readonly || (!isReplica && !readonly) {
+ continue
+ }
+
+ if isReplica && !readonly {
+ err = db.EnableReadonly()
+ }
+
+ if !isReplica && readonly {
+ err = db.DisableReadonly()
+ }
+ if err != nil {
+ return errors.Wrap(err, "enable or disable readonly mode")
+ }
+
+ }
+ return nil
+}
+
+func removeOutdatedChannels(db queries.Database, currentChannels []api.ReplicationChannel) error {
+ dbChannels, err := db.CurrentReplicationChannels()
+ if err != nil {
+ return errors.Wrap(err, "get current replication channels")
+ }
+
+ if len(dbChannels) == 0 {
+ return nil
+ }
+
+ toRemove := make(map[string]struct{})
+ for _, v := range dbChannels {
+ toRemove[v] = struct{}{}
+ }
+
+ for _, v := range currentChannels {
+ if !v.IsSource {
+ delete(toRemove, v.Name)
+ }
+ }
+
+ if len(toRemove) == 0 {
+ return nil
+ }
+
+ for channelToRemove := range toRemove {
+ err = db.StopReplication(channelToRemove)
+ if err != nil {
+ return errors.Wrapf(err, "stop replication for channel %s", channelToRemove)
+ }
+
+ srcList, err := db.ReplicationChannelSources(channelToRemove)
+ if err != nil && err != queries.ErrNotFound {
+ return errors.Wrapf(err, "get src list for outdated channel %s", channelToRemove)
+ }
+ for _, v := range srcList {
+ err = db.DeleteReplicationSource(channelToRemove, v.Host, v.Port)
+ if err != nil {
+ return errors.Wrapf(err, "delete replication source for outdated channel %s", channelToRemove)
+ }
+ }
+ }
+ return nil
+}
+
+func manageReplicationChannel(log logr.Logger, primaryDB queries.Database, channel api.ReplicationChannel, replicaPW string) error {
+ currentSources, err := primaryDB.ReplicationChannelSources(channel.Name)
+ if err != nil && err != queries.ErrNotFound {
+ return errors.Wrapf(err, "get current replication sources for channel %s", channel.Name)
+ }
+
+ replicationStatus, err := primaryDB.ReplicationStatus(channel.Name)
+ if err != nil {
+ return errors.Wrap(err, "failed to check replication status")
+ }
+
+ if !isSourcesChanged(channel.SourcesList, currentSources) {
+ if replicationStatus == queries.ReplicationStatusError {
+ log.Info("Replication for channel is not running. Please, check the replication status", "channel", channel.Name)
+ return nil
+ }
+
+ if replicationStatus == queries.ReplicationStatusActive {
+ return nil
+ }
+ }
+
+ if replicationStatus == queries.ReplicationStatusActive {
+ err = primaryDB.StopReplication(channel.Name)
+ if err != nil {
+ return errors.Wrapf(err, "stop replication for channel %s", channel.Name)
+ }
+ }
+
+ for _, src := range currentSources {
+ err = primaryDB.DeleteReplicationSource(channel.Name, src.Host, src.Port)
+ if err != nil {
+ return errors.Wrapf(err, "delete replication source for channel %s", channel.Name)
+ }
+ }
+
+ maxWeight := 0
+ maxWeightSrc := channel.SourcesList[0]
+
+ for _, src := range channel.SourcesList {
+ if src.Weight > maxWeight {
+ maxWeightSrc = src
+ }
+ err := primaryDB.AddReplicationSource(channel.Name, src.Host, src.Port, src.Weight)
+ if err != nil {
+ return errors.Wrapf(err, "add replication source for channel %s", channel.Name)
+ }
+ }
+
+ return primaryDB.StartReplication(replicaPW, queries.ReplicationChannelSource{
+ Name: channel.Name,
+ Host: maxWeightSrc.Host,
+ Port: maxWeightSrc.Port,
+ })
+}
+
+func isSourcesChanged(new []api.ReplicationSource, old []queries.ReplicationChannelSource) bool {
+ if len(new) != len(old) {
+ return true
+ }
+
+ oldSrc := make(map[string]queries.ReplicationChannelSource)
+ for _, src := range old {
+ oldSrc[src.Host] = src
+ }
+
+ for _, v := range new {
+ oldSource, ok := oldSrc[v.Host]
+ if !ok {
+ return true
+ }
+ if oldSource.Port != v.Port || oldSource.Weight != v.Weight {
+ return true
+ }
+ delete(oldSrc, v.Host)
+ }
+
+ return len(oldSrc) != 0
+}
+
+func deleteReplicaLabels(client client.Client, pods []corev1.Pod) error {
+ for _, pod := range pods {
+ if _, ok := pod.Labels[replicationPodLabel]; ok {
+ delete(pod.Labels, replicationPodLabel)
+ err := client.Update(context.TODO(), &pod)
+ if err != nil {
+ return errors.Wrap(err, "failed to remove replication label from pod")
+ }
+ }
+ }
+ return nil
+}
+
func (r *ReconcilePerconaXtraDBCluster) removePxcPodServices(cr *api.PerconaXtraDBCluster) error {
if cr.Spec.Pause {
return nil
@@ -162,3 +491,16 @@ func NewExposedPXCService(svcName string, cr *api.PerconaXtraDBCluster) *corev1.
return svc
}
+
+// isPodReady returns a boolean reflecting if a pod is in a "ready" state
+func isPodReady(pod corev1.Pod) bool {
+ for _, condition := range pod.Status.Conditions {
+ if condition.Status != corev1.ConditionTrue {
+ continue
+ }
+ if condition.Type == corev1.PodReady {
+ return true
+ }
+ }
+ return false
+}

diff:
@@ -5,6 +5,7 @@ import (
"crypto/md5"
"fmt"
"sort"
+ "strconv"
"strings"
"time"
@@ -13,9 +14,6 @@ import (
"github.com/pkg/errors"
- "strconv"
-
- "github.com/percona/percona-xtradb-cluster-operator/pkg/apis/pxc/v1"
api "github.com/percona/percona-xtradb-cluster-operator/pkg/apis/pxc/v1"
"github.com/percona/percona-xtradb-cluster-operator/pkg/pxc"
"github.com/percona/percona-xtradb-cluster-operator/pkg/pxc/queries"
@@ -203,7 +201,7 @@ func (r *ReconcilePerconaXtraDBCluster) updatePod(sfs api.StatefulApp, podSpec *
return errors.Wrap(err, "update error")
}
- if cr.Spec.UpdateStrategy != v1.SmartUpdateStatefulSetStrategyType {
+ if cr.Spec.UpdateStrategy != api.SmartUpdateStatefulSetStrategyType {
return nil
}

diff:
@@ -266,14 +266,16 @@ func (r *ReconcilePerconaXtraDBCluster) fetchVersionFromPXC(cr *apiv1.PerconaXtr
user := "root"
port := int32(3306)
+ secrets := cr.Spec.SecretsName
if cr.CompareVersionWith("1.6.0") >= 0 {
port = int32(33062)
+ secrets = "internal-" + cr.Name
}
logger := r.logger(cr.Name, cr.Namespace)
for _, pod := range list.Items {
- database, err := queries.New(r.client, cr.Namespace, cr.Spec.SecretsName, user, pod.Name+"."+cr.Name+"-pxc."+cr.Namespace, port)
+ database, err := queries.New(r.client, cr.Namespace, secrets, user, pod.Name+"."+cr.Name+"-pxc."+cr.Namespace, port)
if err != nil {
logger.Error(err, "failed to create db instance")
continue

diff:
@@ -2,6 +2,7 @@ package statefulset
import (
"fmt"
+ "hash/fnv"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
@@ -189,6 +190,21 @@ func (c *Node) AppContainer(spec *api.PodSpec, secrets string, cr *api.PerconaXt
Optional: &fvar,
},
})
+ serverIDHash := fnv.New32()
+ serverIDHash.Write([]byte(string(cr.UID)))
+
+ // we cut first 3 symbols to give a space for hostname(actially, pod number)
+ // which is appended to all server ids. If we do not do this, it
+ // can cause a int32 overflow
+ // P.S max value is 4294967295
+ serverIDHashStr := fmt.Sprint(serverIDHash.Sum32())
+ if len(serverIDHashStr) > 7 {
+ serverIDHashStr = serverIDHashStr[:7]
+ }
+ appc.Env = append(appc.Env, corev1.EnvVar{
+ Name: "CLUSTER_HASH",
+ Value: serverIDHashStr,
+ })
}
if cr.CompareVersionWith("1.3.0") >= 0 {
for k, v := range appc.VolumeMounts {

diff:
@@ -3,8 +3,10 @@ package queries
import (
"context"
"database/sql"
- "errors"
"fmt"
+ "strings"
+
+ "github.com/pkg/errors"
_ "github.com/go-sql-driver/mysql"
corev1 "k8s.io/api/core/v1"
@@ -12,6 +14,14 @@ import (
"sigs.k8s.io/controller-runtime/pkg/client"
)
+type ReplicationStatus int8
+
+const (
+ ReplicationStatusActive ReplicationStatus = iota
+ ReplicationStatusError
+ ReplicationStatusNotInitiated
+)
+
// value of writer group is hardcoded in ProxySQL config inside docker image
// https://github.com/percona/percona-docker/blob/pxc-operator-1.3.0/proxysql/dockerdir/etc/proxysql-admin.cnf#L23
const writerID = 11
@@ -20,6 +30,13 @@ type Database struct {
db *sql.DB
}
+type ReplicationChannelSource struct {
+ Name string
+ Host string
+ Port int
+ Weight int
+}
+
var ErrNotFound = errors.New("not found")
func New(client client.Client, namespace, secretName, user, host string, port int32) (Database, error) {
@@ -52,8 +69,156 @@ func New(client client.Client, namespace, secretName, user, host string, port in
}, nil
}
+func (p *Database) CurrentReplicationChannels() ([]string, error) {
+ rows, err := p.db.Query(`SELECT DISTINCT(Channel_name) from replication_asynchronous_connection_failover`)
+ if err != nil {
+ if errors.Is(err, sql.ErrNoRows) {
+ return nil, nil
+ }
+ return nil, errors.Wrap(err, "select current replication channels")
+ }
+
+ defer rows.Close()
+
+ result := make([]string, 0)
+ for rows.Next() {
+ src := ""
+ err = rows.Scan(&src)
+ if err != nil {
+ return nil, errors.Wrap(err, "scan channel name")
+ }
+ result = append(result, src)
+ }
+ return result, nil
+}
+
+func (p *Database) ReplicationStatus(channel string) (ReplicationStatus, error) {
+ rows, err := p.db.Query(`SHOW REPLICA STATUS FOR CHANNEL ?`, channel)
+ if err != nil {
+ if strings.HasSuffix(err.Error(), "does not exist.") || errors.Is(err, sql.ErrNoRows) {
+ return ReplicationStatusNotInitiated, nil
+ }
+ return ReplicationStatusError, errors.Wrap(err, "get current replica status")
+ }
+
+ defer rows.Close()
+ cols, err := rows.Columns()
+ if err != nil {
+ return ReplicationStatusError, errors.Wrap(err, "get columns")
+ }
+ vals := make([]interface{}, len(cols))
+ for i := range cols {
+ vals[i] = new(sql.RawBytes)
+ }
+
+ for rows.Next() {
+ err = rows.Scan(vals...)
+ if err != nil {
+ return ReplicationStatusError, errors.Wrap(err, "scan replication status")
+ }
+ }
+
+ IORunning := string(*vals[10].(*sql.RawBytes))
+ SQLRunning := string(*vals[11].(*sql.RawBytes))
+ LastErrNo := string(*vals[18].(*sql.RawBytes))
+ if IORunning == "Yes" && SQLRunning == "Yes" {
+ return ReplicationStatusActive, nil
+ }
+
+ if IORunning == "No" && SQLRunning == "No" && LastErrNo == "0" {
+ return ReplicationStatusNotInitiated, nil
+ }
+
+ return ReplicationStatusError, nil
+}
+
+func (p *Database) StopAllReplication() error {
+ _, err := p.db.Exec("STOP REPLICA")
+ return errors.Wrap(err, "failed to stop replication")
+}
+
+func (p *Database) AddReplicationSource(name, host string, port, weight int) error {
+ _, err := p.db.Exec("SELECT asynchronous_connection_failover_add_source(?, ?, ?, null, ?)", name, host, port, weight)
+ return errors.Wrap(err, "add replication source")
+}
+
+func (p *Database) ReplicationChannelSources(channelName string) ([]ReplicationChannelSource, error) {
+ rows, err := p.db.Query(`
+ SELECT host,
+ port,
+ weight
+ FROM replication_asynchronous_connection_failover
+ WHERE channel_name = ?
+ `, channelName)
+ if err != nil {
+ if errors.Is(err, sql.ErrNoRows) {
+ return nil, ErrNotFound
+ }
+ return nil, errors.Wrap(err, "get replication channels")
+ }
+ defer rows.Close()
+ result := make([]ReplicationChannelSource, 0)
+ for rows.Next() {
+ r := ReplicationChannelSource{}
+ err = rows.Scan(&r.Host, &r.Port, &r.Weight)
+ if err != nil {
+ return nil, errors.Wrap(err, "read replication channel info")
+ }
+ result = append(result, r)
+ }
+ return result, nil
+}
+
+func (p *Database) StopReplication(name string) error {
+ _, err := p.db.Exec("STOP REPLICA FOR CHANNEL ?", name)
+ return errors.Wrap(err, "stop replication for channel "+name)
+}
+
+func (p *Database) EnableReadonly() error {
+ _, err := p.db.Exec("SET GLOBAL READ_ONLY=1")
+ return errors.Wrap(err, "set global read_only param to 1")
+}
+
+func (p *Database) DisableReadonly() error {
+ _, err := p.db.Exec("SET GLOBAL READ_ONLY=0")
+ return errors.Wrap(err, "set global read_only param to 0")
+}
+
+func (p *Database) IsReadonly() (bool, error) {
+ readonly := 0
+ err := p.db.QueryRow("select @@read_only").Scan(&readonly)
+ return readonly == 1, errors.Wrap(err, "select global read_only param")
+}
+
+func (p *Database) StartReplication(replicaPass string, src ReplicationChannelSource) error {
+ _, err := p.db.Exec(`
+ CHANGE REPLICATION SOURCE TO
+ master_user='replication',
+ master_password=?,
+ master_host=?,
+ master_port=?,
+ source_connection_auto_failover=1,
+ master_auto_position=1,
+ master_retry_count=3,
+ master_connect_retry=60
+ FOR CHANNEL ?
+`, replicaPass, src.Host, src.Port, src.Name)
+ if err != nil {
+ return errors.Wrap(err, "change source for channel "+src.Name)
+ }
+
+ _, err = p.db.Exec(`START REPLICA FOR CHANNEL ?`, src.Name)
+ return errors.Wrap(err, "start replica for source "+src.Name)
+
+}
+
+func (p *Database) DeleteReplicationSource(name, host string, port int) error {
+ _, err := p.db.Exec("SELECT asynchronous_connection_failover_delete_source(?, ?, ?, null)", name, host, port)
+ return errors.Wrap(err, "delete replication source "+name)
+}
+
func (p *Database) Status(host, ip string) ([]string, error) {
- rows, err := p.db.Query("select status from mysql_servers where hostname like ? or hostname = ?;", host+"%", ip)
+ rows, err := p.db.Query("SELECT status FROM mysql_servers WHERE hostname LIKE ? OR hostname = ?;", host+"%", ip)
if err != nil {
if errors.Is(err, sql.ErrNoRows) {
return nil, ErrNotFound

original_message:
K8SPXC-308: implement pxc replication (#941)

https://jira.percona.com/browse/K8SPXC-308


message: prooper new replica pod info message

diff:
@@ -185,14 +185,11 @@ func (r *ReconcilePerconaXtraDBCluster) reconcileReplication(cr *api.PerconaXtra
}
// if primary pod is not a replica, we need to make it as replica, and stop replication on other pods
- _, ok := primaryPod.Labels[replicationPodLabel]
- if !ok {
for _, pod := range podList {
if pod.Name == primaryPod.Name {
continue
}
if _, ok := pod.Labels[replicationPodLabel]; ok {
- r.logger(cr.Name, cr.Namespace).Info("Replication pod has changed", "new replication pod", pod.Name)
db, err := queries.New(r.client, cr.Namespace, internalPrefix+cr.Name, user, pod.Name+"."+cr.Name+"-pxc."+cr.Namespace, port)
if err != nil {
return errors.Wrapf(err, "failed to connect to pod %s", pod.Name)
@@ -209,11 +206,14 @@ func (r *ReconcilePerconaXtraDBCluster) reconcileReplication(cr *api.PerconaXtra
}
}
}
+
+ if _, ok := primaryPod.Labels[replicationPodLabel]; !ok {
primaryPod.Labels[replicationPodLabel] = "true"
err = r.client.Update(context.TODO(), primaryPod)
if err != nil {
return errors.Wrap(err, "add label to main replica pod")
}
+ r.logger(cr.Name, cr.Namespace).Info("Replication pod has changed", "new replication pod", primaryPod.Name)
}
sysUsersSecretObj := corev1.Secret{}

original_message: prooper new replica pod info message


message: fix version service current cr version

diff:
@@ -39,7 +39,7 @@ func (vs VersionServiceClient) GetExactVersion(cr *api.PerconaXtraDBCluster, end
KubeVersion: &vm.KubeVersion,
LogCollectorVersion: &vm.LogCollectorVersion,
NamespaceUID: new(string),
- OperatorVersion: vs.OpVersion,
+ OperatorVersion: cr.Spec.CRVersion,
Platform: &vm.Platform,
PmmVersion: &vm.PMMVersion,
Product: productName,

original_message: fix version service current cr version (#963)


message: fix mysql version comparing

diff:
@@ -21,7 +21,7 @@ import (
const replicationPodLabel = "percona.com/replicationPod"
-var minReplicationVersion = version.Must(version.NewVersion("8.0.23"))
+var minReplicationVersion = version.Must(version.NewVersion("8.0.23-14.1"))
func (r *ReconcilePerconaXtraDBCluster) ensurePxcPodServices(cr *api.PerconaXtraDBCluster) error {
if cr.Spec.Pause {
@@ -161,8 +161,7 @@ func (r *ReconcilePerconaXtraDBCluster) reconcileReplication(cr *api.PerconaXtra
return errors.Wrap(err, "failed to get current db version")
}
- if minReplicationVersion.Compare(version.Must(version.NewVersion(dbVer))) < 0 {
- logger.Info("Failed to reconcile replication. Unsupported mysql version", "minimum version", minReplicationVersion.String())
+ if version.Must(version.NewVersion(dbVer)).Compare(minReplicationVersion) < 0 {
return nil
}

original_message: fix mysql version comparing (#966)


message: fix setting array defaults

diff:
@@ -221,15 +221,6 @@ func (cr *PerconaXtraDBCluster) Validate() error {
if len(channel.SourcesList) == 0 {
return errors.Errorf("sources list for replication channel %s should be empty, because it's replica", channel.Name)
}
-
- for _, src := range channel.SourcesList {
- if src.Weight == 0 {
- src.Weight = 100
- }
- if src.Port == 0 {
- src.Port = 3306
- }
- }
}
}
@@ -582,6 +573,17 @@ func (cr *PerconaXtraDBCluster) CheckNSetDefaults(serverVersion *version.ServerV
c.PXC.SSLInternalSecretName = cr.Name + "-ssl-internal"
}
+ for chIdx, channel := range c.PXC.ReplicationChannels {
+ for srcIdx, src := range channel.SourcesList {
+ if src.Weight == 0 {
+ c.PXC.ReplicationChannels[chIdx].SourcesList[srcIdx].Weight = 100
+ }
+ if src.Port == 0 {
+ c.PXC.ReplicationChannels[chIdx].SourcesList[srcIdx].Port = 3306
+ }
+ }
+ }
+
setSafeDefaults(c, logger)
// Set maxUnavailable = 1 by default for PodDisruptionBudget-PXC.

original_message: fix setting array defaults (#976)


message: handle replication password change

diff:
@@ -24,7 +24,9 @@ func NewClient() (*Client, error) {
// Instantiate loader for kubeconfig file.
kubeconfig := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(
clientcmd.NewDefaultClientConfigLoadingRules(),
- &clientcmd.ConfigOverrides{},
+ &clientcmd.ConfigOverrides{
+ Timeout: "10s",
+ },
)
// Get a rest.Config from the kubeconfig file. This will be passed into all

diff:
@@ -274,12 +274,17 @@ func (r *ReconcilePerconaXtraDBCluster) Reconcile(request reconcile.Request) (re
if err != nil {
return reconcile.Result{}, errors.Wrap(err, "reconcile users secret")
}
- var pxcAnnotations, proxysqlAnnotations map[string]string
+
+ userReconcileResult := &ReconcileUsersResult{}
+
if o.CompareVersionWith("1.5.0") >= 0 {
- pxcAnnotations, proxysqlAnnotations, err = r.reconcileUsers(o)
+ urr, err := r.reconcileUsers(o)
if err != nil {
return rr, errors.Wrap(err, "reconcile users")
}
+ if urr != nil {
+ userReconcileResult = urr
+ }
}
r.resyncPXCUsersWithProxySQL(o)
@@ -335,7 +340,7 @@ func (r *ReconcilePerconaXtraDBCluster) Reconcile(request reconcile.Request) (re
}
pxcSet := statefulset.NewNode(o)
- pxc.MergeTemplateAnnotations(pxcSet.StatefulSet(), pxcAnnotations)
+ pxc.MergeTemplateAnnotations(pxcSet.StatefulSet(), userReconcileResult.pxcAnnotations)
err = r.updatePod(pxcSet, o.Spec.PXC.PodSpec, o, inits)
if err != nil {
return reconcile.Result{}, errors.Wrap(err, "pxc upgrade error")
@@ -428,7 +433,7 @@ func (r *ReconcilePerconaXtraDBCluster) Reconcile(request reconcile.Request) (re
}
proxysqlSet := statefulset.NewProxy(o)
- pxc.MergeTemplateAnnotations(proxysqlSet.StatefulSet(), proxysqlAnnotations)
+ pxc.MergeTemplateAnnotations(proxysqlSet.StatefulSet(), userReconcileResult.proxysqlAnnotations)
if o.Spec.ProxySQL != nil && o.Spec.ProxySQL.Enabled {
err = r.updatePod(proxysqlSet, o.Spec.ProxySQL, o, nil)
@@ -511,7 +516,7 @@ func (r *ReconcilePerconaXtraDBCluster) Reconcile(request reconcile.Request) (re
}
if o.CompareVersionWith("1.9.0") >= 0 {
- err = r.reconcileReplication(o)
+ err = r.reconcileReplication(o, userReconcileResult.updateReplicationPassword)
if err != nil {
reqLogger.Info("reconcile replication error", "err", err.Error())
}

diff:
@@ -96,7 +96,7 @@ func (r *ReconcilePerconaXtraDBCluster) removeOutdatedServices(cr *api.PerconaXt
return nil
}
-func (r *ReconcilePerconaXtraDBCluster) reconcileReplication(cr *api.PerconaXtraDBCluster) error {
+func (r *ReconcilePerconaXtraDBCluster) reconcileReplication(cr *api.PerconaXtraDBCluster, replicaPassUpdated bool) error {
if cr.Status.PXC.Ready < 1 || cr.Spec.Pause {
return nil
}
@@ -227,6 +227,13 @@ func (r *ReconcilePerconaXtraDBCluster) reconcileReplication(cr *api.PerconaXtra
return errors.Wrap(err, "get secrets")
}
+ if replicaPassUpdated {
+ err = handleReplicaPasswordChange(primaryDB, string(sysUsersSecretObj.Data["replication"]))
+ if err != nil {
+ return errors.Wrap(err, "failed to change replication password")
+ }
+ }
+
for _, channels := range cr.Spec.PXC.ReplicationChannels {
if channels.IsSource {
continue
@@ -240,6 +247,21 @@ func (r *ReconcilePerconaXtraDBCluster) reconcileReplication(cr *api.PerconaXtra
return nil
}
+func handleReplicaPasswordChange(db queries.Database, newPass string) error {
+ channels, err := db.CurrentReplicationChannels()
+ if err != nil {
+ return errors.Wrap(err, "get current replication channels")
+ }
+
+ for _, channel := range channels {
+ err := db.ChangeChannelPassword(channel, newPass)
+ if err != nil {
+ return errors.Wrapf(err, "change password for channel %s", channel)
+ }
+ }
+ return nil
+}
+
func checkReadonlyStatus(channels []api.ReplicationChannel, pods []corev1.Pod, cr *api.PerconaXtraDBCluster, client client.Client) error {
isReplica := false
if len(channels) > 0 {

diff:
@@ -319,57 +319,57 @@ func TestClusterStatus(t *testing.T) {
want: api.AppStateInit,
},
"PXC init": {
- status: api.PerconaXtraDBClusterStatus{PXC: api.AppStatus{Status: api.AppStateInit}},
+ status: api.PerconaXtraDBClusterStatus{PXC: api.AppStatus{ComponentStatus: api.ComponentStatus{Status: api.AppStateInit}}},
want: api.AppStateInit,
},
"PXC ready": {
- status: api.PerconaXtraDBClusterStatus{PXC: api.AppStatus{Status: api.AppStateReady}},
+ status: api.PerconaXtraDBClusterStatus{PXC: api.AppStatus{ComponentStatus: api.ComponentStatus{Status: api.AppStateReady}}},
want: api.AppStateReady,
},
"PXC stopping": {
- status: api.PerconaXtraDBClusterStatus{PXC: api.AppStatus{Status: api.AppStateStopping}},
+ status: api.PerconaXtraDBClusterStatus{PXC: api.AppStatus{ComponentStatus: api.ComponentStatus{Status: api.AppStateStopping}}},
want: api.AppStateStopping,
},
"PXC paused, HAProxy stopping": {
status: api.PerconaXtraDBClusterStatus{
- PXC: api.AppStatus{Status: api.AppStatePaused},
- HAProxy: api.AppStatus{Status: api.AppStateStopping},
+ PXC: api.AppStatus{ComponentStatus: api.ComponentStatus{Status: api.AppStatePaused}},
+ HAProxy: api.AppStatus{ComponentStatus: api.ComponentStatus{Status: api.AppStateStopping}},
},
want: api.AppStateStopping,
},
"PXC paused, ProxySQL stopping": {
status: api.PerconaXtraDBClusterStatus{
- PXC: api.AppStatus{Status: api.AppStatePaused},
- ProxySQL: api.AppStatus{Status: api.AppStateStopping},
+ PXC: api.AppStatus{ComponentStatus: api.ComponentStatus{Status: api.AppStatePaused}},
+ ProxySQL: api.AppStatus{ComponentStatus: api.ComponentStatus{Status: api.AppStateStopping}},
},
want: api.AppStateStopping,
},
"PXC paused, HAProxy paused": {
status: api.PerconaXtraDBClusterStatus{
- PXC: api.AppStatus{Status: api.AppStatePaused},
- HAProxy: api.AppStatus{Status: api.AppStatePaused},
+ PXC: api.AppStatus{ComponentStatus: api.ComponentStatus{Status: api.AppStatePaused}},
+ HAProxy: api.AppStatus{ComponentStatus: api.ComponentStatus{Status: api.AppStatePaused}},
},
want: api.AppStatePaused,
},
"HAProxy init": {
- status: api.PerconaXtraDBClusterStatus{HAProxy: api.AppStatus{Status: api.AppStateInit}},
+ status: api.PerconaXtraDBClusterStatus{HAProxy: api.AppStatus{ComponentStatus: api.ComponentStatus{Status: api.AppStateInit}}},
want: api.AppStateInit,
},
"HAProxy ready": {
status: api.PerconaXtraDBClusterStatus{
- PXC: api.AppStatus{Status: api.AppStateReady},
- HAProxy: api.AppStatus{Status: api.AppStateReady},
+ PXC: api.AppStatus{ComponentStatus: api.ComponentStatus{Status: api.AppStateReady}},
+ HAProxy: api.AppStatus{ComponentStatus: api.ComponentStatus{Status: api.AppStateReady}},
},
want: api.AppStateReady,
},
"ProxySQL init": {
- status: api.PerconaXtraDBClusterStatus{ProxySQL: api.AppStatus{Status: api.AppStateInit}},
+ status: api.PerconaXtraDBClusterStatus{ProxySQL: api.AppStatus{ComponentStatus: api.ComponentStatus{Status: api.AppStateInit}}},
want: api.AppStateInit,
},
"ProxySQL ready": {
status: api.PerconaXtraDBClusterStatus{
- PXC: api.AppStatus{Status: api.AppStateReady},
- ProxySQL: api.AppStatus{Status: api.AppStateReady},
+ PXC: api.AppStatus{ComponentStatus: api.ComponentStatus{Status: api.AppStateReady}},
+ ProxySQL: api.AppStatus{ComponentStatus: api.ComponentStatus{Status: api.AppStateReady}},
},
want: api.AppStateReady,
},
@@ -377,7 +377,7 @@ func TestClusterStatus(t *testing.T) {
for name, test := range tests {
t.Run(name, func(tt *testing.T) {
- got := test.status.ClusterStatus(false)
+ got := test.status.ClusterStatus(false, false)
if got != test.want {
t.Errorf("AppState got %#v, want %#v", got, test.want)

diff:
@@ -19,9 +19,21 @@ import (
const internalPrefix = "internal-"
-func (r *ReconcilePerconaXtraDBCluster) reconcileUsers(cr *api.PerconaXtraDBCluster) (pxcAnnotations, proxysqlAnnotations map[string]string, err error) {
+type userUpdateRestart struct {
+ restartPXC bool
+ restartProxy bool
+ updateReplicationPass bool
+}
+
+type ReconcileUsersResult struct {
+ pxcAnnotations map[string]string
+ proxysqlAnnotations map[string]string
+ updateReplicationPassword bool
+}
+
+func (r *ReconcilePerconaXtraDBCluster) reconcileUsers(cr *api.PerconaXtraDBCluster) (*ReconcileUsersResult, error) {
sysUsersSecretObj := corev1.Secret{}
- err = r.client.Get(context.TODO(),
+ err := r.client.Get(context.TODO(),
types.NamespacedName{
Namespace: cr.Namespace,
Name: cr.Spec.SecretsName,
@@ -29,9 +41,9 @@ func (r *ReconcilePerconaXtraDBCluster) reconcileUsers(cr *api.PerconaXtraDBClus
&sysUsersSecretObj,
)
if err != nil && k8serrors.IsNotFound(err) {
- return nil, nil, nil
+ return nil, nil
} else if err != nil {
- return nil, nil, errors.Wrapf(err, "get sys users secret '%s'", cr.Spec.SecretsName)
+ return nil, errors.Wrapf(err, "get sys users secret '%s'", cr.Spec.SecretsName)
}
secretName := internalPrefix + cr.Name
@@ -46,7 +58,7 @@ func (r *ReconcilePerconaXtraDBCluster) reconcileUsers(cr *api.PerconaXtraDBClus
&internalSysSecretObj,
)
if err != nil && !k8serrors.IsNotFound(err) {
- return nil, nil, errors.Wrap(err, "get internal sys users secret")
+ return nil, errors.Wrap(err, "get internal sys users secret")
}
if k8serrors.IsNotFound(err) {
@@ -57,78 +69,80 @@ func (r *ReconcilePerconaXtraDBCluster) reconcileUsers(cr *api.PerconaXtraDBClus
}
err = r.client.Create(context.TODO(), internalSysUsersSecret)
if err != nil {
- return nil, nil, errors.Wrap(err, "create internal sys users secret")
+ return nil, errors.Wrap(err, "create internal sys users secret")
}
- return nil, nil, nil
+ return nil, nil
}
if cr.Status.PXC.Ready > 0 {
err := r.manageOperatorAdminUser(cr, &sysUsersSecretObj, &internalSysSecretObj)
if err != nil {
- return nil, nil, errors.Wrap(err, "manage operator admin user")
+ return nil, errors.Wrap(err, "manage operator admin user")
}
if cr.CompareVersionWith("1.6.0") >= 0 {
// monitor user need more grants for work in version more then 1.6.0
err = r.manageMonitorUser(cr, &internalSysSecretObj)
if err != nil {
- return nil, nil, errors.Wrap(err, "manage monitor user")
+ return nil, errors.Wrap(err, "manage monitor user")
}
}
if cr.CompareVersionWith("1.7.0") >= 0 {
// xtrabackup user need more grants for work in version more then 1.7.0
err = r.manageXtrabackupUser(cr, &internalSysSecretObj)
if err != nil {
- return nil, nil, errors.Wrap(err, "manage xtrabackup user")
+ return nil, errors.Wrap(err, "manage xtrabackup user")
}
}
if cr.CompareVersionWith("1.9.0") >= 0 {
err = r.manageReplicationUser(cr, &sysUsersSecretObj, &internalSysSecretObj)
if err != nil {
- return nil, nil, errors.Wrap(err, "manage replication user")
+ return nil, errors.Wrap(err, "manage replication user")
}
}
}
if cr.Status.Status != api.AppStateReady {
- return nil, nil, nil
+ return nil, nil
}
newSysData, err := json.Marshal(sysUsersSecretObj.Data)
if err != nil {
- return nil, nil, errors.Wrap(err, "marshal sys secret data")
+ return nil, errors.Wrap(err, "marshal sys secret data")
}
newSecretDataHash := sha256Hash(newSysData)
dataChanged, err := sysUsersSecretDataChanged(newSecretDataHash, &internalSysSecretObj)
if err != nil {
- return nil, nil, errors.Wrap(err, "check sys users data changes")
+ return nil, errors.Wrap(err, "check sys users data changes")
}
if !dataChanged {
- return nil, nil, nil
+ return nil, nil
}
- restartPXC, restartProxy, err := r.manageSysUsers(cr, &sysUsersSecretObj, &internalSysSecretObj)
+ restarts, err := r.manageSysUsers(cr, &sysUsersSecretObj, &internalSysSecretObj)
if err != nil {
- return nil, nil, errors.Wrap(err, "manage sys users")
+ return nil, errors.Wrap(err, "manage sys users")
}
internalSysSecretObj.Data = sysUsersSecretObj.Data
err = r.client.Update(context.TODO(), &internalSysSecretObj)
if err != nil {
- return nil, nil, errors.Wrap(err, "update internal sys users secret")
+ return nil, errors.Wrap(err, "update internal sys users secret")
}
- if restartProxy {
- proxysqlAnnotations = make(map[string]string)
- proxysqlAnnotations["last-applied-secret"] = newSecretDataHash
+ result := &ReconcileUsersResult{
+ updateReplicationPassword: restarts.updateReplicationPass,
}
- if restartPXC {
- pxcAnnotations = make(map[string]string)
- pxcAnnotations["last-applied-secret"] = newSecretDataHash
+
+ if restarts.restartProxy {
+ result.proxysqlAnnotations = map[string]string{"last-applied-secret": newSecretDataHash}
+ }
+ if restarts.restartPXC {
+ result.pxcAnnotations = map[string]string{"last-applied-secret": newSecretDataHash}
}
- return pxcAnnotations, proxysqlAnnotations, nil
+ return result, nil
}
func (r *ReconcilePerconaXtraDBCluster) manageMonitorUser(cr *api.PerconaXtraDBCluster, internalSysSecretObj *corev1.Secret) error {
@@ -223,7 +237,7 @@ func (r *ReconcilePerconaXtraDBCluster) manageXtrabackupUser(cr *api.PerconaXtra
return nil
}
-func (r *ReconcilePerconaXtraDBCluster) manageSysUsers(cr *api.PerconaXtraDBCluster, sysUsersSecretObj, internalSysSecretObj *corev1.Secret) (bool, bool, error) {
+func (r *ReconcilePerconaXtraDBCluster) manageSysUsers(cr *api.PerconaXtraDBCluster, sysUsersSecretObj, internalSysSecretObj *corev1.Secret) (*userUpdateRestart, error) {
type action int
const (
@@ -232,6 +246,7 @@ func (r *ReconcilePerconaXtraDBCluster) manageSysUsers(cr *api.PerconaXtraDBClus
rProxy
rProxyifPMM
syncProxyUsers
+ syncReplicaUser
)
type user struct {
@@ -277,6 +292,7 @@ func (r *ReconcilePerconaXtraDBCluster) manageSysUsers(cr *api.PerconaXtraDBClus
requiredUsers = append(requiredUsers, user{
name: "replication",
hosts: []string{"%"},
+ action: syncReplicaUser,
})
}
@@ -294,11 +310,14 @@ func (r *ReconcilePerconaXtraDBCluster) manageSysUsers(cr *api.PerconaXtraDBClus
})
}
- var sysUsers, proxyUsers []users.SysUser
- var todo action
+ var (
+ sysUsers, proxyUsers []users.SysUser
+ todo action
+ )
+
for _, user := range requiredUsers {
if len(sysUsersSecretObj.Data[user.name]) == 0 {
- return false, false, errors.New("undefined or not exist user " + user.name)
+ return nil, errors.New("undefined or not exist user " + user.name)
}
if bytes.Equal(sysUsersSecretObj.Data[user.name], internalSysSecretObj.Data[user.name]) {
@@ -327,8 +346,11 @@ func (r *ReconcilePerconaXtraDBCluster) manageSysUsers(cr *api.PerconaXtraDBClus
todo &^= rPXCifPMM | rProxyifPMM
}
- restartPXC := todo&(rPXC|rPXCifPMM) != 0
- restartProxy := todo&(rProxy|rProxyifPMM) != 0
+ res := &userUpdateRestart{
+ restartPXC: todo&(rPXC|rPXCifPMM) != 0,
+ restartProxy: todo&(rProxy|rProxyifPMM) != 0,
+ updateReplicationPass: todo&syncReplicaUser != 0,
+ }
pxcUser := "root"
pxcPass := string(internalSysSecretObj.Data["root"])
@@ -343,28 +365,28 @@ func (r *ReconcilePerconaXtraDBCluster) manageSysUsers(cr *api.PerconaXtraDBClus
}
um, err := users.NewManager(addr, pxcUser, pxcPass)
if err != nil {
- return false, false, errors.Wrap(err, "new users manager")
+ return res, errors.Wrap(err, "new users manager")
}
defer um.Close()
err = um.UpdateUsersPass(sysUsers)
if err != nil {
- return false, false, errors.Wrap(err, "update sys users pass")
+ return res, errors.Wrap(err, "update sys users pass")
}
if cr.Spec.ProxySQL != nil && cr.Spec.ProxySQL.Enabled {
err = r.updateProxyUsers(proxyUsers, internalSysSecretObj, cr)
if err != nil {
- return false, false, errors.Wrap(err, "update Proxy users pass")
+ return res, errors.Wrap(err, "update Proxy users pass")
}
}
- if todo&syncProxyUsers != 0 && !restartProxy {
+ if todo&syncProxyUsers != 0 && !res.restartProxy {
err = r.syncPXCUsersWithProxySQL(cr)
if err != nil {
- return false, false, errors.Wrap(err, "sync users")
+ return res, errors.Wrap(err, "sync users")
}
}
- return restartPXC, restartProxy, nil
+ return res, nil
}
func (r *ReconcilePerconaXtraDBCluster) syncPXCUsersWithProxySQL(cr *api.PerconaXtraDBCluster) error {

diff:
@@ -92,6 +92,38 @@ func (p *Database) CurrentReplicationChannels() ([]string, error) {
return result, nil
}
+func (p *Database) ChangeChannelPassword(channel, password string) error {
+ tx, err := p.db.Begin()
+ if err != nil {
+ return errors.Wrap(err, "start transaction for updating channel password")
+ }
+ _, err = tx.Exec(`STOP REPLICA IO_THREAD FOR CHANNEL ?`, channel)
+ if err != nil {
+ errT := tx.Rollback()
+ if errT != nil {
+ return errors.Wrapf(err, "rollback STOP REPLICA IO_THREAD FOR CHANNEL %s", channel)
+ }
+ return errors.Wrapf(err, "stop replication IO thread for channel %s", channel)
+ }
+ _, err = tx.Exec(`CHANGE REPLICATION SOURCE TO SOURCE_PASSWORD=? FOR CHANNEL ?`, password, channel)
+ if err != nil {
+ errT := tx.Rollback()
+ if errT != nil {
+ return errors.Wrapf(err, "rollback CHANGE SOURCE_PASSWORD FOR CHANNEL %s", channel)
+ }
+ return errors.Wrapf(err, "change master password for channel %s", channel)
+ }
+ _, err = tx.Exec(`START REPLICA IO_THREAD FOR CHANNEL ?`, channel)
+ if err != nil {
+ errT := tx.Rollback()
+ if errT != nil {
+ return errors.Wrapf(err, "rollback START REPLICA IO_THREAD FOR CHANNEL %s", channel)
+ }
+ return errors.Wrapf(err, "start io thread for channel %s", channel)
+ }
+ return tx.Commit()
+}
+
func (p *Database) ReplicationStatus(channel string) (ReplicationStatus, error) {
rows, err := p.db.Query(`SHOW REPLICA STATUS FOR CHANNEL ?`, channel)
if err != nil {
@@ -193,14 +225,14 @@ func (p *Database) IsReadonly() (bool, error) {
func (p *Database) StartReplication(replicaPass string, src ReplicationChannelSource) error {
_, err := p.db.Exec(`
CHANGE REPLICATION SOURCE TO
- master_user='replication',
- master_password=?,
- master_host=?,
- master_port=?,
- source_connection_auto_failover=1,
- master_auto_position=1,
- master_retry_count=3,
- master_connect_retry=60
+ SOURCE_USER='replication',
+ SOURCE_PASSWORD=?,
+ SOURCE_HOST=?,
+ SOURCE_PORT=?,
+ SOURCE_CONNECTION_AUTO_FAILOVER=1,
+ SOURCE_AUTO_POSITION=1,
+ SOURCE_RETRY_COUNT=3,
+ SOURCE_CONNECT_RETRY=60
FOR CHANNEL ?
`, replicaPass, src.Host, src.Port, src.Name)
if err != nil {

original_message: K8SPXC-851: handle replication password change (#981)


message:
delete backup from folder
* delete backup from folder
Before that change, we were undable to delete backup due to path
error. With this fix, we can easily delete backup from user's
folder, not just a bucket

diff:
@@ -338,12 +338,10 @@ func (r *ReconcilePerconaXtraDBClusterBackup) runS3BackupFinalizer(cr *api.Perco
spl := strings.Split(cr.Status.Destination, "/")
backup := spl[len(spl)-1]
- for _, bcp := range []string{backup + ".md5", backup + "sst_info", backup} {
- err = retry.OnError(retry.DefaultBackoff, func(e error) bool { return true }, removeBackup(cr.Status.S3.Bucket, bcp, s3cli))
+ err = retry.OnError(retry.DefaultBackoff, func(e error) bool { return true }, removeBackup(cr.Status.S3.Bucket, backup, s3cli))
if err != nil {
break
}
- }
} else {
finalizers = append(finalizers, f)
}
@@ -365,6 +363,13 @@ func (r *ReconcilePerconaXtraDBClusterBackup) runS3BackupFinalizer(cr *api.Perco
func removeBackup(bucket, backup string, s3cli *minio.Client) func() error {
return func() error {
+ // this is needed to understand if user provided some path
+ // on s3, instead of just a bucket name
+ bucketSplitted := strings.Split(bucket, "/")
+ if len(bucketSplitted) > 1 {
+ bucket = bucketSplitted[0]
+ backup = strings.Join(append(bucketSplitted[1:], backup), "/")
+ }
objs := s3cli.ListObjects(context.Background(), bucket,
minio.ListObjectsOptions{
Recursive: true,

original_message:
K8SPXC-842: delete backup from folder (#999)

* K8SPXC-842: delete backup from folder

Before that change, we were undable to delete backup due to path
error. With this fix, we can easily delete backup from user's
folder, not just a bucket


message:
allow to configure and change replication parameters
Allow tuning sourceConnectRetry and sourceRetryCount parameters
at live cluster

diff:
@@ -63,6 +63,12 @@ type ReplicationChannel struct {
Name string `json:"name,omitempty"`
IsSource bool `json:"isSource,omitempty"`
SourcesList []ReplicationSource `json:"sourcesList,omitempty"`
+ Config *ReplicationChannelConfig `json:"configuration,omitempty"`
+}
+
+type ReplicationChannelConfig struct {
+ SourceRetryCount uint `json:"sourceRetryCount,omitempty"`
+ SourceConnectRetry uint `json:"sourceConnectRetry,omitempty"`
}
type ReplicationSource struct {
@@ -124,6 +130,7 @@ const (
// PerconaXtraDBClusterStatus defines the observed state of PerconaXtraDBCluster
type PerconaXtraDBClusterStatus struct {
PXC AppStatus `json:"pxc,omitempty"`
+ PXCReplication *ReplicationStatus `json:"pxcReplication,omitempty"`
ProxySQL AppStatus `json:"proxysql,omitempty"`
HAProxy AppStatus `json:"haproxy,omitempty"`
Backup ComponentStatus `json:"backup,omitempty"`
@@ -138,6 +145,16 @@ type PerconaXtraDBClusterStatus struct {
Ready int32 `json:"ready"`
}
+// TODO: add replication status(error,active and etc)
+type ReplicationStatus struct {
+ Channels []ReplicationChannelStatus `json:"replicationChannels,omitempty"`
+}
+
+type ReplicationChannelStatus struct {
+ Name string `json:"name,omitempty"`
+ ReplicationChannelConfig
+}
+
type ConditionStatus string
const (
@@ -589,6 +606,12 @@ func (cr *PerconaXtraDBCluster) CheckNSetDefaults(serverVersion *version.ServerV
c.PXC.ReplicationChannels[chIdx].SourcesList[srcIdx].Port = 3306
}
}
+ if !channel.IsSource && channel.Config == nil {
+ c.PXC.ReplicationChannels[chIdx].Config = &ReplicationChannelConfig{
+ SourceRetryCount: 3,
+ SourceConnectRetry: 60,
+ }
+ }
}
setSafeDefaults(c, logger)

diff:
@@ -234,17 +234,21 @@ func (r *ReconcilePerconaXtraDBCluster) reconcileReplication(cr *api.PerconaXtra
}
}
- for _, channels := range cr.Spec.PXC.ReplicationChannels {
- if channels.IsSource {
+ for _, channel := range cr.Spec.PXC.ReplicationChannels {
+ if channel.IsSource {
continue
}
- err = manageReplicationChannel(r.log, primaryDB, channels, string(sysUsersSecretObj.Data["replication"]))
+
+ currConf := currentReplicaConfig(channel.Name, cr.Status.PXCReplication)
+
+ err = manageReplicationChannel(r.log, primaryDB, channel, currConf, string(sysUsersSecretObj.Data["replication"]))
if err != nil {
- return errors.Wrapf(err, "manage replication channel %s", channels.Name)
+ return errors.Wrapf(err, "manage replication channel %s", channel.Name)
}
+ setReplicationChannelStatus(cr, channel)
}
- return nil
+ return r.updateStatus(cr, nil)
}
func handleReplicaPasswordChange(db queries.Database, newPass string) error {
@@ -343,7 +347,7 @@ func removeOutdatedChannels(db queries.Database, currentChannels []api.Replicati
return nil
}
-func manageReplicationChannel(log logr.Logger, primaryDB queries.Database, channel api.ReplicationChannel, replicaPW string) error {
+func manageReplicationChannel(log logr.Logger, primaryDB queries.Database, channel api.ReplicationChannel, currConf api.ReplicationChannelConfig, replicaPW string) error {
currentSources, err := primaryDB.ReplicationChannelSources(channel.Name)
if err != nil && err != queries.ErrNotFound {
return errors.Wrapf(err, "get current replication sources for channel %s", channel.Name)
@@ -360,7 +364,8 @@ func manageReplicationChannel(log logr.Logger, primaryDB queries.Database, chann
return nil
}
- if replicationStatus == queries.ReplicationStatusActive {
+ if replicationStatus == queries.ReplicationStatusActive &&
+ *channel.Config == currConf {
return nil
}
}
@@ -392,10 +397,14 @@ func manageReplicationChannel(log logr.Logger, primaryDB queries.Database, chann
}
}
- return primaryDB.StartReplication(replicaPW, queries.ReplicationChannelSource{
+ return primaryDB.StartReplication(replicaPW, queries.ReplicationConfig{
+ Source: queries.ReplicationChannelSource{
Name: channel.Name,
Host: maxWeightSrc.Host,
Port: maxWeightSrc.Port,
+ },
+ SourceRetryCount: channel.Config.SourceRetryCount,
+ SourceConnectRetry: channel.Config.SourceConnectRetry,
})
}
@@ -525,3 +534,40 @@ func isPodReady(pod corev1.Pod) bool {
}
return false
}
+
+func currentReplicaConfig(name string, status *api.ReplicationStatus) api.ReplicationChannelConfig {
+ res := api.ReplicationChannelConfig{}
+ if status == nil {
+ return res
+ }
+
+ for _, v := range status.Channels {
+ if v.Name == name {
+ return v.ReplicationChannelConfig
+ }
+ }
+ return res
+}
+
+func setReplicationChannelStatus(cr *api.PerconaXtraDBCluster, channel api.ReplicationChannel) {
+ status := api.ReplicationChannelStatus{
+ Name: channel.Name,
+ ReplicationChannelConfig: *channel.Config,
+ }
+
+ if cr.Status.PXCReplication == nil {
+ cr.Status.PXCReplication = &api.ReplicationStatus{
+ Channels: []api.ReplicationChannelStatus{status},
+ }
+ return
+ }
+
+ for k, v := range cr.Status.PXCReplication.Channels {
+ if channel.Name == v.Name {
+ cr.Status.PXCReplication.Channels[k] = status
+ return
+ }
+ }
+
+ cr.Status.PXCReplication.Channels = append(cr.Status.PXCReplication.Channels, status)
+}

diff:
@@ -30,6 +30,12 @@ type Database struct {
db *sql.DB
}
+type ReplicationConfig struct {
+ Source ReplicationChannelSource
+ SourceRetryCount uint
+ SourceConnectRetry uint
+}
+
type ReplicationChannelSource struct {
Name string
Host string
@@ -222,7 +228,7 @@ func (p *Database) IsReadonly() (bool, error) {
return readonly == 1, errors.Wrap(err, "select global read_only param")
}
-func (p *Database) StartReplication(replicaPass string, src ReplicationChannelSource) error {
+func (p *Database) StartReplication(replicaPass string, config ReplicationConfig) error {
_, err := p.db.Exec(`
CHANGE REPLICATION SOURCE TO
SOURCE_USER='replication',
@@ -231,16 +237,16 @@ func (p *Database) StartReplication(replicaPass string, src ReplicationChannelSo
SOURCE_PORT=?,
SOURCE_CONNECTION_AUTO_FAILOVER=1,
SOURCE_AUTO_POSITION=1,
- SOURCE_RETRY_COUNT=3,
- SOURCE_CONNECT_RETRY=60
+ SOURCE_RETRY_COUNT=?,
+ SOURCE_CONNECT_RETRY=?
FOR CHANNEL ?
-`, replicaPass, src.Host, src.Port, src.Name)
+`, replicaPass, config.Source.Host, config.Source.Port, config.SourceRetryCount, config.SourceConnectRetry, config.Source.Name)
if err != nil {
- return errors.Wrap(err, "change source for channel "+src.Name)
+ return errors.Wrapf(err, "change source for channel %s", config.Source.Name)
}
- _, err = p.db.Exec(`START REPLICA FOR CHANNEL ?`, src.Name)
- return errors.Wrap(err, "start replica for source "+src.Name)
+ _, err = p.db.Exec(`START REPLICA FOR CHANNEL ?`, config.Source.Name)
+ return errors.Wrapf(err, "start replica for source %s", config.Source.Name)
}

original_message:
K8SPXC-789: allow to configure and change replication parameters  (#1002)

Allow tuning sourceConnectRetry and sourceRetryCount parameters
at live cluster


diff:
@@ -528,8 +528,11 @@ NS_INLINE CGRect RectShift(CGRect rect, CGFloat dx, CGFloat dy) {
// Get cell color.
UIColor *backgroundColor = _cellBackgroundColor;
if ([_delegate respondsToSelector:@selector(collectionView:cellBackgroundColorAtIndexPath:)]) {
- backgroundColor =
+ UIColor *customBackgroundColor =
[_delegate collectionView:_collectionView cellBackgroundColorAtIndexPath:attr.indexPath];
+ if (customBackgroundColor) {
+ backgroundColor = customBackgroundColor;
+ }
}
NSPointerArray *cellBackgroundCache = _cellBackgroundCaches[backgroundColor];

original_message: Fallback to default cell background color on nil. (#1630)


message:
[Typography] Match large font family on iOS 9+
* Use the same font family when loading fonts as iOS does by default.
Resolves
* Format test.
* Format test.
* Updated comment.

diff:
// Store the system font family name to ensure that we load the system font.
// If we do not explicitly include this UIFontDescriptorFamilyAttribute in the
- // FontDescriptor the OS will default to Helvetica.
- static NSString *systemFontFamilyName;
+ // FontDescriptor the OS will default to Helvetica. On iOS 9+, the Font Family
+ // changes from San Francisco to San Francisco Display at point size 20.
+ static NSString *smallSystemFontFamilyName;
+ static NSString *largeSystemFontFamilyName;
static dispatch_once_t onceToken;
dispatch_once(&onceToken, ^{
- UIFont *systemFont;
+ UIFont *smallSystemFont;
+ UIFont *largeSystemFont;
if ([UIFont respondsToSelector:@selector(systemFontOfSize:weight:)]) {
- systemFont = [UIFont systemFontOfSize:12 weight:UIFontWeightRegular];
+ smallSystemFont = [UIFont systemFontOfSize:12 weight:UIFontWeightRegular];
+ largeSystemFont = [UIFont systemFontOfSize:20 weight:UIFontWeightRegular];
} else {
// TODO: Remove this fallback once we are 8.2+
- systemFont = [UIFont systemFontOfSize:12];
+ smallSystemFont = [UIFont systemFontOfSize:12];
+ largeSystemFont = [UIFont systemFontOfSize:20];
}
- systemFontFamilyName = [systemFont.familyName copy];
+ smallSystemFontFamilyName = [smallSystemFont.familyName copy];
+ largeSystemFontFamilyName = [largeSystemFont.familyName copy];
});
NSDictionary *traits = @{ UIFontWeightTrait : @(materialTraits.weight) };
+ NSString *fontFamily =
+ materialTraits.pointSize < 19.5 ? smallSystemFontFamilyName : largeSystemFontFamilyName;
NSDictionary *attributes = @{
UIFontDescriptorSizeAttribute : @(materialTraits.pointSize),
UIFontDescriptorTraitsAttribute : traits,
- UIFontDescriptorFamilyAttribute : systemFontFamilyName
+ UIFontDescriptorFamilyAttribute : fontFamily
};
UIFontDescriptor *fontDescriptor = [[UIFontDescriptor alloc] initWithFontAttributes:attributes];

diff:
@@ -330,4 +330,32 @@ static const CGFloat kOpacityMedium = 0.87f;
// Cleanup
[MDCTypography setFontLoader:[[MDCSystemFontLoader alloc] init]];
}
+
+- (void)testFontFamilyMatchesSystemFontFamily {
+ // Given
+ NSArray<NSNumber *> *allFontStyles = @[
+ @(MDCFontTextStyleBody1),
+ @(MDCFontTextStyleBody2),
+ @(MDCFontTextStyleCaption),
+ @(MDCFontTextStyleHeadline),
+ @(MDCFontTextStyleSubheadline),
+ @(MDCFontTextStyleTitle),
+ @(MDCFontTextStyleDisplay1),
+ @(MDCFontTextStyleDisplay2),
+ @(MDCFontTextStyleDisplay3),
+ @(MDCFontTextStyleDisplay4),
+ @(MDCFontTextStyleButton),
+ ];
+
+ for (NSNumber *styleObject in allFontStyles) {
+ // When
+ MDCFontTextStyle style = styleObject.integerValue;
+ UIFont *mdcFont = [UIFont mdc_preferredFontForMaterialTextStyle:style];
+ UIFont *systemFont = [UIFont systemFontOfSize:mdcFont.pointSize weight:UIFontWeightRegular];
+
+ // Then
+ XCTAssertEqualObjects(systemFont.familyName, mdcFont.familyName);
+ }
+}
+
@end

original_message:
[Typography] Match large font family on iOS 9+ (#1880)

* Use the same font family when loading fonts as iOS does by default.

Resolves #1805.

* Format test.

* Format test.

* Updated comment.


diff:
@@ -207,6 +207,9 @@ static const CGFloat kSingleCycleRotation =
- (void)startAnimating {
if (_animatingOut) {
+ if ([_delegate respondsToSelector:@selector(activityIndicatorAnimationDidFinish:)]) {
+ [_delegate activityIndicatorAnimationDidFinish:self];
+ }
[self removeAnimations];
}
@@ -782,6 +785,11 @@ static const CGFloat kSingleCycleRotation =
[_strokeLayer removeAllAnimations];
[_outerRotationLayer removeAllAnimations];
+ // Reset current and latest progress, to ensure addProgressAnimationIfRequired adds a progress animation
+ // when returning from hidden.
+ _currentProgress = 0;
+ _lastProgress = 0;
+
// Reset cycle count to 0 rather than cycleStart to reflect default starting position (top).
_cycleCount = 0;
// However _animationInProgress represents the CATransaction that hasn't finished, so we leave it

original_message: Resolve issues caused by starting and stopping activitiy indicator quickly (#2053)


message:
[Shapes] Added missing copyright comments.
No functional code change.

diff:
+/*
+ Copyright 2017-present the Material Components for iOS authors. All Rights Reserved.
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ */
+
#import "MDCCornerTreatment.h"
#import "MDCPathGenerator.h"

diff:
+/*
+ Copyright 2017-present the Material Components for iOS authors. All Rights Reserved.
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ */
+
#import "MDCEdgeTreatment.h"
#import "MDCPathGenerator.h"

original_message:
[Shapes] Added missing copyright comments. (#2211)

No functional code change.


diff:
@@ -78,10 +78,10 @@ class AppBarImagerySwiftExample: UITableViewController {
super.init(coder: aDecoder)
}
- func headerBackgroundImage() -> UIImage {
- let bundle = Bundle(for: AppBarImagerySwiftExample.self)
- let imagePath = bundle.path(forResource: "mdc_theme", ofType: "png")!
- return UIImage(contentsOfFile: imagePath)!
+ func headerBackgroundImage() -> UIImage? {
+ return UIImage(named: "mdc_theme",
+ in: Bundle(for: AppBarImagerySwiftExample.self),
+ compatibleWith: nil)
}
}

original_message: [AppBarExample] Load image assets using UIImage(named:in:compatibleWith:) (#2233)


message:
[Catalog] Pad trailing edge inside.
The constraints for MDCNodeListViewController incorrectly made the padding ADD width to the contained (inner) view rather than SUBTRACTING that width.

diff:
@@ -301,7 +301,7 @@ extension MDCNodeListViewController {
view.leadingAnchor.constraint(equalTo: safeAreaLayoutGuide.leadingAnchor,
constant: padding),
view.trailingAnchor.constraint(equalTo: safeAreaLayoutGuide.trailingAnchor,
- constant: padding),
+ constant: -padding),
view.topAnchor.constraint(equalTo: safeAreaLayoutGuide.topAnchor, constant: top),
view.heightAnchor.constraint(equalToConstant: height)
])
@@ -311,7 +311,7 @@ extension MDCNodeListViewController {
if #available(iOS 9.0, *) {
NSLayoutConstraint.activate([
view.leadingAnchor.constraint(equalTo: containerView.leadingAnchor, constant: padding),
- view.trailingAnchor.constraint(equalTo: containerView.trailingAnchor, constant: padding),
+ view.trailingAnchor.constraint(equalTo: containerView.trailingAnchor, constant: -padding),
view.topAnchor.constraint(equalTo: containerView.topAnchor, constant: top),
view.heightAnchor.constraint(equalToConstant: height)
])
@@ -331,7 +331,7 @@ extension MDCNodeListViewController {
toItem: containerView,
attribute: .trailing,
multiplier: 1.0,
- constant: padding).isActive = true
+ constant: -padding).isActive = true
_ = NSLayoutConstraint(
item: view,
attribute: .top,

original_message:
[Catalog] Pad trailing edge inside. (#2242)

The constraints for MDCNodeListViewController incorrectly made the padding ADD width to the contained (inner) view rather than SUBTRACTING that width.


message:
[Catalog] RTL for settings dialog.
Flipping the debugging panel layout when the layout orientation is right-to-left.

diff:
import UIKit
+import MDFInternationalization
import MaterialComponents.MaterialDialogs
class MDCCatalogDebugSetting {
@@ -146,10 +147,14 @@ fileprivate class MDCCatalogDebugToggleCell: UICollectionViewCell {
let height = contentView.bounds.height
let switchSize = toggleSwitch.sizeThatFits(contentView.bounds.size)
- toggleSwitch.frame = CGRect(x: width - switchSize.width - 10,
+ var toggleSwitchFrame = CGRect(x: width - switchSize.width - 10,
y: (height - switchSize.height) / 2.0,
width: switchSize.width,
height: switchSize.height)
+ if (self.mdf_effectiveUserInterfaceLayoutDirection == .rightToLeft) {
+ toggleSwitchFrame = MDFRectFlippedHorizontally(toggleSwitchFrame, width)
+ }
+ toggleSwitch.frame = toggleSwitchFrame
label.frame = contentView.bounds.insetBy(dx: 10, dy: 10)
}

original_message:
[Catalog] RTL for settings dialog. (#2249)

Flipping the debugging panel layout when the layout orientation is right-to-left.


diff:
@@ -462,6 +462,12 @@ Material Components for iOS is built primarily for adoption with CocoaPods. Ther
1. Enter YES or NO
+### Use umbrella header between components
+
+
+Material Components should always use umbrella headers to access API of other components.
+
+
## Running the checklist
We have automated some of the above checks in a set of scripts. To run all the checks against every component, run:

change_type: ADD

diff:
+#!/usr/bin/env python
+#
+# Copyright 2016-present the Material Components for iOS authors. All Rights
+# Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import re
+import sys
+from collections import namedtuple
+
+# Prefix of all MDC umbrella headers.
+UMBRELLA_HEADER_PREFIX = 'Material'
+
+# Import prefixes that should not be checked.
+IMPORT_FILTER_PREFIXES = (
+ # Objective-C imports
+ 'Availability.h',
+ 'CoreFoundation/',
+ 'CoreGraphics/',
+ 'Foundation/',
+ 'QuartzCore/',
+ 'UIKit/',
+ 'objc/',
+ # Third-party imports
+ 'MDF',
+ 'Motion',
+)
+
+# Import suffixes that should not be checked.
+IMPORT_FILTER_SUFFIXES = (
+ # Color themers do not have umbrella headers.
+ 'ColorThemer.h',
+)
+
+# Regex to match import of two kinds:
+# 1) #import "Foo.h", group #1
+# 2) #import <Foo.h>, group #2
+IMPORT_RE = re.compile(r'#import (\"(.*\.h)\"|\<(.*\.h)\>)')
+
+
+def umbrella_header(component):
+ """Returns an umbrella header for the component."""
+ return '{}{}.h'.format(UMBRELLA_HEADER_PREFIX, component)
+
+
+def framework_umbrella_header(component):
+ """Returns a framework umbrella header for the component."""
+ return 'MaterialComponents/{}'.format(umbrella_header(component))
+
+
+# Context in which the check is executed.
+# @checked_file - the name of the file that is checked.
+# @module_files - list of files in currently analyzed sub-component.
+CheckContext = namedtuple('CheckContext', ['checked_file', 'module_files'])
+
+
+class ImportChecks(object):
+ """A class containing factory methods for various checks and filters."""
+
+ @staticmethod
+ def import_filter():
+ """Returns a function to filter Objective-C and third-party framework imports."""
+
+ def check(import_str, _):
+ filter = import_str.startswith(IMPORT_FILTER_PREFIXES) or \
+ import_str.endswith(IMPORT_FILTER_SUFFIXES)
+ return filter, None
+
+ return check
+
+ @staticmethod
+ def module_import_filter():
+ """Returns a function to filter same module file imports."""
+
+ def check(import_str, check_context):
+ return any(module_file.endswith(import_str) for module_file in check_context.module_files), None
+
+ return check
+
+ @staticmethod
+ def non_umbrella_header_check():
+ """Returns a function to check that an umbrella header is imported."""
+
+ def check(import_str, check_context):
+ if not import_str.startswith(UMBRELLA_HEADER_PREFIX):
+ error_msg = '{} imports a non-umbrella header: {}'.format(check_context.checked_file, import_str)
+ return True, error_msg
+ return False, None
+
+ return check
+
+ @staticmethod
+ def own_umbrella_header_check(umbrella_header):
+ """Returns a function to check that checked component umbrella header is not used."""
+
+ def check(import_str, check_context):
+ if import_str == umbrella_header:
+ return True, '{} imports an own umbrella header: {}'.format(check_context.checked_file, import_str)
+ return False, None
+
+ return check
+
+
+class ImportChecker(object):
+ """A class to run checks over the import strings."""
+
+ def __init__(self, check_functions):
+ self.check_functions = check_functions
+
+ def check_import(self, import_str, check_context):
+ """Runs checks over the import string and returns an error message in check failed."""
+ for check in self.check_functions:
+ stop, error_msg = check(import_str, check_context)
+ if error_msg:
+ return error_msg
+ if stop:
+ break
+
+ return None
+
+ @staticmethod
+ def src_checker(component):
+ """Returns an 'src' code checker."""
+ return ImportChecker([
+ ImportChecks.import_filter(),
+ ImportChecks.own_umbrella_header_check(umbrella_header(component)),
+ ImportChecks.own_umbrella_header_check(framework_umbrella_header(component)),
+ ImportChecks.module_import_filter(),
+ ImportChecks.non_umbrella_header_check(),
+ ])
+
+ @staticmethod
+ def general_checker():
+ """Returns a general purpose checker."""
+ return ImportChecker([
+ ImportChecks.import_filter(),
+ ImportChecks.module_import_filter(),
+ ImportChecks.non_umbrella_header_check(),
+ ])
+
+
+def imports_in_file(file_path):
+ """Returns a list of import strings."""
+ imports = []
+ with open(file_path) as f:
+ for line in f.readlines():
+ match = IMPORT_RE.match(line)
+ if not match:
+ # No imports found in file.
+ continue
+
+ groups = match.groups()
+ if groups[1]:
+ imports.append(groups[1])
+ elif groups[2]:
+ imports.append(groups[2])
+
+ return imports
+
+
+def list_objc_files(path, skip_dirs=()):
+ """Returns a list of Objective-C files in the path."""
+ files_relative_paths = []
+ for dirpath, dirnames, files in os.walk(path):
+ for f in files:
+ if os.path.basename(dirpath) in skip_dirs:
+ continue
+ if f.endswith(('.h', '.m')):
+ files_relative_paths.append(os.path.relpath(os.path.join(dirpath, f), path))
+
+ return files_relative_paths
+
+
+def check_src_path(src_path, component_name):
+ """Returns whether src_path has any import errors."""
+ sub_modules = []
+ submodules_have_errors = False
+ for dir in os.listdir(src_path):
+ # If a directory in 'src' is capitalized, treat it as a separate module.
+ if os.path.isdir(os.path.join(src_path, dir)) and dir[0].isupper():
+ sub_modules.append(dir)
+ check_successful = check_path(os.path.join(src_path, dir))
+ submodules_have_errors = submodules_have_errors or check_successful
+
+ src_has_errors = check_path(src_path,
+ checker=ImportChecker.src_checker(component_name),
+ excludes=sub_modules)
+ return src_has_errors or submodules_have_errors
+
+
+def check_path(path, checker=ImportChecker.general_checker(), excludes=()):
+ """Returns whether imports in files at path have import errors."""
+ has_errors = False
+ files = list_objc_files(path, skip_dirs=excludes)
+ for f in files:
+ check_context = CheckContext(checked_file=os.path.basename(f),
+ module_files=files)
+
+ imports = imports_in_file(os.path.join(path, f))
+ for import_str in imports:
+ error = checker.check_import(import_str, check_context)
+ if error:
+ print 'ERROR:', error
+ has_errors = True
+
+ return has_errors
+
+
+def main(args):
+ if len(args) < 1:
+ print 'ERROR: Component path should be passed as an input parameter.'
+ sys.exit(-1)
+
+ component_path = os.path.normpath(args[0])
+ component_name = os.path.basename(component_path)
+
+ src_path = os.path.join(component_path, 'src')
+ examples_path = os.path.join(component_path, 'examples')
+ has_errors = check_src_path(src_path, component_name) or \
+ check_path(examples_path)
+
+ sys.exit(-1 if has_errors else 0)
+
+
+if __name__ == '__main__':
+ main(sys.argv[1:])

original_message: Added a script to check that umbrella header are used between components (#2241)


message: [Tabs] Migrated to MDFi18n

diff:
@@ -431,13 +431,13 @@ Pod::Spec.new do |s|
sss.source_files = "components/#{ss.base_name}/src/*.{h,m}", "components/#{ss.base_name}/src/private/*.{h,m}"
sss.resources = ["components/#{ss.base_name}/src/Material#{ss.base_name}.bundle"]
+ sss.dependency "MDFInternationalization"
sss.dependency "MaterialComponents/AnimationTiming"
sss.dependency "MaterialComponents/Ink"
sss.dependency "MaterialComponents/ShadowElevations"
sss.dependency "MaterialComponents/ShadowLayer"
sss.dependency "MaterialComponents/Typography"
sss.dependency "MaterialComponents/private/Math"
- sss.dependency "MaterialComponents/private/RTL"
end
ss.subspec "ColorThemer" do |sss|
sss.ios.deployment_target = '8.0'

diff:
#import "MDCItemBarCell.h"
#import "MDCItemBarStyle.h"
+#import "MDFInternationalization.h"
#import "MaterialAnimationTiming.h"
-#import "MaterialRTL.h"
/// Height in points of the bar shown under selected items.
static const CGFloat kSelectionIndicatorHeight = 2.0f;
@@ -832,8 +832,7 @@ static void *kItemPropertyContext = &kItemPropertyContext;
// Must call super here to ensure we have the original collection bounds.
CGRect collectionBounds = {CGPointZero, [super collectionViewContentSize]};
- newAttributes.frame = MDCRectFlippedForRTL(itemFrame, CGRectGetWidth(collectionBounds),
- UIUserInterfaceLayoutDirectionRightToLeft);
+ newAttributes.frame = MDFRectFlippedHorizontally(itemFrame, CGRectGetWidth(collectionBounds));
return newAttributes;
}
@@ -846,7 +845,7 @@ static void *kItemPropertyContext = &kItemPropertyContext;
NSProcessInfo *processInfo = [NSProcessInfo processInfo];
if ([processInfo respondsToSelector:@selector(isOperatingSystemAtLeastVersion:)] &&
[processInfo isOperatingSystemAtLeastVersion:iOS9Version]) {
- if (self.collectionView.mdc_effectiveUserInterfaceLayoutDirection ==
+ if (self.collectionView.mdf_effectiveUserInterfaceLayoutDirection ==
UIUserInterfaceLayoutDirectionRightToLeft) {
enforceRTL = YES;
}

diff:
#import "MDCItemBarStringConstants.h"
#import "MDCItemBarStyle.h"
+#import "MDFInternationalization.h"
#import "MaterialAnimationTiming.h"
#import "MaterialInk.h"
#import "MaterialMath.h"
-#import "MaterialRTL.h"
#import "MaterialTypography.h"
/// Size of image in points.
@@ -245,7 +245,7 @@ static const NSTimeInterval kSelectionAnimationDuration = 0.3f;
if (_style.shouldDisplayBadge) {
CGFloat badgeOffset = (imageBounds.size.width / 2) + (badgeSize.width / 2);
- if (self.mdc_effectiveUserInterfaceLayoutDirection ==
+ if (self.mdf_effectiveUserInterfaceLayoutDirection ==
UIUserInterfaceLayoutDirectionRightToLeft) {
badgeOffset *= -1;
}

original_message: [Tabs] Migrated to MDFi18n (#2282)


message: [TextFields] Migrated to MDFi18n

diff:
@@ -458,8 +458,8 @@ Pod::Spec.new do |s|
sss.dependency "MaterialComponents/Palettes"
sss.dependency "MaterialComponents/Typography"
+ sss.dependency "MDFInternationalization"
sss.dependency "MaterialComponents/private/Math"
- sss.dependency "MaterialComponents/private/RTL"
end
ss.subspec "ColorThemer" do |sss|
sss.ios.deployment_target = '8.0'

diff:
#import "private/MDCTextInputCommonFundament.h"
#import "MaterialMath.h"
-#import "MaterialRTL.h"
#import "MaterialTypography.h"
static NSString *const MDCMultilineTextFieldExpandsOnOverflowKey =

diff:
#import "MDCTextInputCharacterCounter.h"
#import "private/MDCTextInputCommonFundament.h"
+#import "MDFInternationalization.h"
#import "MaterialMath.h"
-#import "MaterialRTL.h"
#import "MaterialTypography.h"
static NSString *const MDCTextFieldFundamentKey = @"MDCTextFieldFundamentKey";
@@ -290,7 +290,7 @@ static const CGFloat MDCTextInputEditingRectRightViewPaddingCorrection = -2.f;
}
- (UITextFieldViewMode)trailingViewMode {
- if (self.mdc_effectiveUserInterfaceLayoutDirection == UIUserInterfaceLayoutDirectionLeftToRight) {
+ if (self.mdf_effectiveUserInterfaceLayoutDirection == UIUserInterfaceLayoutDirectionLeftToRight) {
return self.rightViewMode;
} else {
return self.leftViewMode;
@@ -298,7 +298,7 @@ static const CGFloat MDCTextInputEditingRectRightViewPaddingCorrection = -2.f;
}
- (void)setTrailingViewMode:(UITextFieldViewMode)trailingViewMode {
- if (self.mdc_effectiveUserInterfaceLayoutDirection == UIUserInterfaceLayoutDirectionLeftToRight) {
+ if (self.mdf_effectiveUserInterfaceLayoutDirection == UIUserInterfaceLayoutDirectionLeftToRight) {
self.rightViewMode = trailingViewMode;
} else {
self.leftViewMode = trailingViewMode;
@@ -360,7 +360,7 @@ static const CGFloat MDCTextInputEditingRectRightViewPaddingCorrection = -2.f;
}
- (UITextFieldViewMode)leadingViewMode {
- if (self.mdc_effectiveUserInterfaceLayoutDirection == UIUserInterfaceLayoutDirectionLeftToRight) {
+ if (self.mdf_effectiveUserInterfaceLayoutDirection == UIUserInterfaceLayoutDirectionLeftToRight) {
return self.leftViewMode;
} else {
return self.rightViewMode;
@@ -368,7 +368,7 @@ static const CGFloat MDCTextInputEditingRectRightViewPaddingCorrection = -2.f;
}
- (void)setLeadingViewMode:(UITextFieldViewMode)leadingViewMode {
- if (self.mdc_effectiveUserInterfaceLayoutDirection == UIUserInterfaceLayoutDirectionLeftToRight) {
+ if (self.mdf_effectiveUserInterfaceLayoutDirection == UIUserInterfaceLayoutDirectionLeftToRight) {
self.leftViewMode = leadingViewMode;
} else {
self.rightViewMode = leadingViewMode;
@@ -415,11 +415,11 @@ static const CGFloat MDCTextInputEditingRectRightViewPaddingCorrection = -2.f;
// and .rightView gets the value for rightViewRectForBounds.
CGFloat leftViewWidth =
- self.mdc_effectiveUserInterfaceLayoutDirection == UIUserInterfaceLayoutDirectionRightToLeft
+ self.mdf_effectiveUserInterfaceLayoutDirection == UIUserInterfaceLayoutDirectionRightToLeft
? CGRectGetWidth([self rightViewRectForBounds:bounds])
: CGRectGetWidth([self leftViewRectForBounds:bounds]);
CGFloat rightViewWidth =
- self.mdc_effectiveUserInterfaceLayoutDirection == UIUserInterfaceLayoutDirectionRightToLeft
+ self.mdf_effectiveUserInterfaceLayoutDirection == UIUserInterfaceLayoutDirectionRightToLeft
? CGRectGetWidth([self leftViewRectForBounds:bounds])
: CGRectGetWidth([self rightViewRectForBounds:bounds]);
@@ -458,10 +458,9 @@ static const CGFloat MDCTextInputEditingRectRightViewPaddingCorrection = -2.f;
actualY = textInsets.top - actualY;
textRect.origin.y = actualY;
- if (self.mdc_effectiveUserInterfaceLayoutDirection == UIUserInterfaceLayoutDirectionRightToLeft) {
+ if (self.mdf_effectiveUserInterfaceLayoutDirection == UIUserInterfaceLayoutDirectionRightToLeft) {
// Now that the text field is laid out as if it were LTR, we can flip it if necessary.
- textRect = MDCRectFlippedForRTL(textRect, CGRectGetWidth(bounds),
- UIUserInterfaceLayoutDirectionRightToLeft);
+ textRect = MDFRectFlippedHorizontally(textRect, CGRectGetWidth(bounds));
}
return textRect;
@@ -472,9 +471,8 @@ static const CGFloat MDCTextInputEditingRectRightViewPaddingCorrection = -2.f;
CGRect editingRect = [self textRectForBounds:bounds];
// The textRect comes to us flipped for RTL (if RTL) so we flip it back before adjusting.
- if (self.mdc_effectiveUserInterfaceLayoutDirection == UIUserInterfaceLayoutDirectionRightToLeft) {
- editingRect = MDCRectFlippedForRTL(editingRect, CGRectGetWidth(bounds),
- UIUserInterfaceLayoutDirectionRightToLeft);
+ if (self.mdf_effectiveUserInterfaceLayoutDirection == UIUserInterfaceLayoutDirectionRightToLeft) {
+ editingRect = MDFRectFlippedHorizontally(editingRect, CGRectGetWidth(bounds));
}
// UITextFields show EITHER the clear button or the rightView. If the rightView has a superview,
@@ -503,9 +501,8 @@ static const CGFloat MDCTextInputEditingRectRightViewPaddingCorrection = -2.f;
}
}
- if (self.mdc_effectiveUserInterfaceLayoutDirection == UIUserInterfaceLayoutDirectionRightToLeft) {
- editingRect = MDCRectFlippedForRTL(editingRect, CGRectGetWidth(bounds),
- UIUserInterfaceLayoutDirectionRightToLeft);
+ if (self.mdf_effectiveUserInterfaceLayoutDirection == UIUserInterfaceLayoutDirectionRightToLeft) {
+ editingRect = MDFRectFlippedHorizontally(editingRect, CGRectGetWidth(bounds));
}
if ([self.fundament.positioningDelegate

diff:
#import "MaterialAnimationTiming.h"
#import "MaterialMath.h"
#import "MaterialPalettes.h"
-#import "MaterialRTL.h"
#import "MaterialTypography.h"
#pragma mark - Constants

diff:
#import "MaterialAnimationTiming.h"
#import "MaterialMath.h"
#import "MaterialPalettes.h"
-#import "MaterialRTL.h"
#import "MaterialTypography.h"
static const CGFloat MDCTextInputFullWidthHintTextOpacity = 0.54f;

diff:
#import "MaterialAnimationTiming.h"
#import "MaterialMath.h"
#import "MaterialPalettes.h"
-#import "MaterialRTL.h"
#import "MaterialTypography.h"
static const CGFloat MDCTextInputControllerLegacyFullWidthClearButtonImageSquareWidthHeight = 24.f;

original_message: [TextFields] Migrated to MDFi18n (#2283)


message: [ThumbTrack] Migrated to MDFi18n

diff:
@@ -551,8 +551,8 @@ Pod::Spec.new do |s|
ss.dependency "MaterialComponents/ShadowLayer"
ss.dependency "MaterialComponents/Typography"
+ ss.dependency "MDFInternationalization"
ss.dependency "MaterialComponents/private/Math"
- ss.dependency "MaterialComponents/private/RTL"
end
pss.subspec "UIMetrics" do |ss|

diff:
#import "MDCNumericValueLabel.h"
#import "MDCThumbView.h"
+#import "MDFInternationalization.h"
#import "MaterialInk.h"
#import "MaterialMath.h"
-#import "MaterialRTL.h"
static const CGFloat kAnimationDuration = 0.25f;
static const CGFloat kThumbChangeAnimationDuration = 0.12f;
@@ -824,7 +824,7 @@ static inline CGFloat DistanceFromPointToPoint(CGPoint point1, CGPoint point2) {
CGFloat relValue = (position.x - _thumbRadius) / self.thumbPanRange;
relValue = MAX(0, MIN(relValue, 1));
// For RTL we invert the value
- if (self.mdc_effectiveUserInterfaceLayoutDirection == UIUserInterfaceLayoutDirectionRightToLeft) {
+ if (self.mdf_effectiveUserInterfaceLayoutDirection == UIUserInterfaceLayoutDirectionRightToLeft) {
relValue = 1 - relValue;
}
return (1 - relValue) * _minimumValue + relValue * _maximumValue;
@@ -868,7 +868,7 @@ static inline CGFloat DistanceFromPointToPoint(CGPoint point1, CGPoint point2) {
}
CGFloat relValue = (value - _minimumValue) / MDCFabs(_minimumValue - _maximumValue);
// For RTL we invert the value
- if (self.mdc_effectiveUserInterfaceLayoutDirection == UIUserInterfaceLayoutDirectionRightToLeft) {
+ if (self.mdf_effectiveUserInterfaceLayoutDirection == UIUserInterfaceLayoutDirectionRightToLeft) {
relValue = 1 - relValue;
}
return relValue;

original_message: [ThumbTrack] Migrated to MDFi18n (#2284)


diff:
#import "MDCBottomNavigationBar.h"
#import "MaterialMath.h"
-#import "MaterialRTL.h"
#import "MaterialShadowLayer.h"
+#import "MDFInternationalization.h"
#import "private/MaterialBottomNavigationStrings.h"
#import "private/MaterialBottomNavigationStrings_table.h"
#import "private/MDCBottomNavigationItemView.h"
@@ -160,7 +160,7 @@ static NSString *const kMDCBottomNavigationBarTitleString = @"title";
}
- (void)layoutItemViews {
- UIUserInterfaceLayoutDirection layoutDirection = self.mdc_effectiveUserInterfaceLayoutDirection;
+ UIUserInterfaceLayoutDirection layoutDirection = self.mdf_effectiveUserInterfaceLayoutDirection;
NSInteger numItems = self.items.count;
if (numItems == 0) {
return;

diff:
#import "MDCBottomNavigationItemView.h"
-#import "MaterialRTL.h"
#import "MaterialBottomNavigationStrings.h"
#import "MaterialBottomNavigationStrings_table.h"
#import "MDCBottomNavigationItemBadge.h"
+#import "MDFInternationalization.h"
static const CGFloat kMDCBottomNavigationItemViewCircleLayerOffset = -6.f;
static const CGFloat kMDCBottomNavigationItemViewCircleLayerDimension = 36.f;
@@ -146,7 +146,7 @@ static NSString *const kMDCBottomNavigationItemViewTabString = @"tab";
}
self.label.textAlignment = NSTextAlignmentCenter;
} else {
- UIUserInterfaceLayoutDirection layoutDirection = self.mdc_effectiveUserInterfaceLayoutDirection;
+ UIUserInterfaceLayoutDirection layoutDirection = self.mdf_effectiveUserInterfaceLayoutDirection;
if (layoutDirection == UIUserInterfaceLayoutDirectionLeftToRight) {
CGPoint iconImageViewCenter =
CGPointMake(CGRectGetMidX(self.bounds) - CGRectGetWidth(self.bounds) * 0.2f,

original_message: [BotteomNavigation] Migrated to MDFi18n. (#2306)