SetHeaderRoute sometimes does not change the virtual service

[diegobernardes]

I have a problem with Argo Rollout where setHeaderRoute do not always update the virtual service, it may update 1 out of 10. Now the interesting part is, the setWeight always work, I had absolutely no issue with it.

Any idea on how to debug this issue?

ArgoCD: v2.14.7+d107d4e
Argo Rollout: v1.8.1+1ad2c6a
Istio: 1.24.2

---
apiVersion: networking.istio.io/v1alpha3
kind: DestinationRule
metadata:
  name: internal-service-rollout
spec:
  host: internal-service.default.svc.cluster.local
  subsets:
  - name: canary
    labels:
      app: myapp-internal-service
  - name: stable
    labels:
      app: myapp-internal-service
---
apiVersion: argoproj.io/v1alpha1
kind: Rollout
metadata:
  name: myapp-internal-service
  labels:
    tags.datadoghq.com/env: "dev"
    tags.datadoghq.com/service: "internal-service"
    tags.datadoghq.com/version: "latest"
    app: "internal-service"
    version: "latest"
    app.kubernetes.io/name: internal-service
    helm.sh/chart: internal-service-1.0.0
    app.kubernetes.io/instance: myapp
    app.kubernetes.io/version: "latest"
    app.kubernetes.io/managed-by: Helm
spec:
  selector:
    matchLabels:
      app.kubernetes.io/name: internal-service
      app.kubernetes.io/instance: myapp
  strategy:
    canary:
      trafficRouting:
        managedRoutes:
          - name: test-e2e
        istio:
          virtualService:
            name: internal-service-virtual-service
            routes:
            - internal-service-primary
          destinationRule:
            name: internal-service-rollout
            canarySubsetName: canary
            stableSubsetName: stable
      steps:
          - setCanaryScale:
              replicas: 1 
          - setHeaderRoute:
              match:
              - headerName: test-e2e
                headerValue:
                  exact: "true"
              name: test-e2e
          - analysis:
              templates:
                - templateName: analysis-test-e2e
          - setCanaryScale:
              matchTrafficWeight: true
          - setWeight: 10
          - pause: { duration: 5m }
          - setWeight: 50
          - pause: { duration: 5m }
  template:
    metadata:
      annotations:
        reload-on-secret-change/auto: "true"
        sidecar.istio.io/inject: "true"
      labels:
        app: "internal-service"
        version: "latest"
        app.kubernetes.io/name: internal-service
        app.kubernetes.io/instance: myapp
        app.kubernetes.io/version: "latest"
        tags.datadoghq.com/env: "dev"
        tags.datadoghq.com/service: "internal-service"
        tags.datadoghq.com/version: "latest"
    spec:
      serviceAccountName: myapp-internal-service
      containers:
        - name: internal-service
          securityContext:
            {}
          image: "my-image"
          imagePullPolicy: IfNotPresent
          ports:
            - name: http
              containerPort: 8080
              protocol: TCP
          livenessProbe:
            httpGet:
              path: /health
              port: 8080
            initialDelaySeconds: 30
            timeoutSeconds: 5
          readinessProbe:
            httpGet:
              path: /health
              port: 8080
            initialDelaySeconds: 30
            timeoutSeconds: 5
            successThreshold: 3
          resources:
            limits:
              cpu: 500m
              memory: 2Gi
            requests:
              cpu: 200m
              memory: 1Gi
      affinity:
        nodeAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
            nodeSelectorTerms:
            - matchExpressions:
              - key: node-category/internal-category
                operator: In
                values:
                - "true"
---
apiVersion: networking.istio.io/v1alpha3
kind: VirtualService
metadata:
  name: internal-service-virtual-service
spec:
  hosts:
    - internal-service.default.svc.cluster.local
  http:
    - name: test-e2e
      match:
      - headers:
          test-e2e:
            exact: 'true'
      route:
        - destination:
            host: internal-service.internal-namespace.svc.cluster.local
            subset: canary
          weight: 100
    - name: internal-service-primary
      route:
        - destination:
            host: myapp-internal-service
            subset: stable
          weight: 100
        - destination:
            host: myapp-internal-service
            subset: canary
          weight: 0