Configuration-as-code-plugin: Support YAML syntax features: Anchors, Aliases and Extensions

[almiroshnich]

Checklist for this issue

• [x] 🚨review the guidelines for contributing to this repository.

Feature Request

Would be nice to have native support for situations like this

x-ec2_anchor: &ec2_anchor
  description: "ec2-anchor"
  labelString: "ec2-anchor"
  type: "ec2-anchor"
  ami: "ec2-anchor"
  amiType:
    unixData:
      sshPort: "22"
  associatePublicIp: false
  connectBySSHProcess: false
  connectionStrategy: "PRIVATE_IP"
  deleteRootOnTermination: true
  ebsOptimized: false
  idleTerminationMinutes: "2"
  initScript: ""
  maxTotalUses: 1
  minimumNumberOfInstances: 0
  instanceCapStr: 50
  iamInstanceProfile: ""
  jvmopts: ""
  mode: "NORMAL"
  monitoring: false
  numExecutors: 1
  remoteAdmin: "ubuntu"
  remoteFS: "/home/ubuntu"
  spotConfig:
    useBidPrice: false
    fallbackToOndemand: true
    spotBlockReservationDurationStr: 1
  stopOnTerminate: false
  securityGroups: "default"
  subnetId: "subnet-XXXXXX"
  t2Unlimited: false
  useDedicatedTenancy: false
  useEphemeralDevices: false
  userData: ""
  zone: ""
  tags:
    - name: "label"
      value: "jenkins-slave"
    - name: "environment"
      value: "development"
    - name: "cost_center"
      value: "devinfra"

jenkins:
  clouds:
    - amazonEC2:
        cloudName: "us-east-common"
        useInstanceProfileForCredentials: true
        instanceCapStr: 200
        noDelayProvisioning: true
        privateKey: ${EC2_PRIVATE_KEY}
        region: "us-east-1"
        templates:
          - <<: *ec2_anchor
            ami: "ami-XXXXXX"
            type: "T3Small"
            description: "T3Small:xenial"
            labelString: "ubuntu:xenial aws:small"
          - <<: *ec2_anchor
            ami: "ami-XXXXXX"
            type: "T3Large"
            description: "T3Large:xenial"
            labelString: "ubuntu:xenial aws:large"
          - <<: *ec2_anchor
            ami: "ami-YYYYYY"
            type: "T3Small"
            description: "T3Small:bionic"
            labelString: "ubuntu:bionic aws:small"
          - <<: *ec2_anchor
            ami: "ami-YYYYYY"
            type: "T3Large"
            description: "T3Large:bionic"
            labelString: "ubuntu:bionic aws:large"

Currently, this config causes such exception:

Stack Trace (clickable)

io.jenkins.plugins.casc.ConfiguratorException: No configurator for root element <x-ec2_anchor>
    at io.jenkins.plugins.casc.ConfigurationAsCode.invokeWith(ConfigurationAsCode.java:710)
    at io.jenkins.plugins.casc.ConfigurationAsCode.checkWith(ConfigurationAsCode.java:746)
    at io.jenkins.plugins.casc.ConfigurationAsCode.configureWith(ConfigurationAsCode.java:731)
    at io.jenkins.plugins.casc.ConfigurationAsCode.configureWith(ConfigurationAsCode.java:611)
    at io.jenkins.plugins.casc.ConfigurationAsCode.configure(ConfigurationAsCode.java:292)
    at io.jenkins.plugins.casc.ConfigurationAsCode.doReload(ConfigurationAsCode.java:156)
    at java.lang.invoke.MethodHandle.invokeWithArguments(MethodHandle.java:627)
    at org.kohsuke.stapler.Function$MethodFunction.invoke(Function.java:396)
    at org.kohsuke.stapler.Function$InstanceFunction.invoke(Function.java:408)
    at org.kohsuke.stapler.interceptor.RequirePOST$Processor.invoke(RequirePOST.java:77)
    at org.kohsuke.stapler.PreInvokeInterceptedFunction.invoke(PreInvokeInterceptedFunction.java:26)
    at org.kohsuke.stapler.Function.bindAndInvoke(Function.java:212)
    at org.kohsuke.stapler.Function.bindAndInvokeAndServeResponse(Function.java:145)
    at org.kohsuke.stapler.MetaClass$11.doDispatch(MetaClass.java:535)
    at org.kohsuke.stapler.NameBasedDispatcher.dispatch(NameBasedDispatcher.java:58)
    at org.kohsuke.stapler.Stapler.tryInvoke(Stapler.java:747)
    at org.kohsuke.stapler.Stapler.invoke(Stapler.java:878)
    at org.kohsuke.stapler.MetaClass$9.dispatch(MetaClass.java:456)
    at org.kohsuke.stapler.Stapler.tryInvoke(Stapler.java:747)
    at org.kohsuke.stapler.Stapler.invoke(Stapler.java:878)
    at org.kohsuke.stapler.Stapler.invoke(Stapler.java:676)
    at org.kohsuke.stapler.Stapler.service(Stapler.java:238)
    at javax.servlet.http.HttpServlet.service(HttpServlet.java:790)
    at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:873)
    at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1623)
    at hudson.util.PluginServletFilter$1.doFilter(PluginServletFilter.java:154)
    at com.cloudbees.jenkins.support.slowrequest.SlowRequestFilter.doFilter(SlowRequestFilter.java:37)
    at hudson.util.PluginServletFilter$1.doFilter(PluginServletFilter.java:151)
    at jenkins.metrics.impl.MetricsFilter.doFilter(MetricsFilter.java:125)
    at hudson.util.PluginServletFilter$1.doFilter(PluginServletFilter.java:151)
    at jenkins.telemetry.impl.UserLanguages$AcceptLanguageFilter.doFilter(UserLanguages.java:128)
    at hudson.util.PluginServletFilter$1.doFilter(PluginServletFilter.java:151)
    at hudson.util.PluginServletFilter.doFilter(PluginServletFilter.java:157)
    at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610)
    at hudson.security.csrf.CrumbFilter.doFilter(CrumbFilter.java:64)
    at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610)
    at hudson.security.ChainedServletFilter$1.doFilter(ChainedServletFilter.java:84)
    at hudson.security.UnwrapSecurityExceptionFilter.doFilter(UnwrapSecurityExceptionFilter.java:51)
    at hudson.security.ChainedServletFilter$1.doFilter(ChainedServletFilter.java:87)
    at jenkins.security.ExceptionTranslationFilter.doFilter(ExceptionTranslationFilter.java:117)
    at hudson.security.ChainedServletFilter$1.doFilter(ChainedServletFilter.java:87)
    at org.acegisecurity.providers.anonymous.AnonymousProcessingFilter.doFilter(AnonymousProcessingFilter.java:125)
    at hudson.security.ChainedServletFilter$1.doFilter(ChainedServletFilter.java:87)
    at org.acegisecurity.ui.rememberme.RememberMeProcessingFilter.doFilter(RememberMeProcessingFilter.java:142)
    at hudson.security.ChainedServletFilter$1.doFilter(ChainedServletFilter.java:87)
    at org.acegisecurity.ui.AbstractProcessingFilter.doFilter(AbstractProcessingFilter.java:271)
    at hudson.security.ChainedServletFilter$1.doFilter(ChainedServletFilter.java:87)
    at jenkins.security.BasicHeaderProcessor.doFilter(BasicHeaderProcessor.java:93)
    at hudson.security.ChainedServletFilter$1.doFilter(ChainedServletFilter.java:87)
    at org.acegisecurity.context.HttpSessionContextIntegrationFilter.doFilter(HttpSessionContextIntegrationFilter.java:249)
    at hudson.security.HttpSessionContextIntegrationFilter2.doFilter(HttpSessionContextIntegrationFilter2.java:67)
    at hudson.security.ChainedServletFilter$1.doFilter(ChainedServletFilter.java:87)
    at hudson.security.ChainedServletFilter.doFilter(ChainedServletFilter.java:90)
    at hudson.security.HudsonFilter.doFilter(HudsonFilter.java:171)
    at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610)
    at org.kohsuke.stapler.compression.CompressionFilter.doFilter(CompressionFilter.java:49)
    at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610)
    at hudson.util.CharacterEncodingFilter.doFilter(CharacterEncodingFilter.java:82)
    at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610)
    at org.kohsuke.stapler.DiagnosticThreadNameFilter.doFilter(DiagnosticThreadNameFilter.java:30)
    at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610)
    at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:540)
    at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:146)
    at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:524)
    at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132)
    at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:257)
    at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1700)
    at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255)
    at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1345)
    at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203)
    at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:480)
    at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1667)
    at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201)
    at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1247)
    at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144)
    at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132)
    at org.eclipse.jetty.server.Server.handle(Server.java:505)
    at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:370)
    at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:267)
    at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:305)
    at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103)
    at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:117)
    at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:333)
    at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:310)
    at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)
    at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:126)
    at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:366)
    at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:698)
    at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:804)
    at java.lang.Thread.run(Thread.java:748)

Relevant links:

• jenkinsci/configuration-as-code-plugin/demos/ec2
• medium/anchors-aliases-extensions (docker-compose)

[jetersen]

If we would introduce this, it would only be supported on import.
However I would have no idea what changes it would require.

One pass to find all anchors an second pass to configure?

Secondly we would have no way of validating the schema.

Thirdly most editors have shortcut keys to find next matching element, multiline editing and search & replace.
I honestly prefer the declarative nature even though it might be repeating.

Fourthly there plenty of yaml templating tools that can perform the operation for you.

[almiroshnich]

Yeah, I can imagine the struggle...
At the moment, I use a python script to expand this. But the approach to reload this _on-the-fly_ is going to be super clumsy 🙁

[nanderson94]

Anchors/Aliases are a feature of the yaml parser (snakeyaml in this case).

The above example should be working regarding the anchor/alias, but is failing because there is no configuration module for the x-ec2_anchor property. If there is a way to put the anchor on a "noop" configuration module that accepts any schema, it should work.

[almiroshnich]

@nanderson94 good point!
I've tested adding x-ec2_anchor to the _unclassified_ block, but it seems it also has a strict schema validation, so that didn't work out either.

[jetersen]

I'd welcome any PR that looks if a root element is an anchor and should than not throw an exception.

[jetersen]

Sometimes it's simpler than you think @almiroshnich please see #1379

Do know that anchors and aliases will not be supported on export.
That would require a whole lot more work that I do not see the benefit from.

[jetersen]

The fix is included in v1.41

[jeffreymcclain]

Does anyone know if it is possible to use an anchor, alias, or extension for a job build parameter's default value?
Note that I want everything else to be statically defined as usual, but I want a parameter's default value to be dynamic.
The syntax for my examples is probably wrong, but I'm just trying to get my basic objective across:

x-AliasTest: 'dynamicValue'
stringParam('AliasTest', {{x-AliasTest}}, 'TEST yaml anchor.')
stringParam('Var', 'abc', 'example variable..')
stringParam('OtherVar', '123', 'other example variable')

I'd also be willing to accept the entire line as an alias in a format like so:

x-AliasTestAlt: "stringParam('AliasTest', 'static value', 'TEST yaml anchor.')"
{{x-AliasTestAlt}}
stringParam('Var', 'abc', 'example variable..')
stringParam('OtherVar', '123', 'other example variable')

Examples within the context of a larger casc file:

jobs:
  - script: >
      pipelineJob('example') {
          description('test job')
          properties {
            pipelineTriggers {
              triggers {
                cron {
                  spec('H 2 * * 3')
                }
              }
            }
          }
          parameters {
            stringParam('AliasTest', "{{x-AliasTest}}", 'TEST yaml anchor.')
            {{x-AliasTestAlt}}
            stringParam('Var', 'abc', 'example variable..')
            stringParam('OtherVar', '123', 'other example variable')
          }
          definition {
            cpsScm {
              scriptPath 'example/test.groovy'
              scm {
                git {
                  remote { 
                    url 'https://github.com/example'
                    credentials('jenkins-test-credentials')
                  }
                  branch('test')
                  extensions {}
                }
              }
              lightweight(lightweight = true)
            }
          }
        }

[jetersen]

What you want is to use the JCasC secrets which has support for default values. ${VALUE:-defaultvalue}

https://github.com/jenkinsci/configuration-as-code-plugin/blob/master/docs/features/secrets.adoc#passing-secrets-through-variables

[jeffreymcclain]

Thanks! I read the document you linked and it sounds like I can use the "secrets" functionality to create / pass variables (even though my use case isn't really for secret data, just some file paths).

I'm still not sure I understand what your example code ${VALUE:-defaultvalue} would look like in an actual casc file, does the following example code below appear to have the correct syntax?:

#initalize the alias variable and set a default value
${ExampleAliasName:"/example/file/path/value.txt"}

jenkins:
<other code that isn't relevant to the example>

jobs:
  - script: >
      pipelineJob('example') {
          description('test job')
          properties {
            pipelineTriggers {
              triggers {
                cron {
                  spec('H 2 * * 3')
                }
              }
            }
          }
          parameters {
            stringParam('AliasTest', $ExampleAliasName, 'TEST yaml anchor.')
            stringParam('Var', 'abc', 'example variable..')
            stringParam('OtherVar', '123', 'other example variable')
          }
          definition {
            cpsScm {
              scriptPath 'example/test.groovy'
              scm {
                git {
                  remote { 
                    url 'https://github.com/example'
                    credentials('jenkins-test-credentials')
                  }
                  branch('test')
                  extensions {}
                }
              }
              lightweight(lightweight = true)
            }
          }
        }

If not, and if it's not too much trouble, would you mind providing a basic working example? Apologies if this should be obvious / self-explanatory, I've never used YAML aliases before. I know in most programming languages that initializing and using a variable should be trivial, so I'm probably just not understanding how the example syntax in the documentation you linked actually looks / fits into the larger casc file.

[jetersen]

jenkins:
<other code that isn't relevant to the example>

jobs:
  - script: >
      pipelineJob('example') {
          description('test job')
          properties {
            pipelineTriggers {
              triggers {
                cron {
                  spec('H 2 * * 3')
                }
              }
            }
          }
          parameters {
            stringParam('AliasTest', "${ExampleAliasName:-/example/file/path/value.txt}", 'TEST yaml anchor.')
            stringParam('Var', 'abc', 'example variable..')
            stringParam('OtherVar', '123', 'other example variable')
          }
          definition {
            cpsScm {
              scriptPath 'example/test.groovy'
              scm {
                git {
                  remote { 
                    url 'https://github.com/example'
                    credentials('jenkins-test-credentials')
                  }
                  branch('test')
                  extensions {}
                }
              }
              lightweight(lightweight = true)
            }
          }
        }