Skip to content

Commit e00050f

Browse files
authored
Migrate ci-mgmt plugins entries to mise (#1792)
This PR migrates the `plugins` entry in `ci-mgmt.yaml` to `.config/mise.toml`. To simplify the mise migrations I also refactored a little so that we now have the following flow: - `deleteOldMiseConfig`: Runs first and deletes the old root level `mise.toml` if it exists and is not the overrides file - `migrateCimgmtOverrides`: Migrates things from `ci-mgmt.yaml` to the root level override file (currently just `toolVersions`). - `migrateCimgmtToMise`: Migrates things from `ci-mgmt.yaml` to the `.config/mise.toml` file. - `maintainMiseLock`: Runs last and runs `mise install` to update the lockfiles. I've set the plugins version to "latest" for all the plugins so that we can implement an upgrade workflow (e.g. via `mise upgrade`). Also, since the plugins are added to the `.config/mise.toml` we can add specific version overrides to the root level `mise.toml`. ## Additional things - I've added some generic functionality for dealing with `toml` and `yaml` files so it will be easier to maintain these going forward - Since `mise` can automatically install `plugins`, I've also created a backend plugin for installing the plugins https://github.com/pulumi/vfox-pulumi - This plugin will be automatically installed with `mise install` so that the tools defined using the plugins can be installed. - Added `mise_install` and `mise_env` make targets. This should make it so that as long as you have `mise` installed, running any of the `make` targets should automatically use the `mise` environment. closes #1474
1 parent ef4150e commit e00050f

File tree

124 files changed

+1205
-280
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

124 files changed

+1205
-280
lines changed

.github/workflows/test-windows.yml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,5 +10,7 @@ jobs:
1010
- name: Install mise
1111
uses: jdx/mise-action@e3d7b8d67a7958d1207f6ed871e83b1ea780e7b0 # v3
1212
with:
13+
# Latest working version. See https://github.com/jdx/mise/discussions/6781
14+
version: 2025.10.16
1315
github_token: ${{ secrets.GITHUB_TOKEN }}
1416

.github/workflows/update-workflows.yml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -95,6 +95,8 @@ jobs:
9595
- name: Install mise
9696
uses: jdx/mise-action@e3d7b8d67a7958d1207f6ed871e83b1ea780e7b0 # v3
9797
with:
98+
# Latest working version. See https://github.com/jdx/mise/discussions/6781
99+
version: 2025.10.16
98100
install: false
99101
cache: false
100102
working_directory: pulumi-${{ inputs.provider_name }}

provider-ci/Makefile

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ ACTIONLINT := bin/actionlint-$(ACTIONLINT_VERSION)
88

99
.PHONY: all test gen ensure
1010
all: ensure test format lint
11-
test: test-providers
11+
test: test-go test-providers
1212
gen: test-providers
1313
ensure:: bin/provider-ci $(ACTIONLINT)
1414

@@ -20,7 +20,10 @@ $(ACTIONLINT):
2020
mv bin/actionlint $(ACTIONLINT)
2121

2222
# Basic helper targets.
23-
.PHONY: clean lint format
23+
.PHONY: test-go clean lint format
24+
test-go:
25+
go test -v ./...
26+
2427
clean:
2528
rm -rf bin
2629

provider-ci/go.mod

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,6 @@ require (
88
github.com/Masterminds/sprig v2.22.0+incompatible
99
github.com/bitfield/script v0.24.0
1010
github.com/spf13/cobra v1.7.0
11-
golang.org/x/mod v0.27.0
1211
gopkg.in/yaml.v3 v3.0.1
1312
)
1413

provider-ci/go.sum

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -48,8 +48,6 @@ github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5Cc
4848
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
4949
golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U=
5050
golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
51-
golang.org/x/mod v0.27.0 h1:kb+q2PyFnEADO2IEF935ehFUXlWiNjJWtRNgBLSfbxQ=
52-
golang.org/x/mod v0.27.0/go.mod h1:rWI627Fq0DEoudcK+MBkNkCe0EetEaDSwJJkCcjpazc=
5351
golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI=
5452
golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
5553
golang.org/x/tools v0.36.0 h1:kWS0uv/zsvHEle1LbV5LE8QujrxB3wfQyxHfhOk0Qkg=

provider-ci/internal/pkg/generate.go

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -147,6 +147,7 @@ func getDeletedFiles(templateName string) []string {
147147
".goreleaser.yml",
148148
".goreleaser.prerelease.yml",
149149
".github/actions/setup-tools",
150+
"scripts/plugins.mk",
150151
}
151152
case "external-bridged-provider":
152153
return []string{
@@ -156,15 +157,18 @@ func getDeletedFiles(templateName string) []string {
156157
".goreleaser.yml",
157158
".goreleaser.prerelease.yml",
158159
".github/actions/setup-tools",
160+
"scripts/plugins.mk",
159161
}
160162
case "generic":
161163
return []string{
162164
".upgrade-config.yml", // Previously accidentally generated empty file.
163165
".github/actions/setup-tools",
166+
"scripts/plugins.mk",
164167
}
165168
case "parameterized-go":
166169
return []string{
167170
".github/actions/setup-tools",
171+
"scripts/plugins.mk",
168172
}
169173
default:
170174
return nil
Lines changed: 102 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,102 @@
1+
package migrations
2+
3+
import (
4+
"fmt"
5+
"os"
6+
7+
"gopkg.in/yaml.v3"
8+
)
9+
10+
// cimgmtYaml wraps the contents of a .ci-mgmt.yaml file along with its source path.
11+
type cimgmtYaml struct {
12+
path string
13+
// we operate on the yaml.Node so that we can preserve comments and ordering
14+
node *yaml.Node
15+
}
16+
17+
// newCimgmtYaml loads the YAML document from disk and prepares it for mutation.
18+
func newCimgmtYaml(path string) (*cimgmtYaml, error) {
19+
ciMgmtPath := path
20+
ciMgmtFile, err := os.ReadFile(ciMgmtPath)
21+
if err != nil {
22+
return nil, fmt.Errorf("error reading .ci-mgmt.yaml: %w", err)
23+
}
24+
25+
var ciMgmt yaml.Node
26+
if err := yaml.Unmarshal(ciMgmtFile, &ciMgmt); err != nil {
27+
return nil, fmt.Errorf("error unmarshaling .ci-mgmt.yaml: %w", err)
28+
}
29+
30+
return &cimgmtYaml{
31+
node: &ciMgmt,
32+
path: path,
33+
}, nil
34+
}
35+
36+
// writeFile persists the in-memory YAML representation back to its original path.
37+
func (c *cimgmtYaml) writeFile() error {
38+
newCiMgmt, err := yaml.Marshal(c.node)
39+
if err != nil {
40+
return fmt.Errorf("error marshaling .ci-mgmt.yaml: %w", err)
41+
}
42+
if err := os.WriteFile(c.path, newCiMgmt, 0644); err != nil {
43+
return fmt.Errorf("error writing .ci-mgmt.yaml: %w", err)
44+
}
45+
46+
return nil
47+
}
48+
49+
// deleteKey deletes a top level field from the ci-mgmt.yaml file
50+
func (c *cimgmtYaml) deleteKey(key string) {
51+
node := c.node.Content[0]
52+
if node == nil || node.Kind != yaml.MappingNode {
53+
return
54+
}
55+
56+
out := node.Content[:0]
57+
for i := 0; i < len(node.Content); i += 2 {
58+
if i+1 >= len(node.Content) {
59+
continue
60+
}
61+
k := node.Content[i]
62+
v := node.Content[i+1]
63+
if k.Value == key {
64+
continue // skip this key/value pair (delete)
65+
}
66+
out = append(out, k, v)
67+
}
68+
node.Content = out
69+
}
70+
71+
// getFieldNode gets a top level field from the ci-mgmt.yaml file
72+
func (c *cimgmtYaml) getFieldNode(key string) *yaml.Node {
73+
node := c.node.Content[0]
74+
if node.Kind != yaml.MappingNode {
75+
return nil
76+
}
77+
for i := 0; i < len(node.Content); i += 2 {
78+
k := node.Content[i]
79+
v := node.Content[i+1]
80+
if k.Value == key {
81+
return v
82+
}
83+
}
84+
return nil
85+
}
86+
87+
// nodeToMap converts a yaml.Node with object data to a map[string]string
88+
func nodeToMap(m *yaml.Node) map[string]string {
89+
if m == nil {
90+
return nil
91+
}
92+
if m.Kind != yaml.MappingNode {
93+
return nil
94+
}
95+
out := make(map[string]string)
96+
for i := 0; i < len(m.Content); i += 2 {
97+
k := m.Content[i]
98+
v := m.Content[i+1]
99+
out[k.Value] = v.Value
100+
}
101+
return out
102+
}
Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
package migrations
2+
3+
import (
4+
"os"
5+
"path/filepath"
6+
"strings"
7+
"testing"
8+
)
9+
10+
func TestCimgmtYamlDeleteKeyAndWrite(t *testing.T) {
11+
dir := t.TempDir()
12+
path := filepath.Join(dir, ".ci-mgmt.yaml")
13+
initial := `toolVersions:
14+
pulumictl: "1.x"
15+
plugins:
16+
- name: random
17+
version: latest
18+
`
19+
if err := os.WriteFile(path, []byte(initial), 0o644); err != nil {
20+
t.Fatalf("write fixture: %v", err)
21+
}
22+
23+
cimgmt, err := newCimgmtYaml(path)
24+
if err != nil {
25+
t.Fatalf("newCimgmtYaml: %v", err)
26+
}
27+
28+
if node := cimgmt.getFieldNode("missing"); node != nil {
29+
t.Fatalf("expected missing field to return nil")
30+
}
31+
32+
toolVersions := cimgmt.getFieldNode("toolVersions")
33+
if toolVersions == nil {
34+
t.Fatalf("expected toolVersions node")
35+
}
36+
tvMap := nodeToMap(toolVersions)
37+
if got := tvMap["pulumictl"]; got != "1.x" {
38+
t.Fatalf("toolVersions map mismatch: got %q", got)
39+
}
40+
41+
cimgmt.deleteKey("plugins")
42+
if err := cimgmt.writeFile(); err != nil {
43+
t.Fatalf("writeFile: %v", err)
44+
}
45+
46+
out, err := os.ReadFile(path)
47+
if err != nil {
48+
t.Fatalf("read result: %v", err)
49+
}
50+
content := string(out)
51+
if strings.Contains(content, "plugins") {
52+
t.Fatalf("expected plugins key removed, got:\n%s", content)
53+
}
54+
if !strings.Contains(content, "toolVersions") {
55+
t.Fatalf("expected toolVersions key retained, got:\n%s", content)
56+
}
57+
}
58+
59+
func TestCimgmtYamlMissingFile(t *testing.T) {
60+
dir := t.TempDir()
61+
path := filepath.Join(dir, ".ci-mgmt.yaml")
62+
if _, err := newCimgmtYaml(path); err == nil {
63+
t.Fatalf("expected error when file missing")
64+
}
65+
}
Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
package migrations
2+
3+
import (
4+
"fmt"
5+
"path/filepath"
6+
"strings"
7+
)
8+
9+
// migrate any ci-mgmt.yml overrides to the top level mise.toml override file
10+
type migrateCimgmtOverrides struct{}
11+
12+
func (migrateCimgmtOverrides) Name() string {
13+
return "Migrate entries from .ci-mgmt.yml to the top level mise.toml override file"
14+
}
15+
func (migrateCimgmtOverrides) ShouldRun(templateName string) bool {
16+
return true
17+
}
18+
19+
// This currently migrates the tool overrides from .ci-mgmt.yaml to a root level
20+
// mise.toml. It can be extended to migrate other fields as well.
21+
func (migrateCimgmtOverrides) Migrate(templateName, outDir string) error {
22+
ciMgmtPath := filepath.Join(outDir, ".ci-mgmt.yaml")
23+
cimgmt, err := newCimgmtYaml(ciMgmtPath)
24+
if err != nil {
25+
return err
26+
}
27+
28+
misePath := filepath.Join(outDir, "mise.toml")
29+
mise, err := newTomlFile(misePath)
30+
if err != nil {
31+
return err
32+
}
33+
34+
toolVersions := cimgmt.getFieldNode("toolVersions")
35+
// if we don't override any toolVersions then we don't need to do anything
36+
if toolVersions == nil {
37+
return nil
38+
}
39+
40+
if len(mise.content) == 0 {
41+
mise.content = []byte("# Overwrites mise configuration at .config/mise.toml\n[tools]\n")
42+
}
43+
44+
miseTools := []sectionEntry{}
45+
46+
// convert any toolVersions overrides to mise tool entries
47+
toolVersionsMap := nodeToMap(toolVersions)
48+
for tool, version := range toolVersionsMap {
49+
if tool == "go" {
50+
// don't use go overrides anymore
51+
continue
52+
}
53+
version = strings.TrimSuffix(version, ".x")
54+
if tool == "java" {
55+
version = fmt.Sprintf("corretto-%s", version)
56+
}
57+
miseTools = append(miseTools, sectionEntry{
58+
key: tool,
59+
value: version,
60+
})
61+
}
62+
63+
updated, err := mise.ensureSectionEntries("tools", miseTools)
64+
if err != nil {
65+
return fmt.Errorf("error writing toolVersions to mise.toml: %w", err)
66+
}
67+
if updated {
68+
err := mise.writeFile()
69+
if err != nil {
70+
return err
71+
}
72+
}
73+
74+
// Finally remove the toolVersions from .ci-mgmt.yaml
75+
cimgmt.deleteKey("toolVersions")
76+
return cimgmt.writeFile()
77+
}

0 commit comments

Comments
 (0)