Skip to content
GitLab
Menu
Projects
Groups
Snippets
Help
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Menu
Open sidebar
truenas-rk3588
k3s
Commits
d49ef317
Commit
d49ef317
authored
5 years ago
by
galal-hussein
Browse files
Options
Download
Email Patches
Plain Diff
Inject node config on startup
parent
f3606776
No related merge requests found
Changes
3
Hide whitespace changes
Inline
Side-by-side
Showing
3 changed files
pkg/agent/run.go
+11
-2
pkg/agent/run.go
pkg/nodeconfig/nodeconfig.go
+111
-0
pkg/nodeconfig/nodeconfig.go
pkg/nodeconfig/nodeconfig_test.go
+86
-0
pkg/nodeconfig/nodeconfig_test.go
with
208 additions
and
2 deletions
+208
-2
pkg/agent/run.go
View file @
d49ef317
...
...
@@ -23,6 +23,7 @@ import (
"github.com/rancher/k3s/pkg/clientaccess"
"github.com/rancher/k3s/pkg/daemons/agent"
daemonconfig
"github.com/rancher/k3s/pkg/daemons/config"
"github.com/rancher/k3s/pkg/nodeconfig"
"github.com/rancher/k3s/pkg/rootless"
"github.com/sirupsen/logrus"
"k8s.io/apimachinery/pkg/api/equality"
...
...
@@ -71,7 +72,7 @@ func run(ctx context.Context, cfg cmds.Agent, lb *loadbalancer.LoadBalancer) err
}
}
if
err
:=
syncLabels
(
ctx
,
&
nodeConfig
.
AgentConfig
,
coreClient
.
CoreV1
()
.
Nodes
());
err
!=
nil
{
if
err
:=
configureNode
(
ctx
,
&
nodeConfig
.
AgentConfig
,
coreClient
.
CoreV1
()
.
Nodes
());
err
!=
nil
{
return
err
}
...
...
@@ -155,7 +156,7 @@ func validate() error {
return
nil
}
func
syncLabels
(
ctx
context
.
Context
,
agentConfig
*
daemonconfig
.
Agent
,
nodes
v1
.
NodeInterface
)
error
{
func
configureNode
(
ctx
context
.
Context
,
agentConfig
*
daemonconfig
.
Agent
,
nodes
v1
.
NodeInterface
)
error
{
for
{
node
,
err
:=
nodes
.
Get
(
agentConfig
.
NodeName
,
metav1
.
GetOptions
{})
if
err
!=
nil
{
...
...
@@ -171,8 +172,16 @@ func syncLabels(ctx context.Context, agentConfig *daemonconfig.Agent, nodes v1.N
newLabels
,
updateAddresses
=
updateAddressLabels
(
agentConfig
,
newLabels
)
}
// inject node config
updateNode
,
err
:=
nodeconfig
.
SetNodeConfigAnnotations
(
node
)
if
err
!=
nil
{
return
err
}
if
updateAddresses
||
updateMutables
{
node
.
Labels
=
newLabels
updateNode
=
true
}
if
updateNode
{
if
_
,
err
:=
nodes
.
Update
(
node
);
err
!=
nil
{
logrus
.
Infof
(
"Failed to update node %s: %v"
,
agentConfig
.
NodeName
,
err
)
select
{
...
...
This diff is collapsed.
Click to expand it.
pkg/nodeconfig/nodeconfig.go
0 → 100644
View file @
d49ef317
package
nodeconfig
import
(
"crypto/sha256"
"encoding/base32"
"encoding/json"
"fmt"
"os"
"strings"
"github.com/pkg/errors"
corev1
"k8s.io/api/core/v1"
)
const
(
NodeArgsAnnotation
=
"k3s.io/node-args"
NodeEnvAnnotation
=
"k3s.io/node-env"
NodeConfigHashAnnotation
=
"k3s.io/node-config-hash"
)
func
getNodeArgs
()
(
string
,
error
)
{
nodeArgsList
:=
[]
string
{}
for
_
,
arg
:=
range
os
.
Args
[
1
:
]
{
if
strings
.
Contains
(
arg
,
"="
)
{
parsedArg
:=
strings
.
SplitN
(
arg
,
"="
,
2
)
nodeArgsList
=
append
(
nodeArgsList
,
parsedArg
...
)
continue
}
nodeArgsList
=
append
(
nodeArgsList
,
arg
)
}
for
i
,
arg
:=
range
nodeArgsList
{
if
isSecret
(
arg
)
{
if
i
+
1
<
len
(
nodeArgsList
)
{
nodeArgsList
[
i
+
1
]
=
""
}
}
}
nodeArgs
,
err
:=
json
.
Marshal
(
nodeArgsList
)
if
err
!=
nil
{
return
""
,
errors
.
Wrap
(
err
,
"Failed to retrieve argument list for node"
)
}
return
string
(
nodeArgs
),
nil
}
func
getNodeEnv
()
(
string
,
error
)
{
k3sEnv
:=
make
(
map
[
string
]
string
)
for
_
,
v
:=
range
os
.
Environ
()
{
keyValue
:=
strings
.
SplitN
(
v
,
"="
,
2
)
if
strings
.
HasPrefix
(
keyValue
[
0
],
"K3S_"
)
{
k3sEnv
[
keyValue
[
0
]]
=
keyValue
[
1
]
}
}
for
key
:=
range
k3sEnv
{
if
isSecret
(
key
)
{
k3sEnv
[
key
]
=
""
}
}
k3sEnvJSON
,
err
:=
json
.
Marshal
(
k3sEnv
)
if
err
!=
nil
{
return
""
,
errors
.
Wrap
(
err
,
"Failed to retrieve environment map for node"
)
}
return
string
(
k3sEnvJSON
),
nil
}
func
SetNodeConfigAnnotations
(
node
*
corev1
.
Node
)
(
bool
,
error
)
{
nodeArgs
,
err
:=
getNodeArgs
()
if
err
!=
nil
{
return
false
,
err
}
nodeEnv
,
err
:=
getNodeEnv
()
if
err
!=
nil
{
return
false
,
err
}
h
:=
sha256
.
New
()
_
,
err
=
h
.
Write
([]
byte
(
nodeArgs
+
nodeEnv
))
if
err
!=
nil
{
return
false
,
fmt
.
Errorf
(
"Failed to hash the node config: %v"
,
err
)
}
if
node
.
Annotations
==
nil
{
node
.
Annotations
=
make
(
map
[
string
]
string
)
}
configHash
:=
h
.
Sum
(
nil
)
encoded
:=
base32
.
StdEncoding
.
EncodeToString
(
configHash
[
:
])
if
node
.
Annotations
[
NodeConfigHashAnnotation
]
==
encoded
{
return
false
,
nil
}
node
.
Annotations
[
NodeEnvAnnotation
]
=
nodeEnv
node
.
Annotations
[
NodeArgsAnnotation
]
=
nodeArgs
node
.
Annotations
[
NodeConfigHashAnnotation
]
=
encoded
return
true
,
nil
}
func
isSecret
(
key
string
)
bool
{
secretData
:=
[]
string
{
"K3S_TOKEN"
,
"K3S_DATASTORE_ENDPOINT"
,
"K3S_AGENT_TOKEN"
,
"K3S_CLUSTER_SECRET"
,
"--token"
,
"-t"
,
"--agent-token"
,
"--datastore-endpoint"
,
"--cluster-secret"
,
}
for
_
,
secret
:=
range
secretData
{
if
key
==
secret
{
return
true
}
}
return
false
}
This diff is collapsed.
Click to expand it.
pkg/nodeconfig/nodeconfig_test.go
0 → 100644
View file @
d49ef317
package
nodeconfig
import
(
"os"
"testing"
corev1
"k8s.io/api/core/v1"
metav1
"k8s.io/apimachinery/pkg/apis/meta/v1"
)
var
FakeNodeWithNoAnnotation
=
&
corev1
.
Node
{
TypeMeta
:
metav1
.
TypeMeta
{
Kind
:
"Node"
,
APIVersion
:
"v1"
,
},
ObjectMeta
:
metav1
.
ObjectMeta
{
Name
:
"fakeNode-no-annotation"
,
},
}
var
FakeNodeWithAnnotation
=
&
corev1
.
Node
{
TypeMeta
:
metav1
.
TypeMeta
{
Kind
:
"Node"
,
APIVersion
:
"v1"
,
},
ObjectMeta
:
metav1
.
ObjectMeta
{
Name
:
"fakeNode-with-annotation"
,
Annotations
:
map
[
string
]
string
{
NodeArgsAnnotation
:
`["server","--no-flannel"]`
,
NodeEnvAnnotation
:
`{"K3S_NODE_NAME":"fakeNode-with-annotation"}`
,
NodeConfigHashAnnotation
:
"LNQOAOIMOQIBRMEMACW7LYHXUNPZADF6RFGOSPIHJCOS47UVUJAA===="
,
},
},
}
func
assertEqual
(
t
*
testing
.
T
,
a
interface
{},
b
interface
{})
{
if
a
!=
b
{
t
.
Fatalf
(
"[ %v != %v ]"
,
a
,
b
)
}
}
func
TestSetEmptyNodeConfigAnnotations
(
t
*
testing
.
T
)
{
os
.
Args
=
[]
string
{
"k3s"
,
"server"
,
"--no-flannel"
}
os
.
Setenv
(
"K3S_NODE_NAME"
,
"fakeNode-no-annotation"
)
nodeUpdated
,
err
:=
SetNodeConfigAnnotations
(
FakeNodeWithNoAnnotation
)
if
err
!=
nil
{
t
.
Fatalf
(
"Failed to set node config annotation: %v"
,
err
)
}
assertEqual
(
t
,
true
,
nodeUpdated
)
expectedArgs
:=
`["server","--no-flannel"]`
actualArgs
:=
FakeNodeWithNoAnnotation
.
Annotations
[
NodeArgsAnnotation
]
assertEqual
(
t
,
expectedArgs
,
actualArgs
)
expectedEnv
:=
`{"K3S_NODE_NAME":"fakeNode-no-annotation"}`
actualEnv
:=
FakeNodeWithNoAnnotation
.
Annotations
[
NodeEnvAnnotation
]
assertEqual
(
t
,
expectedEnv
,
actualEnv
)
expectedHash
:=
"MROOIJGRXUZ53BM74K76TZLRXQOLNNBNJBJOY7JJ22EAEUIBW7YA===="
actualHash
:=
FakeNodeWithNoAnnotation
.
Annotations
[
NodeConfigHashAnnotation
]
assertEqual
(
t
,
expectedHash
,
actualHash
)
}
func
TestSetExistingNodeConfigAnnotations
(
t
*
testing
.
T
)
{
// adding same config
os
.
Args
=
[]
string
{
"k3s"
,
"server"
,
"--no-flannel"
}
os
.
Setenv
(
"K3S_NODE_NAME"
,
"fakeNode-with-annotation"
)
nodeUpdated
,
err
:=
SetNodeConfigAnnotations
(
FakeNodeWithAnnotation
)
if
err
!=
nil
{
t
.
Fatalf
(
"Failed to set node config annotation: %v"
,
err
)
}
assertEqual
(
t
,
false
,
nodeUpdated
)
}
func
TestSetArgsWithEqual
(
t
*
testing
.
T
)
{
os
.
Args
=
[]
string
{
"k3s"
,
"server"
,
"--no-flannel"
,
"--write-kubeconfig-mode=777"
}
os
.
Setenv
(
"K3S_NODE_NAME"
,
"fakeNode-with-no-annotation"
)
nodeUpdated
,
err
:=
SetNodeConfigAnnotations
(
FakeNodeWithNoAnnotation
)
if
err
!=
nil
{
t
.
Fatalf
(
"Failed to set node config annotation: %v"
,
err
)
}
assertEqual
(
t
,
true
,
nodeUpdated
)
expectedArgs
:=
`["server","--no-flannel","--write-kubeconfig-mode","777"]`
actualArgs
:=
FakeNodeWithNoAnnotation
.
Annotations
[
NodeArgsAnnotation
]
assertEqual
(
t
,
expectedArgs
,
actualArgs
)
}
This diff is collapsed.
Click to expand it.
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment
Menu
Projects
Groups
Snippets
Help