k8s 认证机制源码分析_token-auth-file

当kube-apiserver收到http请求后，会经过认证，鉴权和准入控制这三个和权限相关的模块的处理，本文主要分析认证模块。
认证模块主要依据请求头携带的token或证书等信息进行认证，支持x509证书，用户名和密码(从1.19已不再支持)，token，webhook和匿名认证等方式，具体的可查看官网，可在kube-apiserver进程启动时配置多种认证，当收到请求时，依次遍历这些认证，有一个认证成功则认为请求通过，并返回user信息，以便鉴权模块根据user信息进行处理，如果认证全部失败，则返回401(Unauthorized)。

如何配置
认证方法比较多，这里选择几种容易理解或常用的进行介绍。

a. 客户端证书认证: 在kube-apiserver进程启动时，通过添加–client-ca-file=SOMEFILE指定客户端证书即可启用。
b. token认证: 在kube-apiserver进程启动时，添加–token-auth-file=SOMEFILE可启动token认证，如果修改了token文件，apiserver进程必须重启才能生效。token文件是一种csv文件，包含最少三列：token,user,uid,“group1,group2,group3”。
c. 匿名认证: 默认是启用的，也可在在kube-apiserver进程启动时通过–anonymous-auth=true/false启用或关闭匿名访问，如果没配置其他任何认证方法时，匿名访问也是默认启用的。未被其他认证方法拒绝的请求将被视为匿名请求，并被赋予system:anonymous的用户名和system:unuthenticated的组名。例如，在配置了token认证和匿名认证情况下，如果提供token无效的请求将收到401 Unauthorized错误，但如果不提供任何token，则将被视为匿名请求。
需要注意的是，如果授权方式为AlwaysAllow，则匿名访问会自动关闭(即使通过参数–anonymous-auth=true启用匿名访问了，也会被自动关闭)，相关代码可参考函数ApplyAuthorization。

认证相关的命令行选项
认证相关命令行选项定义在文件pkg/kubeapiserver/options/authentication.go，初始化流程如下

NewAPIServerCommand
	//初始化选项变量
	//cmd/kube-apiserver/app/options/options.go
	s := options.NewServerRunOptions()
		s := ServerRunOptions{
			Admission:               kubeoptions.NewAdmissionOptions(),
			Authentication:          kubeoptions.NewBuiltInAuthenticationOptions().WithAll(),
			Authorization:           kubeoptions.NewBuiltInAuthorizationOptions(),
		}
		return &s
	...
	//添加命令行参数
	namedFlagSets := s.Flags()
		s.Authentication.AddFlags(fss.FlagSet("authentication"))
			//将是否启动匿名认证保存到o.Anonymous.Allow
			if o.Anonymous != nil {
				fs.BoolVar(&o.Anonymous.Allow, "anonymous-auth", o.Anonymous.Allow, ""+
					"Enables anonymous requests to the secure port of the API server. "+
					"Requests that are not rejected by another authentication method are treated as anonymous requests. "+
					"Anonymous requests have a username of system:anonymous, and a group name of system:unauthenticated.")
			}
			//将客户端证书保存到o.ClientCert.ClientCA
			if o.ClientCert != nil {
				o.ClientCert.AddFlags(fs)
					fs.StringVar(&s.ClientCA, "client-ca-file", s.ClientCA, ""+
						"If set, any request presenting a client certificate signed by one of "+
						"the authorities in the client-ca-file is authenticated with an identity "+
						"corresponding to the CommonName of the client certificate.")
			}
			//将token认证的文件保存到o.TokenFile.TokenFile
			if o.TokenFile != nil {
				fs.StringVar(&o.TokenFile.TokenFile, "token-auth-file", o.TokenFile.TokenFile, ""+
					"If set, the file that will be used to secure the secure port of the API server "+
					"via token authentication.")
			}
			...
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36

kubeoptions.NewBuiltInAuthenticationOptions().WithAll()用来初始化认证相关选项

// NewBuiltInAuthenticationOptions create a new BuiltInAuthenticationOptions, just set default token cache TTL
func NewBuiltInAuthenticationOptions() *BuiltInAuthenticationOptions {
	return &BuiltInAuthenticationOptions{
		TokenSuccessCacheTTL: 10 * time.Second,
		TokenFailureCacheTTL: 0 * time.Second,
	}
}

// WithAll set default value for every build-in authentication option
func (o *BuiltInAuthenticationOptions) WithAll() *BuiltInAuthenticationOptions {
	return o.
		WithAnonymous().
		WithBootstrapToken().
		WithClientCert().
		WithOIDC().
		WithRequestHeader().
		WithServiceAccounts().
		WithTokenFile().
		WithWebHook()
}

// WithAnonymous set default value for anonymous authentication
func (o *BuiltInAuthenticationOptions) WithAnonymous() *BuiltInAuthenticationOptions {
	//默认启动匿名认证
	o.Anonymous = &AnonymousAuthenticationOptions{Allow: true}
	return o
}

// WithClientCert set default value for client cert
func (o *BuiltInAuthenticationOptions) WithClientCert() *BuiltInAuthenticationOptions {
	o.ClientCert = &genericoptions.ClientCertAuthenticationOptions{}
	return o
}

// WithTokenFile set default value for token file authentication
func (o *BuiltInAuthenticationOptions) WithTokenFile() *BuiltInAuthenticationOptions {
	o.TokenFile = &TokenFileAuthenticationOptions{}
	return o
}
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39

上面的选项总结如下

构造认证配置
这里是根据前面的命令行选项构造认证配置

type AuthenticationInfo struct {
	// APIAudiences is a list of identifier that the API identifies as. This is
	// used by some authenticators to validate audience bound credentials.
	APIAudiences authenticator.Audiences
	// Authenticator determines which subject is making the request
	Authenticator authenticator.Request
}

//每种认证方法必须实现AuthenticateRequest接口
// Request attempts to extract authentication information from a request and
// returns a Response or an error if the request could not be checked.
type Request interface {
	AuthenticateRequest(req *http.Request) (*Response, bool, error)
}
1
2
3
4
5
6
7
8
9
10
11
12
13
14

构造过程如下

buildGenericConfig(...)
	s.Authentication.ApplyTo(&genericConfig.Authentication, genericConfig.SecureServing, genericConfig.EgressSelector, genericConfig.OpenAPIConfig, clientgoExternalClient, versionedInformers)

// ApplyTo requires already applied OpenAPIConfig and EgressSelector if present.
func (o *BuiltInAuthenticationOptions) ApplyTo(authInfo *genericapiserver.AuthenticationInfo, secureServing *genericapiserver.SecureServingInfo, egressSelector *egressselector.EgressSelector, openAPIConfig *openapicommon.Config, extclient kubernetes.Interface, versionedInformer informers.SharedInformerFactory) error {
	//根据BuiltInAuthenticationOptions构造authenticatorConfig
	authenticatorConfig, err := o.ToAuthenticationConfig()
	...
	//认证方法初始化，见下面注释
	authInfo.Authenticator, openAPIConfig.SecurityDefinitions, err = authenticatorConfig.New()

	return nil
}
1
2
3
4
5
6
7
8
9
10
11
12
13

根据配置信息将配置的认证方法进行初始化

// New returns an authenticator.Request or an error that supports the standard
// Kubernetes authentication mechanisms.
func (config Config) New() (authenticator.Request, *spec.SecurityDefinitions, error) {
	var authenticators []authenticator.Request
	var tokenAuthenticators []authenticator.Token
	securityDefinitions := spec.SecurityDefinitions{}

	//初始化证书认证
	// X509 methods
	if config.ClientCAContentProvider != nil {
		certAuth := x509.NewDynamic(config.ClientCAContentProvider.VerifyOptions, x509.CommonNameUserConversion)
		authenticators = append(authenticators, certAuth)
	}

	//初始化token认证
	// Bearer token methods, local first, then remote
	if len(config.TokenAuthFile) > 0 {
		tokenAuth, err := newAuthenticatorFromTokenFile(config.TokenAuthFile)
		if err != nil {
			return nil, nil, err
		}
		tokenAuthenticators = append(tokenAuthenticators, authenticator.WrapAudienceAgnosticToken(config.APIAudiences, tokenAuth))
	}
	...
	if len(tokenAuthenticators) > 0 {
		// Union the token authenticators
		//将配置的可以通过token认证的方法统一放到unionAuthTokenHandler中
		tokenAuth := tokenunion.New(tokenAuthenticators...)
		// Optionally cache authentication results
		if config.TokenSuccessCacheTTL > 0 || config.TokenFailureCacheTTL > 0 {
			tokenAuth = tokencache.New(tokenAuth, true, config.TokenSuccessCacheTTL, config.TokenFailureCacheTTL)
		}
		authenticators = append(authenticators, bearertoken.New(tokenAuth), websocket.NewProtocolAuthenticator(tokenAuth))
		...
	}

	//如果没配置任何认证方法，并且配置了匿名认证，则启用匿名认证
	if len(authenticators) == 0 {
		if config.Anonymous {
			return anonymous.NewAuthenticator(), &securityDefinitions, nil
		}
		return nil, &securityDefinitions, nil
	}

	//将配置的认证方法统一到unionAuthRequestHandler中
	authenticator := union.New(authenticators...)

	//又将authenticator保存到AuthenticatedGroupAdder中，如果认证成功后，AuthenticatedGroupAdder会额外增加组system:authenticated
	authenticator = group.NewAuthenticatedGroupAdder(authenticator)

	if config.Anonymous {
		// If the authenticator chain returns an error, return an error (don't consider a bad bearer token
		// or invalid username/password combination anonymous).
		authenticator = union.NewFailOnError(authenticator, anonymous.NewAuthenticator())
	}

	return authenticator, &securityDefinitions, nil
}
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58

经过上面的初始化后，可总结如下图，所有配置的认证方法都保存到AuthenticationInfo->Authenticator中，其为Request接口类型，实现此接口的结构体为AuthenticatedGroupAdder，它的AuthenticateRequest方法没有实现认证功能，其作用是给经过认证的请求添加组system:authenticated，其成员变量Authenticator也是Request接口类型，对应的结构体为unionAuthRequestHandler，其中Handlers为authenticator.Request类型的数组，每个数组元素保存一种认证方法，其中的unionAuthTokenHandler又保存了多种token认证方法。

执行认证
处理http请求流程如下，认证流程是在FullHandlerChain中完成的

aggregatorserver FullHandlerChain -> aggregatorserver director -> apiserver director -> extensionserver director -> NotFound
1

认证处理函数是在DefaultBuildHandlerChain中注册，其为withAuthentication的返回值http.HandlerFunc

DefaultBuildHandlerChain
	failedHandler := genericapifilters.Unauthorized(c.Serializer)
	handler = genericapifilters.WithAuthentication(handler, c.Authentication.Authenticator, failedHandler, c.Authentication.APIAudiences)

//k8s.io/apiserver/pkg/endpoints/filters/authentication.go
// WithAuthentication creates an http handler that tries to authenticate the given request as a user, and then
// stores any such user found onto the provided context for the request. If authentication fails or returns an error
// the failed handler is used. On success, "Authorization" header is removed from the request and handler
// is invoked to serve the request.
func WithAuthentication(handler http.Handler, auth authenticator.Request, failed http.Handler, apiAuds authenticator.Audiences) http.Handler {
	//返回认证处理函数
	return withAuthentication(handler, auth, failed, apiAuds, recordAuthMetrics)
}

func withAuthentication(handler http.Handler, auth authenticator.Request, failed http.Handler, apiAuds authenticator.Audiences, metrics recordMetrics) http.Handler {
	if auth == nil {
		klog.Warning("Authentication is disabled")
		return handler
	}
	//认证处理函数
	return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
		authenticationStart := time.Now()

		if len(apiAuds) > 0 {
			req = req.WithContext(authenticator.WithAudiences(req.Context(), apiAuds))
		}
		//执行认证处理流程，见后面注释
		resp, ok, err := auth.AuthenticateRequest(req)
		authenticationFinish := time.Now()
		defer func() {
			metrics(req.Context(), resp, ok, err, apiAuds, authenticationStart, authenticationFinish)
		}()
		//如果认证失败，则返回401
		if err != nil || !ok {
			if err != nil {
				klog.ErrorS(err, "Unable to authenticate the request")
			}
			failed.ServeHTTP(w, req)
			return
		}
		...
		//认证成功后，将Authorization字段从请求头里删除
		// authorization header is not required anymore in case of a successful authentication.
		req.Header.Del("Authorization")

		//将认证成功后的user信息保存到请求上下文中，下一阶段会根据user信息进行鉴权
		req = req.WithContext(genericapirequest.WithUser(req.Context(), resp.User))
		handler.ServeHTTP(w, req)
	})
}
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50

前面初始化时，Authenticator指向AuthenticatedGroupAdder，所以auth.AuthenticateRequest(req)为AuthenticatedGroupAdder的AuthenticateRequest方法

//k8s.io/apiserver/pkg/authentication/group/authenticated_group_adder.go
func (g *AuthenticatedGroupAdder) AuthenticateRequest(req *http.Request) (*authenticator.Response, bool, error) {
	//调用unionAuthRequestHandler的AuthenticateRequest
	r, ok, err := g.Authenticator.AuthenticateRequest(req)
	if err != nil || !ok {
		return nil, ok, err
	}
	...
	r.User = &user.DefaultInfo{
		Name:   r.User.GetName(),
		UID:    r.User.GetUID(),
		//追加组system:authenticated
		Groups: append(r.User.GetGroups(), user.AllAuthenticated),
		Extra:  r.User.GetExtra(),
	}
	return r, true, nil
}
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17

unionAuthRequestHandler遍历启用的认证方法进行认证

//k8s.io/apiserver/pkg/authentication/request/union/unios.go: AuthenticateRequest
// AuthenticateRequest authenticates the request using a chain of authenticator.Request objects.
func (authHandler *unionAuthRequestHandler) AuthenticateRequest(req *http.Request) (*authenticator.Response, bool, error) {
	var errlist []error
	//遍历所有配置的认证方法，执行其AuthenticateRequest函数，有一个成功就返回成功
	for _, currAuthRequestHandler := range authHandler.Handlers {
		resp, ok, err := currAuthRequestHandler.AuthenticateRequest(req)
		//认证失败，继续下一个认证
		if err != nil {
			if authHandler.FailOnError {
				return resp, ok, err
			}
			errlist = append(errlist, err)
			continue
		}

		if ok {
			return resp, ok, err
		}
	}

	return nil, false, utilerrors.NewAggregate(errlist)
}
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23

下面举两个认证方法的例子: 证书认证和token认证。
证书认证处理函数

//k8s.io/apiserver/pkg/Authenticator/request/x509/x509.go
// AuthenticateRequest authenticates the request using presented client certificates
func (a *Authenticator) AuthenticateRequest(req *http.Request) (*authenticator.Response, bool, error) {
	if req.TLS == nil || len(req.TLS.PeerCertificates) == 0 {
		return nil, false, nil
	}

	// Use intermediates, if provided
	optsCopy, ok := a.verifyOptionsFn()
	// if there are intentionally no verify options, then we cannot authenticate this request
	if !ok {
		return nil, false, nil
	}
	if optsCopy.Intermediates == nil && len(req.TLS.PeerCertificates) > 1 {
		optsCopy.Intermediates = x509.NewCertPool()
		for _, intermediate := range req.TLS.PeerCertificates[1:] {
			optsCopy.Intermediates.AddCert(intermediate)
		}
	}

	remaining := req.TLS.PeerCertificates[0].NotAfter.Sub(time.Now())
	clientCertificateExpirationHistogram.WithContext(req.Context()).Observe(remaining.Seconds())
	chains, err := req.TLS.PeerCertificates[0].Verify(optsCopy)
	if err != nil {
		return nil, false, fmt.Errorf(
			"verifying certificate %s failed: %w",
			certificateIdentifier(req.TLS.PeerCertificates[0]),
			err,
		)
	}

	var errlist []error
	for _, chain := range chains {
		user, ok, err := a.user.User(chain)
		if err != nil {
			errlist = append(errlist, err)
			continue
		}

		if ok {
			return user, ok, err
		}
	}
	return nil, false, utilerrors.NewAggregate(errlist)
}
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45

token认证处理函数

//k8s.io/apiserver/pkg/Authenticator/bearertoken/bearertoken.go
func (a *Authenticator) AuthenticateRequest(req *http.Request) (*authenticator.Response, bool, error) {
	//获取请求头里的认证信息，格式为：Authorization: bearer XXXX
	//经过处理后(去除最开始和后面的空格)，auth应该为：bearer XXXX
	auth := strings.TrimSpace(req.Header.Get("Authorization"))
	if auth == "" {
		return nil, false, nil
	}
	//将认证信息按空格分开
	parts := strings.SplitN(auth, " ", 3)
	//parts不能小于2，parts[0]必须是bearer
	if len(parts) < 2 || strings.ToLower(parts[0]) != "bearer" {
		return nil, false, nil
	}

	token := parts[1]

	// Empty bearer tokens aren't valid
	if len(token) == 0 {
		return nil, false, nil
	}
	//调用 AuthenticateToken，进行token认证
	resp, ok, err := a.auth.AuthenticateToken(req.Context(), token)
	// if we authenticated successfully, go ahead and remove the bearer token so that no one
	// is ever tempted to use it inside of the API server
	if ok {
		req.Header.Del("Authorization")
	}

	// If the token authenticator didn't error, provide a default error
	if !ok && err == nil {
		err = invalidToken
	}

	return resp, ok, err
}
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36

调用 AuthenticateToken，进行token认证

//k8s.io/apiserver/pkg/Authenticator/token/union/union.go
// AuthenticateToken authenticates the token using a chain of authenticator.Token objects.
func (authHandler *unionAuthTokenHandler) AuthenticateToken(ctx context.Context, token string) (*authenticator.Response, bool, error) {
	var errlist []error
	//遍历token列表，有一个成功则返回
	for _, currAuthRequestHandler := range authHandler.Handlers {
		info, ok, err := currAuthRequestHandler.AuthenticateToken(ctx, token)
		if err != nil {
			if authHandler.FailOnError {
				return info, ok, err
			}
			errlist = append(errlist, err)
			continue
		}

		if ok {
			return info, ok, err
		}
	}

	return nil, false, utilerrors.NewAggregate(errlist)
}
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22

举一个token认证的例子

//k8s.io/apiserver/pkg/Authenticator/token/tokenfile/tokenfile.go
func (a *TokenAuthenticator) AuthenticateToken(ctx context.Context, value string) (*authenticator.Response, bool, error) {
	//token是否存在，如果不存在说明认证失败
	//a.tokens在newAuthenticatorFromTokenFile中通过解析配置文件获取，保存的是token对应的user信息
	user, ok := a.tokens[value]
	if !ok {
		return nil, false, nil
	}
	//token存在说明认证成功，返回user信息
	return &authenticator.Response{User: user}, true, nil
}
1
2
3
4
5
6
7
8
9
10
11