cert: Name the post-quantum algorithms of a version 3 certificate with a PQSuite, like its curve, and pick the hybrid subtype by the suite's KEM

handshake/patterns.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 
 	"github.com/DefinedNet/noise"
+	"github.com/slackhq/nebula/cert"
 	"github.com/slackhq/nebula/header"
 )
 
@@ -22,9 +23,10 @@ type subtypeInfo struct {
 	pskPlacement int // noise psk modifier position, 0 means no psk token
 	msgs         []msgFlags
 
-	// hybrid patterns carry composite static keys and need version 3
-	// certificates on both sides; every other pattern refuses them.
-	hybrid bool
+	// kem is the KEM of a hybrid pattern, nil for any other. Hybrid patterns
+	// carry composite static keys and need version 3 certificates on both
+	// sides, in suites using this KEM; every other pattern refuses them.
+	kem noise.KEMFunc
 }
 
 // subtypeInfos defines the noise pattern and message content layout for each
@@ -68,7 +70,7 @@ var subtypeInfos = map[header.MessageSubType]subtypeInfo{
 	// first packet under the new keys does.
 	header.HandshakeIXHybridMLKEM1024: {
 		pattern: noise.HandshakeHybridIX,
-		hybrid:  true,
+		kem:     KEMMLKEM1024,
 		msgs: []msgFlags{
 			{expectsPayload: true, expectsCert: true},
 			{expectsPayload: true, expectsCert: true},
@@ -90,6 +92,26 @@ var subtypeInfos = map[header.MessageSubType]subtypeInfo{
 	//},
 }
 
+// hybridSubtypes maps the suite of a version 3 certificate to the hybrid
+// subtype for its KEM. Peers need the same KEM, not the same suite, so suites
+// sharing a KEM share a subtype; the suite itself travels in the certificate.
+var hybridSubtypes = map[cert.PQSuite]header.MessageSubType{
+	cert.PQSuite_MLKEM1024_MLDSA87: header.HandshakeIXHybridMLKEM1024,
+}
+
+// HybridSubtype returns the hybrid handshake subtype for the KEM of a version 3
+// certificate's suite.
+func HybridSubtype(suite cert.PQSuite) (header.MessageSubType, bool) {
+	st, ok := hybridSubtypes[suite]
+	return st, ok
+}
+
+// IsHybrid reports whether subtype is a hybrid handshake, one that needs version
+// 3 certificates.
+func IsHybrid(subtype header.MessageSubType) bool {
+	return subtypeInfos[subtype].kem != nil
+}
+
 func subtypeInfoFor(subtype header.MessageSubType) (subtypeInfo, error) {
 	if info, ok := subtypeInfos[subtype]; ok {
 		return info, nil