diff --git a/cmd/nebula-monitor/nebula-monitor.go b/cmd/nebula-monitor/nebula-monitor.go new file mode 100644 index 00000000..e9c88d37 --- /dev/null +++ b/cmd/nebula-monitor/nebula-monitor.go @@ -0,0 +1,39 @@ +package main + +import ( + "fmt" + "io" + "log" + "net/http" +) + +func handlePost(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) + return + } + + // Read the body + body, err := io.ReadAll(r.Body) + if err != nil { + http.Error(w, "Error reading body", http.StatusInternalServerError) + return + } + defer r.Body.Close() + + // Print to console + //fmt.Printf("Path: %s\n", r.URL.Path) + //fmt.Printf("Headers: %v\n", r.Header) + fmt.Printf("%s\n", string(body)) + + // Send response + w.WriteHeader(http.StatusOK) + w.Write([]byte("")) +} + +func main() { + http.HandleFunc("/", handlePost) + + fmt.Println("Server starting on :8080") + log.Fatal(http.ListenAndServe(":8080", nil)) +} diff --git a/control.go b/control.go index f8567b50..91df18d4 100644 --- a/control.go +++ b/control.go @@ -1,7 +1,11 @@ package nebula import ( + "bytes" "context" + "encoding/json" + "io" + "net/http" "net/netip" "os" "os/signal" @@ -9,6 +13,7 @@ import ( "github.com/sirupsen/logrus" "github.com/slackhq/nebula/cert" + "github.com/slackhq/nebula/firewall" "github.com/slackhq/nebula/header" "github.com/slackhq/nebula/overlay" ) @@ -71,10 +76,51 @@ func (c *Control) Start() { c.lighthouseStart() } + go c.firewallEventSender(c.ctx) + // Start reading packets. c.f.run() } +func (c *Control) firewallEventSender(ctx context.Context) { + events := make([]firewall.Event, 1, 100) //todo configurable + for { + select { + //todo exceptionally lazy + case e := <-c.f.events: + events = append(events, e) + for { //slurp up everything into one big post + x := false + select { + case e = <-c.f.events: + events = append(events, e) + default: + x = true + } + if x { + break + } + } + if len(events) >= 5 { + out, err := json.Marshal(&events) + if err != nil { + c.l.WithError(err).Errorf("failed to marshal event") + } else { + r := io.Reader(bytes.NewBuffer(out)) + _, err = http.Post("http://127.0.0.1:8080/nebula/event", "application/json", r) + if err != nil { + c.l.WithError(err).Errorf("failed to post event") + } + } + events = events[:0] + } + + case <-ctx.Done(): + return + } + } +} + func (c *Control) Context() context.Context { return c.ctx } diff --git a/firewall.go b/firewall.go index 45dc0691..bd9b3df7 100644 --- a/firewall.go +++ b/firewall.go @@ -45,6 +45,8 @@ type Firewall struct { InSendReject bool OutSendReject bool + InLogDrop bool + OutLogDrop bool //TODO: we should have many more options for TCP, an option for ICMP, and mimic the kernel a bit better // https://www.kernel.org/doc/Documentation/networking/nf_conntrack-sysctl.txt @@ -216,6 +218,7 @@ func NewFirewallFromConfig(l *logrus.Logger, cs *CertState, c *config.C) (*Firew switch inboundAction { case "reject": fw.InSendReject = true + fw.InLogDrop = true //todo case "drop": fw.InSendReject = false default: @@ -227,6 +230,7 @@ func NewFirewallFromConfig(l *logrus.Logger, cs *CertState, c *config.C) (*Firew switch outboundAction { case "reject": fw.OutSendReject = true + fw.OutLogDrop = true //todo case "drop": fw.OutSendReject = false default: @@ -401,9 +405,11 @@ var ErrInvalidRemoteIP = errors.New("remote address is not in remote certificate var ErrInvalidLocalIP = errors.New("local address is not in list of handled local addresses") var ErrNoMatchingRule = errors.New("no matching rule in firewall table") +type DropHandler func(fp firewall.Packet, incoming bool, h *HostInfo, err error) + // Drop returns an error if the packet should be dropped, explaining why. It // returns nil if the packet should not be dropped. -func (f *Firewall) Drop(fp firewall.Packet, incoming bool, h *HostInfo, caPool *cert.CAPool, localCache firewall.ConntrackCache) error { +func (f *Firewall) Drop(fp firewall.Packet, incoming bool, h *HostInfo, caPool *cert.CAPool, localCache firewall.ConntrackCache, onDrop DropHandler) error { // Check if we spoke to this tuple, if we did then allow this packet if f.inConns(fp, h, caPool, localCache) { return nil diff --git a/firewall/event.go b/firewall/event.go new file mode 100644 index 00000000..6a49d5ba --- /dev/null +++ b/firewall/event.go @@ -0,0 +1,15 @@ +package firewall + +import ( + "net/netip" + "time" +) + +type Event struct { + Packet Packet `json:"packet,omitempty"` + At time.Time `json:"at,omitempty"` + Remote netip.AddrPort `json:"remote,omitempty"` + //todo cert info? + //todo connection indexes? + //todo underlay info would actually be amazing, for inbounds +} diff --git a/firewall/packet.go b/firewall/packet.go index 40c7fc5d..4a2341b9 100644 --- a/firewall/packet.go +++ b/firewall/packet.go @@ -44,7 +44,7 @@ func (fp Packet) MarshalJSON() ([]byte, error) { switch fp.Protocol { case ProtoTCP: proto = "tcp" - case ProtoICMP: + case ProtoICMP, ProtoICMPv6: proto = "icmp" case ProtoUDP: proto = "udp" diff --git a/firewall_test.go b/firewall_test.go index 1df62a81..a5451a31 100644 --- a/firewall_test.go +++ b/firewall_test.go @@ -212,44 +212,44 @@ func TestFirewall_Drop(t *testing.T) { cp := cert.NewCAPool() // Drop outbound - assert.Equal(t, ErrNoMatchingRule, fw.Drop(p, false, &h, cp, nil)) + assert.Equal(t, ErrNoMatchingRule, fw.Drop(p, false, &h, cp, nil, nil)) // Allow inbound resetConntrack(fw) - require.NoError(t, fw.Drop(p, true, &h, cp, nil)) + require.NoError(t, fw.Drop(p, true, &h, cp, nil, nil)) // Allow outbound because conntrack - require.NoError(t, fw.Drop(p, false, &h, cp, nil)) + require.NoError(t, fw.Drop(p, false, &h, cp, nil, nil)) // test remote mismatch oldRemote := p.RemoteAddr p.RemoteAddr = netip.MustParseAddr("1.2.3.10") - assert.Equal(t, fw.Drop(p, false, &h, cp, nil), ErrInvalidRemoteIP) + assert.Equal(t, fw.Drop(p, false, &h, cp, nil, nil), ErrInvalidRemoteIP) p.RemoteAddr = oldRemote // ensure signer doesn't get in the way of group checks fw = NewFirewall(l, time.Second, time.Minute, time.Hour, &c) require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"nope"}, "", "", "", "", "signer-shasum")) require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"default-group"}, "", "", "", "", "signer-shasum-bad")) - assert.Equal(t, fw.Drop(p, true, &h, cp, nil), ErrNoMatchingRule) + assert.Equal(t, fw.Drop(p, true, &h, cp, nil, nil), ErrNoMatchingRule) // test caSha doesn't drop on match fw = NewFirewall(l, time.Second, time.Minute, time.Hour, &c) require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"nope"}, "", "", "", "", "signer-shasum-bad")) require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"default-group"}, "", "", "", "", "signer-shasum")) - require.NoError(t, fw.Drop(p, true, &h, cp, nil)) + require.NoError(t, fw.Drop(p, true, &h, cp, nil, nil)) // ensure ca name doesn't get in the way of group checks cp.CAs["signer-shasum"] = &cert.CachedCertificate{Certificate: &dummyCert{name: "ca-good"}} fw = NewFirewall(l, time.Second, time.Minute, time.Hour, &c) require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"nope"}, "", "", "", "ca-good", "")) require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"default-group"}, "", "", "", "ca-good-bad", "")) - assert.Equal(t, fw.Drop(p, true, &h, cp, nil), ErrNoMatchingRule) + assert.Equal(t, fw.Drop(p, true, &h, cp, nil, nil), ErrNoMatchingRule) // test caName doesn't drop on match cp.CAs["signer-shasum"] = &cert.CachedCertificate{Certificate: &dummyCert{name: "ca-good"}} fw = NewFirewall(l, time.Second, time.Minute, time.Hour, &c) require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"nope"}, "", "", "", "ca-good-bad", "")) require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"default-group"}, "", "", "", "ca-good", "")) - require.NoError(t, fw.Drop(p, true, &h, cp, nil)) + require.NoError(t, fw.Drop(p, true, &h, cp, nil, nil)) } func TestFirewall_DropV6(t *testing.T) { @@ -291,44 +291,44 @@ func TestFirewall_DropV6(t *testing.T) { cp := cert.NewCAPool() // Drop outbound - assert.Equal(t, ErrNoMatchingRule, fw.Drop(p, false, &h, cp, nil)) + assert.Equal(t, ErrNoMatchingRule, fw.Drop(p, false, &h, cp, nil, nil)) // Allow inbound resetConntrack(fw) - require.NoError(t, fw.Drop(p, true, &h, cp, nil)) + require.NoError(t, fw.Drop(p, true, &h, cp, nil, nil)) // Allow outbound because conntrack - require.NoError(t, fw.Drop(p, false, &h, cp, nil)) + require.NoError(t, fw.Drop(p, false, &h, cp, nil, nil)) // test remote mismatch oldRemote := p.RemoteAddr p.RemoteAddr = netip.MustParseAddr("fd12::56") - assert.Equal(t, fw.Drop(p, false, &h, cp, nil), ErrInvalidRemoteIP) + assert.Equal(t, fw.Drop(p, false, &h, cp, nil, nil), ErrInvalidRemoteIP) p.RemoteAddr = oldRemote // ensure signer doesn't get in the way of group checks fw = NewFirewall(l, time.Second, time.Minute, time.Hour, &c) require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"nope"}, "", "", "", "", "signer-shasum")) require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"default-group"}, "", "", "", "", "signer-shasum-bad")) - assert.Equal(t, fw.Drop(p, true, &h, cp, nil), ErrNoMatchingRule) + assert.Equal(t, fw.Drop(p, true, &h, cp, nil, nil), ErrNoMatchingRule) // test caSha doesn't drop on match fw = NewFirewall(l, time.Second, time.Minute, time.Hour, &c) require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"nope"}, "", "", "", "", "signer-shasum-bad")) require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"default-group"}, "", "", "", "", "signer-shasum")) - require.NoError(t, fw.Drop(p, true, &h, cp, nil)) + require.NoError(t, fw.Drop(p, true, &h, cp, nil, nil)) // ensure ca name doesn't get in the way of group checks cp.CAs["signer-shasum"] = &cert.CachedCertificate{Certificate: &dummyCert{name: "ca-good"}} fw = NewFirewall(l, time.Second, time.Minute, time.Hour, &c) require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"nope"}, "", "", "", "ca-good", "")) require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"default-group"}, "", "", "", "ca-good-bad", "")) - assert.Equal(t, fw.Drop(p, true, &h, cp, nil), ErrNoMatchingRule) + assert.Equal(t, fw.Drop(p, true, &h, cp, nil, nil), ErrNoMatchingRule) // test caName doesn't drop on match cp.CAs["signer-shasum"] = &cert.CachedCertificate{Certificate: &dummyCert{name: "ca-good"}} fw = NewFirewall(l, time.Second, time.Minute, time.Hour, &c) require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"nope"}, "", "", "", "ca-good-bad", "")) require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"default-group"}, "", "", "", "ca-good", "")) - require.NoError(t, fw.Drop(p, true, &h, cp, nil)) + require.NoError(t, fw.Drop(p, true, &h, cp, nil, nil)) } func BenchmarkFirewallTable_match(b *testing.B) { @@ -536,10 +536,10 @@ func TestFirewall_Drop2(t *testing.T) { cp := cert.NewCAPool() // h1/c1 lacks the proper groups - require.ErrorIs(t, fw.Drop(p, true, &h1, cp, nil), ErrNoMatchingRule) + require.ErrorIs(t, fw.Drop(p, true, &h1, cp, nil, nil), ErrNoMatchingRule) // c has the proper groups resetConntrack(fw) - require.NoError(t, fw.Drop(p, true, &h, cp, nil)) + require.NoError(t, fw.Drop(p, true, &h, cp, nil, nil)) } func TestFirewall_Drop3(t *testing.T) { @@ -617,18 +617,18 @@ func TestFirewall_Drop3(t *testing.T) { cp := cert.NewCAPool() // c1 should pass because host match - require.NoError(t, fw.Drop(p, true, &h1, cp, nil)) + require.NoError(t, fw.Drop(p, true, &h1, cp, nil, nil)) // c2 should pass because ca sha match resetConntrack(fw) - require.NoError(t, fw.Drop(p, true, &h2, cp, nil)) + require.NoError(t, fw.Drop(p, true, &h2, cp, nil, nil)) // c3 should fail because no match resetConntrack(fw) - assert.Equal(t, fw.Drop(p, true, &h3, cp, nil), ErrNoMatchingRule) + assert.Equal(t, fw.Drop(p, true, &h3, cp, nil, nil), ErrNoMatchingRule) // Test a remote address match fw = NewFirewall(l, time.Second, time.Minute, time.Hour, c.Certificate) require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 1, 1, []string{}, "", "1.2.3.4/24", "", "", "")) - require.NoError(t, fw.Drop(p, true, &h1, cp, nil)) + require.NoError(t, fw.Drop(p, true, &h1, cp, nil, nil)) } func TestFirewall_Drop3V6(t *testing.T) { @@ -666,7 +666,7 @@ func TestFirewall_Drop3V6(t *testing.T) { fw := NewFirewall(l, time.Second, time.Minute, time.Hour, c.Certificate) cp := cert.NewCAPool() require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 1, 1, []string{}, "", "fd12::34/120", "", "", "")) - require.NoError(t, fw.Drop(p, true, &h, cp, nil)) + require.NoError(t, fw.Drop(p, true, &h, cp, nil, nil)) } func TestFirewall_DropConntrackReload(t *testing.T) { @@ -708,12 +708,12 @@ func TestFirewall_DropConntrackReload(t *testing.T) { cp := cert.NewCAPool() // Drop outbound - assert.Equal(t, fw.Drop(p, false, &h, cp, nil), ErrNoMatchingRule) + assert.Equal(t, fw.Drop(p, false, &h, cp, nil, nil), ErrNoMatchingRule) // Allow inbound resetConntrack(fw) - require.NoError(t, fw.Drop(p, true, &h, cp, nil)) + require.NoError(t, fw.Drop(p, true, &h, cp, nil, nil)) // Allow outbound because conntrack - require.NoError(t, fw.Drop(p, false, &h, cp, nil)) + require.NoError(t, fw.Drop(p, false, &h, cp, nil, nil)) oldFw := fw fw = NewFirewall(l, time.Second, time.Minute, time.Hour, c.Certificate) @@ -722,7 +722,7 @@ func TestFirewall_DropConntrackReload(t *testing.T) { fw.rulesVersion = oldFw.rulesVersion + 1 // Allow outbound because conntrack and new rules allow port 10 - require.NoError(t, fw.Drop(p, false, &h, cp, nil)) + require.NoError(t, fw.Drop(p, false, &h, cp, nil, nil)) oldFw = fw fw = NewFirewall(l, time.Second, time.Minute, time.Hour, c.Certificate) @@ -731,7 +731,7 @@ func TestFirewall_DropConntrackReload(t *testing.T) { fw.rulesVersion = oldFw.rulesVersion + 1 // Drop outbound because conntrack doesn't match new ruleset - assert.Equal(t, fw.Drop(p, false, &h, cp, nil), ErrNoMatchingRule) + assert.Equal(t, fw.Drop(p, false, &h, cp, nil, nil), ErrNoMatchingRule) } func TestFirewall_DropIPSpoofing(t *testing.T) { @@ -777,7 +777,7 @@ func TestFirewall_DropIPSpoofing(t *testing.T) { Protocol: firewall.ProtoUDP, Fragment: false, } - assert.Equal(t, fw.Drop(p, true, &h1, cp, nil), ErrInvalidRemoteIP) + assert.Equal(t, fw.Drop(p, true, &h1, cp, nil, nil), ErrInvalidRemoteIP) } func BenchmarkLookup(b *testing.B) { @@ -1184,7 +1184,7 @@ func (c *testcase) Test(t *testing.T, fw *Firewall) { t.Helper() cp := cert.NewCAPool() resetConntrack(fw) - err := fw.Drop(c.p, true, c.h, cp, nil) + err := fw.Drop(c.p, true, c.h, cp, nil, nil) if c.err == nil { require.NoError(t, err, "failed to not drop remote address %s", c.p.RemoteAddr) } else { diff --git a/inside.go b/inside.go index 0d53f952..43c0169e 100644 --- a/inside.go +++ b/inside.go @@ -2,6 +2,7 @@ package nebula import ( "net/netip" + "time" "github.com/sirupsen/logrus" "github.com/slackhq/nebula/firewall" @@ -53,7 +54,7 @@ func (f *Interface) consumeInsidePacket(packet []byte, fwPacket *firewall.Packet }) if hostinfo == nil { - f.rejectInside(packet, out, q) + f.rejectInside(fwPacket, packet, out, q) if f.l.Level >= logrus.DebugLevel { f.l.WithField("vpnAddr", fwPacket.RemoteAddr). WithField("fwPacket", fwPacket). @@ -66,12 +67,12 @@ func (f *Interface) consumeInsidePacket(packet []byte, fwPacket *firewall.Packet return } - dropReason := f.firewall.Drop(*fwPacket, false, hostinfo, f.pki.GetCAPool(), localCache) + dropReason := f.firewall.Drop(*fwPacket, false, hostinfo, f.pki.GetCAPool(), localCache, nil) if dropReason == nil { f.sendNoMetrics(header.Message, 0, hostinfo.ConnectionState, hostinfo, netip.AddrPort{}, packet, nb, out, q) } else { - f.rejectInside(packet, out, q) + f.rejectInside(fwPacket, packet, out, q) if f.l.Level >= logrus.DebugLevel { hostinfo.logger(f.l). WithField("fwPacket", fwPacket). @@ -81,7 +82,10 @@ func (f *Interface) consumeInsidePacket(packet []byte, fwPacket *firewall.Packet } } -func (f *Interface) rejectInside(packet []byte, out []byte, q int) { +func (f *Interface) rejectInside(fp *firewall.Packet, packet []byte, out []byte, q int) { + if f.firewall.InLogDrop { + //todo + } if !f.firewall.InSendReject { return } @@ -97,7 +101,16 @@ func (f *Interface) rejectInside(packet []byte, out []byte, q int) { } } -func (f *Interface) rejectOutside(packet []byte, ci *ConnectionState, hostinfo *HostInfo, nb, out []byte, q int) { +func (f *Interface) rejectOutside(fp *firewall.Packet, packet []byte, ci *ConnectionState, hostinfo *HostInfo, nb, out []byte, q int) { + if f.firewall.OutLogDrop { + e := firewall.Event{ + Packet: *fp, + At: time.Now(), + Remote: hostinfo.remote, //todo if nil capture relay? + } + f.events <- e + } + if !f.firewall.OutSendReject { return } @@ -218,7 +231,7 @@ func (f *Interface) sendMessageNow(t header.MessageType, st header.MessageSubTyp } // check if packet is in outbound fw rules - dropReason := f.firewall.Drop(*fp, false, hostinfo, f.pki.GetCAPool(), nil) + dropReason := f.firewall.Drop(*fp, false, hostinfo, f.pki.GetCAPool(), nil, nil) if dropReason != nil { if f.l.Level >= logrus.DebugLevel { f.l.WithField("fwPacket", fp). diff --git a/interface.go b/interface.go index f69ed062..bf05a4cb 100644 --- a/interface.go +++ b/interface.go @@ -93,6 +93,8 @@ type Interface struct { messageMetrics *MessageMetrics cachedPacketMetrics *cachedPacketMetrics + events chan firewall.Event + l *logrus.Logger } @@ -194,6 +196,7 @@ func NewInterface(ctx context.Context, c *InterfaceConfig) (*Interface, error) { sent: metrics.GetOrRegisterCounter("hostinfo.cached_packets.sent", nil), dropped: metrics.GetOrRegisterCounter("hostinfo.cached_packets.dropped", nil), }, + events: make(chan firewall.Event, 100), //todo configurable l: c.l, } diff --git a/outside.go b/outside.go index 172c3e83..c8b5f2da 100644 --- a/outside.go +++ b/outside.go @@ -494,11 +494,11 @@ func (f *Interface) decryptToTun(hostinfo *HostInfo, messageCounter uint64, out return false } - dropReason := f.firewall.Drop(*fwPacket, true, hostinfo, f.pki.GetCAPool(), localCache) + dropReason := f.firewall.Drop(*fwPacket, true, hostinfo, f.pki.GetCAPool(), localCache, nil) if dropReason != nil { // NOTE: We give `packet` as the `out` here since we already decrypted from it and we don't need it anymore // This gives us a buffer to build the reject packet in - f.rejectOutside(out, hostinfo.ConnectionState, hostinfo, nb, packet, q) + f.rejectOutside(fwPacket, out, hostinfo.ConnectionState, hostinfo, nb, packet, q) if f.l.Level >= logrus.DebugLevel { hostinfo.logger(f.l).WithField("fwPacket", fwPacket). WithField("reason", dropReason).