Initial Version

.cargo/config

@@ -0,0 +1,3 @@
+[target.aarch64-unknown-linux-gnu]
+linker = "aarch64-linux-gnu-gcc"
+

.gitignore

@@ -0,0 +1,26 @@
+# Generated by Cargo
+# will have compiled files and executables
+debug/
+target/
+target-*/
+
+# Remove Cargo.lock from gitignore if creating an executable, leave it for libraries
+# More information here https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html
+Cargo.lock
+
+# These are backup files generated by rustfmt
+**/*.rs.bk
+
+# MSVC Windows builds of rustc generate these, which store debugging information
+*.pdb
+
+.vscode
+
+# perf & flamegraph
+perf.data
+perf.data.old
+flamegraph.svg
+
+root-target
+
+nohup.out

Cargo.toml

@@ -0,0 +1,17 @@
+[workspace]
+resolver= "2"
+members = [
+    "easytier-core",
+    "easytier-cli"
+]
+
+default-members = [
+    "easytier-core",
+    "easytier-cli"
+]
+
+[profile.dev]
+panic = "abort"
+
+[profile.release]
+panic = "abort"

LICENSE

@@ -0,0 +1,73 @@
+Apache License
+Version 2.0, January 2004
+http://www.apache.org/licenses/
+
+TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+1. Definitions.
+
+"License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document.
+
+"Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License.
+
+"Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity.
+
+"You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License.
+
+"Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files.
+
+"Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types.
+
+"Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below).
+
+"Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof.
+
+"Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution."
+
+"Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work.
+
+2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form.
+
+3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed.
+
+4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions:
+
+     (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and
+
+     (b) You must cause any modified files to carry prominent notices stating that You changed the files; and
+
+     (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and
+
+     (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License.
+
+     You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License.
+
+5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions.
+
+6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file.
+
+7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License.
+
+8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages.
+
+9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability.
+
+END OF TERMS AND CONDITIONS
+
+APPENDIX: How to apply the Apache License to your work.
+
+To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!)  The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives.
+
+Copyright 2023 sunsijie
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

README.md

@@ -0,0 +1,78 @@
+# EasyTier
+
+```diff
+! NOTICE: THIS SOFTWARE IS STILL BEGIN DEVELOPPING, ONLY POC VERSION IS PROVIDED
+```
+
+A simple out-of-box alternative of ZeroTier & TailScale in rust. Bring all devices to one virtual net.
+
+this software can serve as a substitute for Zerotier or TailScale in certain scenarios due to following features:
+
+1. Easy to deploy. 
+
+    No roles (moons/derp or other dedicated relay server). All nodes are equal and rely on some p2p algorithms to communicate.
+
+2. Smart route decision.
+
+   Use links charged by traffic to reduce latency but free links for high throughout app.
+
+3. Break the UDP throttling.
+
+   Try use TCP to achieve better performance under enviraonment with throttled UDP. Also use some methods to avoid HOL-blocking of TCP over TCP.
+
+4. High availibility.
+
+    Support multipath and switching to healthy paths when high packet loss rate or network error are detected
+
+EasyTIer also have following common features which are already supported by other softwares, but may be easier to use.
+
+1. Multi-platform support.
+
+2. Effcient P2P hole punching, both UDP & TCP.
+
+3. High performance. Try use multiple wan interface.
+
+5. Subnet Route. node can advertise and proxy one or more subnets, so other nodes can directy access these subnets without any iptables/nft trickys.
+
+
+# Usage
+
+Currently a server with public ip is needed.
+
+run first node on the public node:
+
+```
+sudo easytier-core --ipv4 <VIRTUAL_IP>
+```
+
+run other nodes
+
+```
+sudo easytier-core --ipv4 <VIRTUAL_IP> --peers tcp://<public_ip>:11010
+```
+
+use cli tool to inspect nodes with direct link.
+
+```
+easytier-cli peer
+```
+
+cli tool can also be used to inspect the route table
+
+```
+easytier-cli route
+```
+
+
+# RoadMap
+
+- [x] Windows / Mac / Linux support
+- [x] TCP / UDP tunnel.
+- [x] NAT Traverse with relaying.
+- [x] NAT Traverse with UDP hole punching.
+
+- [ ] Support shared public server. So users can use it without a public server.
+- [ ] Encryption. With noise framework or other method.
+- [ ] Support mobile platforms.
+- [ ] Broadcast & Multicast support.
+- [ ] UI tools.

easytier-cli/Cargo.toml

@@ -0,0 +1,15 @@
+[package]
+name = "easytier-cli"
+version = "0.1.0"
+edition = "2021"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+futures = "0.3"
+tokio = { version = "1", features = ["full"] }
+anyhow = "1.0"
+easytier-core = { path = "../easytier-core" }
+clap = { version = "4.4.8", features = ["unicode", "derive", "wrap_help"] }
+tonic = "0.10"
+thiserror = "1.0"

easytier-cli/src/main.rs

@@ -0,0 +1,171 @@
+use clap::{command, Args, Parser, Subcommand};
+use easytier_rpc::{
+    connector_manage_rpc_client::ConnectorManageRpcClient,
+    peer_manage_rpc_client::PeerManageRpcClient, ListConnectorRequest, ListPeerRequest,
+    ListRouteRequest,
+};
+
+#[derive(Parser, Debug)]
+#[command(author, version, about, long_about = None)]
+struct Cli {
+    /// the instance name
+    #[arg(short = 'n', long, default_value = "default")]
+    instance_name: String,
+
+    #[command(subcommand)]
+    sub_command: Option<SubCommand>,
+}
+
+#[derive(Subcommand, Debug)]
+enum SubCommand {
+    Peer(PeerArgs),
+    Connector(ConnectorArgs),
+    Route,
+}
+
+#[derive(Args, Debug)]
+struct PeerArgs {
+    #[arg(short, long)]
+    ipv4: Option<String>,
+
+    #[arg(short, long)]
+    peers: Vec<String>,
+
+    #[command(subcommand)]
+    sub_command: Option<PeerSubCommand>,
+}
+
+#[derive(Subcommand, Debug)]
+enum PeerSubCommand {
+    Add,
+    Remove,
+    List,
+}
+
+#[derive(Args, Debug)]
+struct ConnectorArgs {
+    #[arg(short, long)]
+    ipv4: Option<String>,
+
+    #[arg(short, long)]
+    peers: Vec<String>,
+
+    #[command(subcommand)]
+    sub_command: Option<ConnectorSubCommand>,
+}
+
+#[derive(Subcommand, Debug)]
+enum ConnectorSubCommand {
+    Add,
+    Remove,
+    List,
+}
+
+#[derive(thiserror::Error, Debug)]
+enum Error {
+    #[error("tonic transport error")]
+    TonicTransportError(#[from] tonic::transport::Error),
+    #[error("tonic rpc error")]
+    TonicRpcError(#[from] tonic::Status),
+}
+
+struct CommandHandler {
+    addr: String,
+}
+
+impl CommandHandler {
+    async fn get_peer_manager_client(
+        &self,
+    ) -> Result<PeerManageRpcClient<tonic::transport::Channel>, Error> {
+        Ok(PeerManageRpcClient::connect(self.addr.clone()).await?)
+    }
+
+    async fn get_connector_manager_client(
+        &self,
+    ) -> Result<ConnectorManageRpcClient<tonic::transport::Channel>, Error> {
+        Ok(ConnectorManageRpcClient::connect(self.addr.clone()).await?)
+    }
+
+    #[allow(dead_code)]
+    fn handle_peer_add(&self, _args: PeerArgs) {
+        println!("add peer");
+    }
+
+    #[allow(dead_code)]
+    fn handle_peer_remove(&self, _args: PeerArgs) {
+        println!("remove peer");
+    }
+
+    async fn handle_peer_list(&self, _args: PeerArgs) -> Result<(), Error> {
+        let mut client = self.get_peer_manager_client().await?;
+        let request = tonic::Request::new(ListPeerRequest::default());
+        let response = client.list_peer(request).await?;
+        println!("response: {:#?}", response.into_inner());
+        Ok(())
+    }
+
+    async fn handle_route_list(&self) -> Result<(), Error> {
+        let mut client = self.get_peer_manager_client().await?;
+        let request = tonic::Request::new(ListRouteRequest::default());
+        let response = client.list_route(request).await?;
+        println!("response: {:#?}", response.into_inner());
+        Ok(())
+    }
+
+    async fn handle_connector_list(&self) -> Result<(), Error> {
+        let mut client = self.get_connector_manager_client().await?;
+        let request = tonic::Request::new(ListConnectorRequest::default());
+        let response = client.list_connector(request).await?;
+        println!("response: {:#?}", response.into_inner());
+        Ok(())
+    }
+}
+
+#[tokio::main]
+async fn main() -> Result<(), Error> {
+    let cli = Cli::parse();
+    println!("cli: {:?}", cli);
+
+    let handler = CommandHandler {
+        addr: "http://127.0.0.1:15888".to_string(),
+    };
+
+    match cli.sub_command {
+        Some(SubCommand::Peer(peer_args)) => match peer_args.sub_command {
+            Some(PeerSubCommand::Add) => {
+                println!("add peer");
+            }
+            Some(PeerSubCommand::Remove) => {
+                println!("remove peer");
+            }
+            Some(PeerSubCommand::List) => {
+                handler.handle_peer_list(peer_args).await?;
+            }
+            None => {
+                handler.handle_peer_list(peer_args).await?;
+            }
+        },
+        Some(SubCommand::Connector(conn_args)) => match conn_args.sub_command {
+            Some(ConnectorSubCommand::Add) => {
+                println!("add connector");
+            }
+            Some(ConnectorSubCommand::Remove) => {
+                println!("remove connector");
+            }
+            Some(ConnectorSubCommand::List) => {
+                handler.handle_connector_list().await?;
+            }
+            None => {
+                handler.handle_connector_list().await?;
+            }
+        },
+        Some(SubCommand::Route) => {
+            handler.handle_route_list().await?;
+        }
+        None => {
+            println!("list peer");
+        }
+    }
+
+    Ok(())
+}

easytier-core/Cargo.toml

@@ -0,0 +1,105 @@
+[package]
+name = "easytier-core"
+version = "0.1.0"
+edition = "2021"
+authors = ["easytier"]
+rust-version = "1.75"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[lib]
+name = "easytier_rpc"
+path = "src/rpc/lib.rs"
+
+[dependencies]
+tracing = { version = "0.1", features = ["log"] }
+tracing-subscriber = { version = "0.3", features = ["env-filter"] }
+tracing-appender = "0.2.3"
+log = "0.4"
+thiserror = "1.0"
+auto_impl = "1.1.0"
+crossbeam = "0.8.4"
+
+gethostname = "0.4.3"
+
+futures = "0.3"
+
+tokio = { version = "1", features = ["full"] }
+tokio-stream = "0.1"
+tokio-util = { version = "0.7.9", features = ["codec", "net"] }
+
+async-stream = "0.3.5"
+async-trait = "0.1.74"
+
+dashmap = "5.5.3"
+timedmap = "1.0.1"
+
+# for tap device
+tun = { version = "0.6.1", features = ["async"] }
+# for net ns
+nix = { version = "0.27", features = ["sched", "socket", "ioctl"] }
+
+uuid = { version = "1.5.0", features = [
+    "v4",
+    "fast-rng",
+    "macro-diagnostics",
+    "serde",
+] }
+
+# for ring tunnel
+crossbeam-queue = "0.3"
+once_cell = "1.18.0"
+
+# for packet
+rkyv = { "version" = "0.7.42", features = ["validation", "archive_le"] }
+
+# for rpc
+tonic = "0.10"
+prost = "0.12"
+anyhow = "1.0"
+tarpc = { version = "0.32", features = ["tokio1", "serde1"] }
+bincode = "1.3"
+
+url = "2.5.0"
+
+# for tun packet
+byteorder = "1.5.0"
+
+# for proxy
+cidr = "0.2.2"
+socket2 = "0.5.5"
+
+# for hole punching
+stun-format = { git = "https://github.com/KKRainbow/stun-format.git", features = [
+    "fmt",
+    "rfc3489",
+    "iana",
+] }
+rand = "0.8.5"
+
+[dependencies.serde]
+version = "1.0"
+features = ["derive"]
+
+[dependencies.pnet]
+version = "0.34.0"
+features = ["serde"]
+
+[dependencies.clap]
+version = "4.4"
+features = ["derive"]
+
+[dependencies.public-ip]
+version = "0.2"
+features = ["default"]
+
+[build-dependencies]
+tonic-build = "0.10"
+
+[target.'cfg(windows)'.build-dependencies]
+reqwest = { version = "0.11", features = ["blocking"] }
+zip = "*"
+
+
+[dev-dependencies]
+serial_test = "*"

easytier-core/build.rs

@@ -0,0 +1,95 @@
+#[cfg(target_os = "windows")]
+use std::{
+    env,
+    fs::File,
+    io::{copy, Cursor},
+    path::PathBuf,
+};
+
+#[cfg(target_os = "windows")]
+struct WindowsBuild {}
+
+#[cfg(target_os = "windows")]
+impl WindowsBuild {
+    fn check_protoc_exist() -> Option<PathBuf> {
+        let path = env::var_os("PROTOC").map(PathBuf::from);
+        if path.is_some() && path.as_ref().unwrap().exists() {
+            return path;
+        }
+
+        let path = env::var_os("PATH").unwrap_or_default();
+        for p in env::split_paths(&path) {
+            let p = p.join("protoc");
+            if p.exists() {
+                return Some(p);
+            }
+        }
+
+        None
+    }
+
+    fn get_cargo_target_dir() -> Result<std::path::PathBuf, Box<dyn std::error::Error>> {
+        let out_dir = std::path::PathBuf::from(std::env::var("OUT_DIR")?);
+        let profile = std::env::var("PROFILE")?;
+        let mut target_dir = None;
+        let mut sub_path = out_dir.as_path();
+        while let Some(parent) = sub_path.parent() {
+            if parent.ends_with(&profile) {
+                target_dir = Some(parent);
+                break;
+            }
+            sub_path = parent;
+        }
+        let target_dir = target_dir.ok_or("not found")?;
+        Ok(target_dir.to_path_buf())
+    }
+
+    fn download_protoc() -> PathBuf {
+        println!("cargo:info=use exist protoc: {:?}", "k");
+        let out_dir = Self::get_cargo_target_dir().unwrap();
+        let fname = out_dir.join("protoc");
+        if fname.exists() {
+            println!("cargo:info=use exist protoc: {:?}", fname);
+            return fname;
+        }
+
+        println!("cargo:info=need download protoc, please wait...");
+
+        let url = "https://github.com/protocolbuffers/protobuf/releases/download/v26.0-rc1/protoc-26.0-rc-1-win64.zip";
+        let response = reqwest::blocking::get(url).unwrap();
+        println!("{:?}", response);
+        let mut content = response
+            .bytes()
+            .map(|v| v.to_vec())
+            .map(Cursor::new)
+            .map(zip::ZipArchive::new)
+            .unwrap()
+            .unwrap();
+        let protoc_zipped_file = content.by_name("bin/protoc.exe").unwrap();
+        let mut content = protoc_zipped_file;
+
+        copy(&mut content, &mut File::create(&fname).unwrap()).unwrap();
+
+        fname
+    }
+
+    pub fn check_for_win() {
+        // add third_party dir to link search path
+        println!("cargo:rustc-link-search=native=third_party/");
+        let protoc_path = if let Some(o) = Self::check_protoc_exist() {
+            println!("cargo:info=use os exist protoc: {:?}", o);
+            o
+        } else {
+            Self::download_protoc()
+        };
+        std::env::set_var("PROTOC", protoc_path);
+    }
+}
+
+fn main() -> Result<(), Box<dyn std::error::Error>> {
+    #[cfg(target_os = "windows")]
+    WindowsBuild::check_for_win();
+
+    tonic_build::compile_protos("proto/cli.proto")?;
+    Ok(())
+}

easytier-core/proto/cli.proto

@@ -0,0 +1,116 @@
+syntax = "proto3";
+package cli;
+
+message Status {
+    int32 code = 1;
+    string message = 2;
+}
+
+message PeerConnStats {
+    uint64 rx_bytes = 1;
+    uint64 tx_bytes = 2;
+
+    uint64 rx_packets = 3;
+    uint64 tx_packets = 4;
+
+    uint64 latency_us = 5;
+}
+
+message TunnelInfo {
+    string tunnel_type = 1;
+    string local_addr = 2;
+    string remote_addr = 3;
+}
+
+message PeerConnInfo {
+    string conn_id = 1;
+    string my_node_id = 2;
+    string peer_id = 3;
+    repeated string features = 4;
+    TunnelInfo tunnel = 5;
+    PeerConnStats stats = 6;
+}
+
+message PeerInfo {
+    string peer_id = 1;
+    repeated PeerConnInfo conns = 2;
+}
+
+message ListPeerRequest {}
+
+message ListPeerResponse {
+    repeated PeerInfo peer_infos = 1;
+}
+
+enum NatType {
+    // has NAT; but own a single public IP, port is not changed
+    Unknown = 0;
+    OpenInternet = 1;
+    NoPAT = 2;
+    FullCone = 3;
+    Restricted = 4;
+    PortRestricted = 5;
+    Symmetric = 6;
+    SymUdpFirewall = 7;
+}
+
+message StunInfo {
+    NatType udp_nat_type = 1;
+    NatType tcp_nat_type = 2;
+    int64 last_update_time = 3;
+}
+
+message Route {
+    string peer_id = 1;
+    string ipv4_addr = 2;
+    string next_hop_peer_id = 3;
+    int32 cost = 4;
+    repeated string proxy_cidrs = 5;
+    string hostname = 6;
+    StunInfo stun_info = 7;
+}
+
+message ListRouteRequest {}
+
+message ListRouteResponse {
+    repeated Route routes = 1;
+}
+
+service PeerManageRpc {
+   rpc ListPeer (ListPeerRequest) returns (ListPeerResponse);
+   rpc ListRoute (ListRouteRequest) returns (ListRouteResponse);
+}
+
+enum ConnectorStatus {
+    CONNECTED = 0;
+    DISCONNECTED = 1;
+    CONNECTING = 2;
+}
+
+message Connector {
+    string url = 1;
+    ConnectorStatus status = 2;
+}
+
+message ListConnectorRequest {}
+
+message ListConnectorResponse {
+    repeated Connector connectors = 1;
+}
+
+enum ConnectorManageAction {
+    ADD = 0;
+    REMOVE = 1;
+}
+
+message ManageConnectorRequest {
+    ConnectorManageAction action = 1;
+    string url = 2;
+}
+
+message ManageConnectorResponse { }
+
+service ConnectorManageRpc {
+    rpc ListConnector (ListConnectorRequest) returns (ListConnectorResponse);
+    rpc ManageConnector (ManageConnectorRequest) returns (ManageConnectorResponse);
+}

easytier-core/src/common/config_fs.rs

@@ -0,0 +1,161 @@
+// use filesystem as a config store
+
+use std::{
+    ffi::OsStr,
+    io::Write,
+    path::{Path, PathBuf},
+};
+
+static DEFAULT_BASE_DIR: &str = "/var/lib/easytier";
+static DIR_ROOT_CONFIG_FILE_NAME: &str = "__root__";
+
+pub struct ConfigFs {
+    _db_name: String,
+    db_path: PathBuf,
+}
+
+impl ConfigFs {
+    pub fn new(db_name: &str) -> Self {
+        Self::new_with_dir(db_name, DEFAULT_BASE_DIR)
+    }
+
+    pub fn new_with_dir(db_name: &str, dir: &str) -> Self {
+        let p = Path::new(OsStr::new(dir)).join(OsStr::new(db_name));
+        std::fs::create_dir_all(&p).unwrap();
+        ConfigFs {
+            _db_name: db_name.to_string(),
+            db_path: p,
+        }
+    }
+
+    pub fn get(&self, key: &str) -> Result<String, std::io::Error> {
+        let path = self.db_path.join(OsStr::new(key));
+        // if path is dir, read the DIR_ROOT_CONFIG_FILE_NAME in it
+        if path.is_dir() {
+            let path = path.join(OsStr::new(DIR_ROOT_CONFIG_FILE_NAME));
+            std::fs::read_to_string(path)
+        } else if path.is_file() {
+            return std::fs::read_to_string(path);
+        } else {
+            return Err(std::io::Error::new(
+                std::io::ErrorKind::NotFound,
+                "key not found",
+            ));
+        }
+    }
+
+    pub fn list_keys(&self, key: &str) -> Result<Vec<String>, std::io::Error> {
+        let path = self.db_path.join(OsStr::new(key));
+        let mut keys = Vec::new();
+        for entry in std::fs::read_dir(path)? {
+            let entry = entry?;
+            let path = entry.path();
+            let key = path.file_name().unwrap().to_str().unwrap().to_string();
+            if key != DIR_ROOT_CONFIG_FILE_NAME {
+                keys.push(key);
+            }
+        }
+        Ok(keys)
+    }
+
+    #[allow(dead_code)]
+    pub fn remove(&self, key: &str) -> Result<(), std::io::Error> {
+        let path = self.db_path.join(OsStr::new(key));
+        // if path is dir, remove the DIR_ROOT_CONFIG_FILE_NAME in it
+        if path.is_dir() {
+            std::fs::remove_dir_all(path)
+        } else if path.is_file() {
+            return std::fs::remove_file(path);
+        } else {
+            return Err(std::io::Error::new(
+                std::io::ErrorKind::NotFound,
+                "key not found",
+            ));
+        }
+    }
+
+    pub fn add_dir(&self, key: &str) -> Result<std::fs::File, std::io::Error> {
+        let path = self.db_path.join(OsStr::new(key));
+        // if path is dir, write the DIR_ROOT_CONFIG_FILE_NAME in it
+        if path.is_file() {
+            Err(std::io::Error::new(
+                std::io::ErrorKind::AlreadyExists,
+                "key already exists",
+            ))
+        } else {
+            std::fs::create_dir_all(&path)?;
+            return std::fs::File::create(path.join(OsStr::new(DIR_ROOT_CONFIG_FILE_NAME)));
+        }
+    }
+
+    pub fn add_file(&self, key: &str) -> Result<std::fs::File, std::io::Error> {
+        let path = self.db_path.join(OsStr::new(key));
+        let base_dir = path.parent().unwrap();
+        if !path.is_file() {
+            std::fs::create_dir_all(base_dir)?;
+        }
+        std::fs::File::create(path)
+    }
+
+    pub fn get_or_add<F>(
+        &self,
+        key: &str,
+        val_fn: F,
+        add_dir: bool,
+    ) -> Result<String, std::io::Error>
+    where
+        F: FnOnce() -> String,
+    {
+        let get_ret = self.get(key);
+        match get_ret {
+            Ok(v) => Ok(v),
+            Err(e) => {
+                if e.kind() == std::io::ErrorKind::NotFound {
+                    let val = val_fn();
+                    if add_dir {
+                        let mut f = self.add_dir(key)?;
+                        f.write_all(val.as_bytes())?;
+                    } else {
+                        let mut f = self.add_file(key)?;
+                        f.write_all(val.as_bytes())?;
+                    }
+                    Ok(val)
+                } else {
+                    Err(e)
+                }
+            }
+        }
+    }
+
+    #[allow(dead_code)]
+    pub fn get_or_add_dir<F>(&self, key: &str, val_fn: F) -> Result<String, std::io::Error>
+    where
+        F: FnOnce() -> String,
+    {
+        self.get_or_add(key, val_fn, true)
+    }
+
+    pub fn get_or_add_file<F>(&self, key: &str, val_fn: F) -> Result<String, std::io::Error>
+    where
+        F: FnOnce() -> String,
+    {
+        self.get_or_add(key, val_fn, false)
+    }
+
+    pub fn get_or_default<F>(&self, key: &str, default: F) -> Result<String, std::io::Error>
+    where
+        F: FnOnce() -> String,
+    {
+        let get_ret = self.get(key);
+        match get_ret {
+            Ok(v) => Ok(v),
+            Err(e) => {
+                if e.kind() == std::io::ErrorKind::NotFound {
+                    Ok(default())
+                } else {
+                    Err(e)
+                }
+            }
+        }
+    }
+}

easytier-core/src/common/constants.rs

@@ -0,0 +1,28 @@
+pub const DIRECT_CONNECTOR_SERVICE_ID: u32 = 1;
+pub const DIRECT_CONNECTOR_BLACKLIST_TIMEOUT_SEC: u64 = 60;
+pub const DIRECT_CONNECTOR_IP_LIST_TIMEOUT_SEC: u64 = 60;
+
+macro_rules! define_global_var {
+    ($name:ident, $type:ty, $init:expr) => {
+        pub static $name: once_cell::sync::Lazy<tokio::sync::Mutex<$type>> =
+            once_cell::sync::Lazy::new(|| tokio::sync::Mutex::new($init));
+    };
+}
+
+#[macro_export]
+macro_rules! use_global_var {
+    ($name:ident) => {
+        crate::common::constants::$name.lock().await.to_owned()
+    };
+}
+
+#[macro_export]
+macro_rules! set_global_var {
+    ($name:ident, $val:expr) => {
+        *crate::common::constants::$name.lock().await = $val
+    };
+}
+
+define_global_var!(MANUAL_CONNECTOR_RECONNECT_INTERVAL_MS, u64, 1000);
+
+pub const UDP_HOLE_PUNCH_CONNECTOR_SERVICE_ID: u32 = 2;

easytier-core/src/common/error.rs

@@ -0,0 +1,43 @@
+use std::{io, result};
+
+use thiserror::Error;
+
+use crate::tunnels;
+
+#[derive(Error, Debug)]
+pub enum Error {
+    #[error("io error")]
+    IOError(#[from] io::Error),
+    #[error("rust tun error {0}")]
+    TunError(#[from] tun::Error),
+    #[error("tunnel error {0}")]
+    TunnelError(#[from] tunnels::TunnelError),
+    #[error("Peer has no conn, PeerId: {0}")]
+    PeerNoConnectionError(uuid::Uuid),
+    #[error("RouteError: {0}")]
+    RouteError(String),
+    #[error("Not found")]
+    NotFound,
+    #[error("Invalid Url: {0}")]
+    InvalidUrl(String),
+    #[error("Shell Command error: {0}")]
+    ShellCommandError(String),
+    // #[error("Rpc listen error: {0}")]
+    // RpcListenError(String),
+    #[error("Rpc connect error: {0}")]
+    RpcConnectError(String),
+    #[error("Rpc error: {0}")]
+    RpcClientError(#[from] tarpc::client::RpcError),
+    #[error("Timeout error: {0}")]
+    Timeout(#[from] tokio::time::error::Elapsed),
+    #[error("url in blacklist")]
+    UrlInBlacklist,
+    #[error("unknown data store error")]
+    Unknown,
+    #[error("anyhow error: {0}")]
+    AnyhowError(#[from] anyhow::Error),
+}
+
+pub type Result<T> = result::Result<T, Error>;
+
+// impl From for std::

easytier-core/src/common/global_ctx.rs

@@ -0,0 +1,259 @@
+use std::{io::Write, sync::Arc};
+
+use crossbeam::atomic::AtomicCell;
+use easytier_rpc::PeerConnInfo;
+
+use super::{
+    config_fs::ConfigFs,
+    netns::NetNS,
+    network::IPCollector,
+    stun::{StunInfoCollector, StunInfoCollectorTrait},
+};
+
+#[derive(Debug, Clone, PartialEq)]
+pub enum GlobalCtxEvent {
+    PeerAdded,
+    PeerRemoved,
+    PeerConnAdded(PeerConnInfo),
+    PeerConnRemoved(PeerConnInfo),
+}
+
+type EventBus = tokio::sync::broadcast::Sender<GlobalCtxEvent>;
+type EventBusSubscriber = tokio::sync::broadcast::Receiver<GlobalCtxEvent>;
+
+pub struct GlobalCtx {
+    pub inst_name: String,
+    pub id: uuid::Uuid,
+    pub config_fs: ConfigFs,
+    pub net_ns: NetNS,
+
+    event_bus: EventBus,
+
+    cached_ipv4: AtomicCell<Option<std::net::Ipv4Addr>>,
+    cached_proxy_cidrs: AtomicCell<Option<Vec<cidr::IpCidr>>>,
+
+    ip_collector: Arc<IPCollector>,
+
+    hotname: AtomicCell<Option<String>>,
+
+    stun_info_collection: Box<dyn StunInfoCollectorTrait>,
+}
+
+impl std::fmt::Debug for GlobalCtx {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("GlobalCtx")
+            .field("inst_name", &self.inst_name)
+            .field("id", &self.id)
+            .field("net_ns", &self.net_ns.name())
+            .field("event_bus", &"EventBus")
+            .field("ipv4", &self.cached_ipv4)
+            .finish()
+    }
+}
+
+pub type ArcGlobalCtx = std::sync::Arc<GlobalCtx>;
+
+impl GlobalCtx {
+    pub fn new(inst_name: &str, config_fs: ConfigFs, net_ns: NetNS) -> Self {
+        let id = config_fs
+            .get_or_add_file("inst_id", || uuid::Uuid::new_v4().to_string())
+            .unwrap();
+        let id = uuid::Uuid::parse_str(&id).unwrap();
+
+        let (event_bus, _) = tokio::sync::broadcast::channel(100);
+
+        // NOTICE: we may need to choose stun stun server based on geo location
+        // stun server cross nation may return a external ip address with high latency and loss rate
+        let default_stun_servers = vec![
+            "stun.miwifi.com:3478".to_string(),
+            "stun.qq.com:3478".to_string(),
+            "stun.chat.bilibili.com:3478".to_string(),
+            "fwa.lifesizecloud.com:3478".to_string(),
+            "stun.isp.net.au:3478".to_string(),
+            "stun.nextcloud.com:3478".to_string(),
+            "stun.freeswitch.org:3478".to_string(),
+            "stun.voip.blackberry.com:3478".to_string(),
+            "stunserver.stunprotocol.org:3478".to_string(),
+            "stun.sipnet.com:3478".to_string(),
+            "stun.radiojar.com:3478".to_string(),
+            "stun.sonetel.com:3478".to_string(),
+            "stun.voipgate.com:3478".to_string(),
+            "stun.counterpath.com:3478".to_string(),
+            "180.235.108.91:3478".to_string(),
+            "193.22.2.248:3478".to_string(),
+        ];
+
+        GlobalCtx {
+            inst_name: inst_name.to_string(),
+            id,
+            config_fs,
+            net_ns: net_ns.clone(),
+            event_bus,
+            cached_ipv4: AtomicCell::new(None),
+            cached_proxy_cidrs: AtomicCell::new(None),
+
+            ip_collector: Arc::new(IPCollector::new(net_ns)),
+
+            hotname: AtomicCell::new(None),
+
+            stun_info_collection: Box::new(StunInfoCollector::new(default_stun_servers)),
+        }
+    }
+
+    pub fn subscribe(&self) -> EventBusSubscriber {
+        self.event_bus.subscribe()
+    }
+
+    pub fn issue_event(&self, event: GlobalCtxEvent) {
+        if self.event_bus.receiver_count() != 0 {
+            self.event_bus.send(event).unwrap();
+        } else {
+            log::warn!("No subscriber for event: {:?}", event);
+        }
+    }
+
+    pub fn get_ipv4(&self) -> Option<std::net::Ipv4Addr> {
+        if let Some(ret) = self.cached_ipv4.load() {
+            return Some(ret);
+        }
+
+        let Ok(addr) = self.config_fs.get("ipv4") else {
+            return None;
+        };
+
+        let Ok(addr) = addr.parse() else {
+            tracing::error!("invalid ipv4 addr: {}", addr);
+            return None;
+        };
+
+        self.cached_ipv4.store(Some(addr));
+        return Some(addr);
+    }
+
+    pub fn set_ipv4(&mut self, addr: std::net::Ipv4Addr) {
+        self.config_fs
+            .add_file("ipv4")
+            .unwrap()
+            .write_all(addr.to_string().as_bytes())
+            .unwrap();
+
+        self.cached_ipv4.store(None);
+    }
+
+    pub fn add_proxy_cidr(&self, cidr: cidr::IpCidr) -> Result<(), std::io::Error> {
+        let escaped_cidr = cidr.to_string().replace("/", "_");
+        self.config_fs
+            .add_file(&format!("proxy_cidrs/{}", escaped_cidr))?;
+        self.cached_proxy_cidrs.store(None);
+        Ok(())
+    }
+
+    pub fn remove_proxy_cidr(&self, cidr: cidr::IpCidr) -> Result<(), std::io::Error> {
+        let escaped_cidr = cidr.to_string().replace("/", "_");
+        self.config_fs
+            .remove(&format!("proxy_cidrs/{}", escaped_cidr))?;
+        self.cached_proxy_cidrs.store(None);
+        Ok(())
+    }
+
+    pub fn get_proxy_cidrs(&self) -> Vec<cidr::IpCidr> {
+        if let Some(proxy_cidrs) = self.cached_proxy_cidrs.take() {
+            self.cached_proxy_cidrs.store(Some(proxy_cidrs.clone()));
+            return proxy_cidrs;
+        }
+
+        let Ok(keys) = self.config_fs.list_keys("proxy_cidrs") else {
+            return vec![];
+        };
+
+        let mut ret = Vec::new();
+        for key in keys.iter() {
+            let key = key.replace("_", "/");
+            let Ok(cidr) = key.parse() else {
+                tracing::error!("invalid proxy cidr: {}", key);
+                continue;
+            };
+            ret.push(cidr);
+        }
+
+        self.cached_proxy_cidrs.store(Some(ret.clone()));
+        ret
+    }
+
+    pub fn get_ip_collector(&self) -> Arc<IPCollector> {
+        self.ip_collector.clone()
+    }
+
+    pub fn get_hostname(&self) -> Option<String> {
+        if let Some(hostname) = self.hotname.take() {
+            self.hotname.store(Some(hostname.clone()));
+            return Some(hostname);
+        }
+
+        let hostname = gethostname::gethostname().to_string_lossy().to_string();
+        self.hotname.store(Some(hostname.clone()));
+        return Some(hostname);
+    }
+
+    pub fn get_stun_info_collector(&self) -> impl StunInfoCollectorTrait + '_ {
+        self.stun_info_collection.as_ref()
+    }
+
+    #[cfg(test)]
+    pub fn replace_stun_info_collector(&self, collector: Box<dyn StunInfoCollectorTrait>) {
+        // force replace the stun_info_collection without mut and drop the old one
+        let ptr = &self.stun_info_collection as *const Box<dyn StunInfoCollectorTrait>;
+        let ptr = ptr as *mut Box<dyn StunInfoCollectorTrait>;
+        unsafe {
+            std::ptr::drop_in_place(ptr);
+            std::ptr::write(ptr, collector);
+        }
+    }
+
+    pub fn get_id(&self) -> uuid::Uuid {
+        self.id
+    }
+}
+
+#[cfg(test)]
+pub mod tests {
+    use super::*;
+
+    #[tokio::test]
+    async fn test_global_ctx() {
+        let config_fs = ConfigFs::new("/tmp/easytier");
+        let net_ns = NetNS::new(None);
+        let global_ctx = GlobalCtx::new("test", config_fs, net_ns);
+
+        let mut subscriber = global_ctx.subscribe();
+        global_ctx.issue_event(GlobalCtxEvent::PeerAdded);
+        global_ctx.issue_event(GlobalCtxEvent::PeerRemoved);
+        global_ctx.issue_event(GlobalCtxEvent::PeerConnAdded(PeerConnInfo::default()));
+        global_ctx.issue_event(GlobalCtxEvent::PeerConnRemoved(PeerConnInfo::default()));
+
+        assert_eq!(subscriber.recv().await.unwrap(), GlobalCtxEvent::PeerAdded);
+        assert_eq!(
+            subscriber.recv().await.unwrap(),
+            GlobalCtxEvent::PeerRemoved
+        );
+        assert_eq!(
+            subscriber.recv().await.unwrap(),
+            GlobalCtxEvent::PeerConnAdded(PeerConnInfo::default())
+        );
+        assert_eq!(
+            subscriber.recv().await.unwrap(),
+            GlobalCtxEvent::PeerConnRemoved(PeerConnInfo::default())
+        );
+    }
+
+    pub fn get_mock_global_ctx() -> ArcGlobalCtx {
+        let node_id = uuid::Uuid::new_v4();
+        let config_fs = ConfigFs::new_with_dir(node_id.to_string().as_str(), "/tmp/easytier");
+        let net_ns = NetNS::new(None);
+        std::sync::Arc::new(GlobalCtx::new(
+            format!("test_{}", node_id).as_str(),
+            config_fs,
+            net_ns,
+        ))
+    }
+}

easytier-core/src/common/ifcfg.rs

@@ -0,0 +1,312 @@
+use std::net::Ipv4Addr;
+
+use async_trait::async_trait;
+use tokio::process::Command;
+
+use super::error::Error;
+
+#[async_trait]
+pub trait IfConfiguerTrait {
+    async fn add_ipv4_route(
+        &self,
+        name: &str,
+        address: Ipv4Addr,
+        cidr_prefix: u8,
+    ) -> Result<(), Error>;
+    async fn remove_ipv4_route(
+        &self,
+        name: &str,
+        address: Ipv4Addr,
+        cidr_prefix: u8,
+    ) -> Result<(), Error>;
+    async fn add_ipv4_ip(
+        &self,
+        name: &str,
+        address: Ipv4Addr,
+        cidr_prefix: u8,
+    ) -> Result<(), Error>;
+    async fn set_link_status(&self, name: &str, up: bool) -> Result<(), Error>;
+    async fn remove_ip(&self, name: &str, ip: Option<Ipv4Addr>) -> Result<(), Error>;
+    async fn wait_interface_show(&self, _name: &str) -> Result<(), Error> {
+        return Ok(());
+    }
+}
+
+fn cidr_to_subnet_mask(prefix_length: u8) -> Ipv4Addr {
+    if prefix_length > 32 {
+        panic!("Invalid CIDR prefix length");
+    }
+
+    let subnet_mask: u32 = (!0u32)
+        .checked_shl(32 - u32::from(prefix_length))
+        .unwrap_or(0);
+    Ipv4Addr::new(
+        ((subnet_mask >> 24) & 0xFF) as u8,
+        ((subnet_mask >> 16) & 0xFF) as u8,
+        ((subnet_mask >> 8) & 0xFF) as u8,
+        (subnet_mask & 0xFF) as u8,
+    )
+}
+
+async fn run_shell_cmd(cmd: &str) -> Result<(), Error> {
+    let cmd_out = if cfg!(target_os = "windows") {
+        Command::new("cmd").arg("/C").arg(cmd).output().await?
+    } else {
+        Command::new("sh").arg("-c").arg(cmd).output().await?
+    };
+    let stdout = String::from_utf8_lossy(cmd_out.stdout.as_slice());
+    let stderr = String::from_utf8_lossy(cmd_out.stderr.as_slice());
+    let ec = cmd_out.status.code();
+    let succ = cmd_out.status.success();
+    tracing::info!(?cmd, ?ec, ?succ, ?stdout, ?stderr, "run shell cmd");
+
+    if !cmd_out.status.success() {
+        return Err(Error::ShellCommandError(
+            stdout.to_string() + &stderr.to_string(),
+        ));
+    }
+    Ok(())
+}
+
+pub struct MacIfConfiger {}
+#[async_trait]
+impl IfConfiguerTrait for MacIfConfiger {
+    async fn add_ipv4_route(
+        &self,
+        name: &str,
+        address: Ipv4Addr,
+        cidr_prefix: u8,
+    ) -> Result<(), Error> {
+        run_shell_cmd(
+            format!(
+                "route -n add {} -netmask {} -interface {} -hopcount 7",
+                address,
+                cidr_to_subnet_mask(cidr_prefix),
+                name
+            )
+            .as_str(),
+        )
+        .await
+    }
+
+    async fn remove_ipv4_route(
+        &self,
+        name: &str,
+        address: Ipv4Addr,
+        cidr_prefix: u8,
+    ) -> Result<(), Error> {
+        run_shell_cmd(
+            format!(
+                "route -n delete {} -netmask {} -interface {}",
+                address,
+                cidr_to_subnet_mask(cidr_prefix),
+                name
+            )
+            .as_str(),
+        )
+        .await
+    }
+
+    async fn add_ipv4_ip(
+        &self,
+        name: &str,
+        address: Ipv4Addr,
+        cidr_prefix: u8,
+    ) -> Result<(), Error> {
+        run_shell_cmd(
+            format!(
+                "ifconfig {} {:?}/{:?} 10.8.8.8 up",
+                name, address, cidr_prefix,
+            )
+            .as_str(),
+        )
+        .await
+    }
+
+    async fn set_link_status(&self, name: &str, up: bool) -> Result<(), Error> {
+        run_shell_cmd(format!("ifconfig {} {}", name, if up { "up" } else { "down" }).as_str())
+            .await
+    }
+
+    async fn remove_ip(&self, name: &str, ip: Option<Ipv4Addr>) -> Result<(), Error> {
+        if ip.is_none() {
+            run_shell_cmd(format!("ifconfig {} inet delete", name).as_str()).await
+        } else {
+            run_shell_cmd(
+                format!("ifconfig {} inet {} delete", name, ip.unwrap().to_string()).as_str(),
+            )
+            .await
+        }
+    }
+}
+
+pub struct LinuxIfConfiger {}
+#[async_trait]
+impl IfConfiguerTrait for LinuxIfConfiger {
+    async fn add_ipv4_route(
+        &self,
+        name: &str,
+        address: Ipv4Addr,
+        cidr_prefix: u8,
+    ) -> Result<(), Error> {
+        run_shell_cmd(
+            format!(
+                "ip route add {}/{} dev {} metric 65535",
+                address, cidr_prefix, name
+            )
+            .as_str(),
+        )
+        .await
+    }
+
+    async fn remove_ipv4_route(
+        &self,
+        name: &str,
+        address: Ipv4Addr,
+        cidr_prefix: u8,
+    ) -> Result<(), Error> {
+        run_shell_cmd(format!("ip route del {}/{} dev {}", address, cidr_prefix, name).as_str())
+            .await
+    }
+
+    async fn add_ipv4_ip(
+        &self,
+        name: &str,
+        address: Ipv4Addr,
+        cidr_prefix: u8,
+    ) -> Result<(), Error> {
+        run_shell_cmd(format!("ip addr add {:?}/{:?} dev {}", address, cidr_prefix, name).as_str())
+            .await
+    }
+
+    async fn set_link_status(&self, name: &str, up: bool) -> Result<(), Error> {
+        run_shell_cmd(format!("ip link set {} {}", name, if up { "up" } else { "down" }).as_str())
+            .await
+    }
+
+    async fn remove_ip(&self, name: &str, ip: Option<Ipv4Addr>) -> Result<(), Error> {
+        if ip.is_none() {
+            run_shell_cmd(format!("ip addr flush dev {}", name).as_str()).await
+        } else {
+            run_shell_cmd(
+                format!("ip addr del {:?} dev {}", ip.unwrap().to_string(), name).as_str(),
+            )
+            .await
+        }
+    }
+}
+
+pub struct WindowsIfConfiger {}
+
+#[async_trait]
+impl IfConfiguerTrait for WindowsIfConfiger {
+    async fn add_ipv4_route(
+        &self,
+        name: &str,
+        address: Ipv4Addr,
+        cidr_prefix: u8,
+    ) -> Result<(), Error> {
+        run_shell_cmd(
+            format!(
+                "route add {} mask {} {}",
+                address,
+                cidr_to_subnet_mask(cidr_prefix),
+                name
+            )
+            .as_str(),
+        )
+        .await
+    }
+
+    async fn remove_ipv4_route(
+        &self,
+        name: &str,
+        address: Ipv4Addr,
+        cidr_prefix: u8,
+    ) -> Result<(), Error> {
+        run_shell_cmd(
+            format!(
+                "route delete {} mask {} {}",
+                address,
+                cidr_to_subnet_mask(cidr_prefix),
+                name
+            )
+            .as_str(),
+        )
+        .await
+    }
+
+    async fn add_ipv4_ip(
+        &self,
+        name: &str,
+        address: Ipv4Addr,
+        cidr_prefix: u8,
+    ) -> Result<(), Error> {
+        run_shell_cmd(
+            format!(
+                "netsh interface ipv4 add address {} address={} mask={}",
+                name,
+                address,
+                cidr_to_subnet_mask(cidr_prefix)
+            )
+            .as_str(),
+        )
+        .await
+    }
+
+    async fn set_link_status(&self, name: &str, up: bool) -> Result<(), Error> {
+        run_shell_cmd(
+            format!(
+                "netsh interface set interface {} {}",
+                name,
+                if up { "enable" } else { "disable" }
+            )
+            .as_str(),
+        )
+        .await
+    }
+
+    async fn remove_ip(&self, name: &str, ip: Option<Ipv4Addr>) -> Result<(), Error> {
+        if ip.is_none() {
+            run_shell_cmd(format!("netsh interface ipv4 delete address {}", name).as_str()).await
+        } else {
+            run_shell_cmd(
+                format!(
+                    "netsh interface ipv4 delete address {} address={}",
+                    name,
+                    ip.unwrap().to_string()
+                )
+                .as_str(),
+            )
+            .await
+        }
+    }
+
+    async fn wait_interface_show(&self, name: &str) -> Result<(), Error> {
+        Ok(
+            tokio::time::timeout(std::time::Duration::from_secs(10), async move {
+                loop {
+                    let Ok(_) = run_shell_cmd(
+                        format!("netsh interface ipv4 show interfaces {}", name).as_str(),
+                    )
+                    .await
+                    else {
+                        tokio::time::sleep(std::time::Duration::from_millis(100)).await;
+                        continue;
+                    };
+                    break;
+                }
+            })
+            .await?,
+        )
+    }
+}
+
+#[cfg(target_os = "macos")]
+pub type IfConfiger = MacIfConfiger;
+
+#[cfg(target_os = "linux")]
+pub type IfConfiger = LinuxIfConfiger;
+
+#[cfg(target_os = "windows")]
+pub type IfConfiger = WindowsIfConfiger;

easytier-core/src/common/mod.rs

@@ -0,0 +1,9 @@
+pub mod config_fs;
+pub mod constants;
+pub mod error;
+pub mod global_ctx;
+pub mod ifcfg;
+pub mod netns;
+pub mod network;
+pub mod rkyv_util;
+pub mod stun;

easytier-core/src/common/netns.rs

@@ -0,0 +1,118 @@
+use futures::Future;
+use once_cell::sync::Lazy;
+use tokio::sync::Mutex;
+
+#[cfg(target_os = "linux")]
+use nix::sched::{setns, CloneFlags};
+#[cfg(target_os = "linux")]
+use std::os::fd::AsFd;
+
+pub struct NetNSGuard {
+    #[cfg(target_os = "linux")]
+    old_ns: Option<std::fs::File>,
+}
+
+type NetNSLock = Mutex<()>;
+static LOCK: Lazy<NetNSLock> = Lazy::new(|| Mutex::new(()));
+pub static ROOT_NETNS_NAME: &str = "_root_ns";
+
+#[cfg(target_os = "linux")]
+impl NetNSGuard {
+    pub fn new(ns: Option<String>) -> Box<Self> {
+        let old_ns = if ns.is_some() {
+            let old_ns = if cfg!(target_os = "linux") {
+                Some(std::fs::File::open("/proc/self/ns/net").unwrap())
+            } else {
+                None
+            };
+            Self::switch_ns(ns);
+            old_ns
+        } else {
+            None
+        };
+        Box::new(NetNSGuard { old_ns })
+    }
+
+    fn switch_ns(name: Option<String>) {
+        if name.is_none() {
+            return;
+        }
+
+        let ns_path: String;
+        let name = name.unwrap();
+        if name == ROOT_NETNS_NAME {
+            ns_path = "/proc/1/ns/net".to_string();
+        } else {
+            ns_path = format!("/var/run/netns/{}", name);
+        }
+
+        let ns = std::fs::File::open(ns_path).unwrap();
+        log::info!(
+            "[INIT NS] switching to new ns_name: {:?}, ns_file: {:?}",
+            name,
+            ns
+        );
+
+        setns(ns.as_fd(), CloneFlags::CLONE_NEWNET).unwrap();
+    }
+}
+
+#[cfg(target_os = "linux")]
+impl Drop for NetNSGuard {
+    fn drop(&mut self) {
+        if self.old_ns.is_none() {
+            return;
+        }
+        log::info!("[INIT NS] switching back to old ns, ns: {:?}", self.old_ns);
+        setns(
+            self.old_ns.as_ref().unwrap().as_fd(),
+            CloneFlags::CLONE_NEWNET,
+        )
+        .unwrap();
+    }
+}
+
+#[cfg(not(target_os = "linux"))]
+impl NetNSGuard {
+    pub fn new(_ns: Option<String>) -> Box<Self> {
+        Box::new(NetNSGuard {})
+    }
+}
+
+#[derive(Clone)]
+pub struct NetNS {
+    name: Option<String>,
+}
+
+impl NetNS {
+    pub fn new(name: Option<String>) -> Self {
+        NetNS { name }
+    }
+
+    pub async fn run_async<F, Fut, Ret>(&self, f: F) -> Ret
+    where
+        F: FnOnce() -> Fut,
+        Fut: Future<Output = Ret>,
+    {
+        // TODO: do we really need this lock
+        // let _lock = LOCK.lock().await;
+        let _guard = NetNSGuard::new(self.name.clone());
+        f().await
+    }
+
+    pub fn run<F, Ret>(&self, f: F) -> Ret
+    where
+        F: FnOnce() -> Ret,
+    {
+        let _guard = NetNSGuard::new(self.name.clone());
+        f()
+    }
+
+    pub fn guard(&self) -> Box<NetNSGuard> {
+        NetNSGuard::new(self.name.clone())
+    }
+
+    pub fn name(&self) -> Option<String> {
+        self.name.clone()
+    }
+}

easytier-core/src/common/network.rs

@@ -0,0 +1,218 @@
+use std::{ops::Deref, sync::Arc};
+
+use easytier_rpc::peer::GetIpListResponse;
+use pnet::datalink::NetworkInterface;
+use tokio::{
+    sync::{Mutex, RwLock},
+    task::JoinSet,
+};
+
+use super::{constants::DIRECT_CONNECTOR_IP_LIST_TIMEOUT_SEC, netns::NetNS};
+
+struct InterfaceFilter {
+    iface: NetworkInterface,
+}
+
+#[cfg(target_os = "linux")]
+impl InterfaceFilter {
+    async fn is_iface_bridge(&self) -> bool {
+        let path = format!("/sys/class/net/{}/bridge", self.iface.name);
+        tokio::fs::metadata(&path).await.is_ok()
+    }
+
+    async fn is_iface_phsical(&self) -> bool {
+        let path = format!("/sys/class/net/{}/device", self.iface.name);
+        tokio::fs::metadata(&path).await.is_ok()
+    }
+
+    async fn filter_iface(&self) -> bool {
+        tracing::trace!(
+            "filter linux iface: {:?}, is_point_to_point: {}, is_loopback: {}, is_up: {}, is_lower_up: {}, is_bridge: {}, is_physical: {}",
+            self.iface,
+            self.iface.is_point_to_point(),
+            self.iface.is_loopback(),
+            self.iface.is_up(),
+            self.iface.is_lower_up(),
+            self.is_iface_bridge().await,
+            self.is_iface_phsical().await,
+        );
+
+        !self.iface.is_point_to_point()
+            && !self.iface.is_loopback()
+            && self.iface.is_up()
+            && self.iface.is_lower_up()
+            && (self.is_iface_bridge().await || self.is_iface_phsical().await)
+    }
+}
+
+#[cfg(target_os = "macos")]
+impl InterfaceFilter {
+    async fn is_interface_physical(interface_name: &str) -> bool {
+        let output = tokio::process::Command::new("networksetup")
+            .args(&["-listallhardwareports"])
+            .output()
+            .await
+            .expect("Failed to execute command");
+
+        let stdout = std::str::from_utf8(&output.stdout).expect("Invalid UTF-8");
+
+        let lines: Vec<&str> = stdout.lines().collect();
+
+        for i in 0..lines.len() {
+            let line = lines[i];
+
+            if line.contains("Device:") && line.contains(interface_name) {
+                let next_line = lines[i + 1];
+                if next_line.contains("Virtual Interface") {
+                    return false;
+                } else {
+                    return true;
+                }
+            }
+        }
+
+        false
+    }
+
+    async fn filter_iface(&self) -> bool {
+        !self.iface.is_point_to_point()
+            && !self.iface.is_loopback()
+            && self.iface.is_up()
+            && Self::is_interface_physical(&self.iface.name).await
+    }
+}
+
+#[cfg(target_os = "windows")]
+impl InterfaceFilter {
+    async fn filter_iface(&self) -> bool {
+        !self.iface.is_point_to_point() && !self.iface.is_loopback() && self.iface.is_up()
+    }
+}
+
+pub async fn local_ipv4() -> std::io::Result<std::net::Ipv4Addr> {
+    let socket = tokio::net::UdpSocket::bind("0.0.0.0:0").await?;
+    socket.connect("8.8.8.8:80").await?;
+    let addr = socket.local_addr()?;
+    match addr.ip() {
+        std::net::IpAddr::V4(ip) => Ok(ip),
+        std::net::IpAddr::V6(_) => Err(std::io::Error::new(
+            std::io::ErrorKind::AddrNotAvailable,
+            "no ipv4 address",
+        )),
+    }
+}
+
+pub async fn local_ipv6() -> std::io::Result<std::net::Ipv6Addr> {
+    let socket = tokio::net::UdpSocket::bind("[::]:0").await?;
+    socket
+        .connect("[2001:4860:4860:0000:0000:0000:0000:8888]:80")
+        .await?;
+    let addr = socket.local_addr()?;
+    match addr.ip() {
+        std::net::IpAddr::V6(ip) => Ok(ip),
+        std::net::IpAddr::V4(_) => Err(std::io::Error::new(
+            std::io::ErrorKind::AddrNotAvailable,
+            "no ipv4 address",
+        )),
+    }
+}
+
+pub struct IPCollector {
+    cached_ip_list: Arc<RwLock<GetIpListResponse>>,
+    collect_ip_task: Mutex<JoinSet<()>>,
+    net_ns: NetNS,
+}
+
+impl IPCollector {
+    pub fn new(net_ns: NetNS) -> Self {
+        Self {
+            cached_ip_list: Arc::new(RwLock::new(GetIpListResponse::new())),
+            collect_ip_task: Mutex::new(JoinSet::new()),
+            net_ns,
+        }
+    }
+
+    pub async fn collect_ip_addrs(&self) -> GetIpListResponse {
+        let mut task = self.collect_ip_task.lock().await;
+        if task.is_empty() {
+            let cached_ip_list = self.cached_ip_list.clone();
+            *cached_ip_list.write().await =
+                Self::do_collect_ip_addrs(false, self.net_ns.clone()).await;
+            let net_ns = self.net_ns.clone();
+            task.spawn(async move {
+                loop {
+                    let ip_addrs = Self::do_collect_ip_addrs(true, net_ns.clone()).await;
+                    *cached_ip_list.write().await = ip_addrs;
+                    tokio::time::sleep(std::time::Duration::from_secs(
+                        DIRECT_CONNECTOR_IP_LIST_TIMEOUT_SEC,
+                    ))
+                    .await;
+                }
+            });
+        }
+
+        return self.cached_ip_list.read().await.deref().clone();
+    }
+
+    #[tracing::instrument(skip(net_ns))]
+    async fn do_collect_ip_addrs(with_public: bool, net_ns: NetNS) -> GetIpListResponse {
+        let mut ret = easytier_rpc::peer::GetIpListResponse {
+            public_ipv4: "".to_string(),
+            interface_ipv4s: vec![],
+            public_ipv6: "".to_string(),
+            interface_ipv6s: vec![],
+        };
+
+        if with_public {
+            if let Some(v4_addr) =
+                public_ip::addr_with(public_ip::http::ALL, public_ip::Version::V4).await
+            {
+                ret.public_ipv4 = v4_addr.to_string();
+            }
+
+            if let Some(v6_addr) = public_ip::addr_v6().await {
+                ret.public_ipv6 = v6_addr.to_string();
+            }
+        }
+
+        let _g = net_ns.guard();
+        let ifaces = pnet::datalink::interfaces();
+        for iface in ifaces {
+            let f = InterfaceFilter {
+                iface: iface.clone(),
+            };
+
+            if !f.filter_iface().await {
+                continue;
+            }
+
+            for ip in iface.ips {
+                let ip: std::net::IpAddr = ip.ip();
+                if ip.is_loopback() || ip.is_multicast() {
+                    continue;
+                }
+                if ip.is_ipv4() {
+                    ret.interface_ipv4s.push(ip.to_string());
+                } else if ip.is_ipv6() {
+                    ret.interface_ipv6s.push(ip.to_string());
+                }
+            }
+        }
+
+        if let Ok(v4_addr) = local_ipv4().await {
+            tracing::trace!("got local ipv4: {}", v4_addr);
+            if !ret.interface_ipv4s.contains(&v4_addr.to_string()) {
+                ret.interface_ipv4s.push(v4_addr.to_string());
+            }
+        }
+
+        if let Ok(v6_addr) = local_ipv6().await {
+            tracing::trace!("got local ipv6: {}", v6_addr);
+            if !ret.interface_ipv6s.contains(&v6_addr.to_string()) {
+                ret.interface_ipv6s.push(v6_addr.to_string());
+            }
+        }
+
+        ret
+    }
+}

easytier-core/src/common/rkyv_util.rs

@@ -0,0 +1,54 @@
+use rkyv::{
+    validation::{validators::DefaultValidator, CheckTypeError},
+    vec::ArchivedVec,
+    Archive, CheckBytes, Serialize,
+};
+use tokio_util::bytes::{Bytes, BytesMut};
+
+pub fn decode_from_bytes_checked<'a, T: Archive>(
+    bytes: &'a [u8],
+) -> Result<&'a T::Archived, CheckTypeError<T::Archived, DefaultValidator<'a>>>
+where
+    T::Archived: CheckBytes<DefaultValidator<'a>>,
+{
+    rkyv::check_archived_root::<T>(bytes)
+}
+
+pub fn decode_from_bytes<'a, T: Archive>(
+    bytes: &'a [u8],
+) -> Result<&'a T::Archived, CheckTypeError<T::Archived, DefaultValidator<'a>>>
+where
+    T::Archived: CheckBytes<DefaultValidator<'a>>,
+{
+    // rkyv::check_archived_root::<T>(bytes)
+    unsafe { Ok(rkyv::archived_root::<T>(bytes)) }
+}
+
+// allow deseraial T to Bytes
+pub fn encode_to_bytes<T, const N: usize>(val: &T) -> Bytes
+where
+    T: Serialize<rkyv::ser::serializers::AllocSerializer<N>>,
+{
+    let ret = rkyv::to_bytes::<_, N>(val).unwrap();
+    // let mut r = BytesMut::new();
+    // r.extend_from_slice(&ret);
+    // r.freeze()
+    ret.into_boxed_slice().into()
+}
+
+pub fn extract_bytes_from_archived_vec(raw_data: &Bytes, archived_data: &ArchivedVec<u8>) -> Bytes {
+    let ptr_range = archived_data.as_ptr_range();
+    let offset = ptr_range.start as usize - raw_data.as_ptr() as usize;
+    let len = ptr_range.end as usize - ptr_range.start as usize;
+    return raw_data.slice(offset..offset + len);
+}
+
+pub fn extract_bytes_mut_from_archived_vec(
+    raw_data: &mut BytesMut,
+    archived_data: &ArchivedVec<u8>,
+) -> BytesMut {
+    let ptr_range = archived_data.as_ptr_range();
+    let offset = ptr_range.start as usize - raw_data.as_ptr() as usize;
+    let len = ptr_range.end as usize - ptr_range.start as usize;
+    raw_data.split_off(offset).split_to(len)
+}

easytier-core/src/common/stun.rs

@@ -0,0 +1,433 @@
+use std::net::{Ipv4Addr, Ipv6Addr, SocketAddr, SocketAddrV4, SocketAddrV6};
+use std::sync::Arc;
+use std::time::Duration;
+
+use crossbeam::atomic::AtomicCell;
+use easytier_rpc::{NatType, StunInfo};
+use stun_format::Attr;
+use tokio::net::{lookup_host, UdpSocket};
+use tokio::sync::RwLock;
+use tokio::task::JoinSet;
+
+use crate::common::error::Error;
+
+struct Stun {
+    stun_server: String,
+    req_repeat: u8,
+    resp_timeout: Duration,
+}
+
+#[derive(Debug, Clone, Copy)]
+struct BindRequestResponse {
+    source_addr: SocketAddr,
+    mapped_socket_addr: Option<SocketAddr>,
+    changed_socket_addr: Option<SocketAddr>,
+
+    ip_changed: bool,
+    port_changed: bool,
+}
+
+impl BindRequestResponse {
+    pub fn get_mapped_addr_no_check(&self) -> &SocketAddr {
+        self.mapped_socket_addr.as_ref().unwrap()
+    }
+}
+
+impl Stun {
+    pub fn new(stun_server: String) -> Self {
+        Self {
+            stun_server,
+            req_repeat: 3,
+            resp_timeout: Duration::from_millis(3000),
+        }
+    }
+
+    async fn wait_stun_response<'a, const N: usize>(
+        &self,
+        buf: &'a mut [u8; N],
+        udp: &UdpSocket,
+        tids: &Vec<u128>,
+    ) -> Result<(stun_format::Msg<'a>, SocketAddr), Error> {
+        let mut now = tokio::time::Instant::now();
+        let deadline = now + self.resp_timeout;
+
+        while now < deadline {
+            let mut udp_buf = [0u8; 1500];
+            let (len, remote_addr) =
+                tokio::time::timeout(deadline - now, udp.recv_from(udp_buf.as_mut_slice()))
+                    .await??;
+            now = tokio::time::Instant::now();
+
+            if len < 20 {
+                continue;
+            }
+
+            // TODO:: we cannot borrow `buf` directly in udp recv_from, so we copy it here
+            unsafe { std::ptr::copy(udp_buf.as_ptr(), buf.as_ptr() as *mut u8, len) };
+
+            let msg = stun_format::Msg::<'a>::from(&buf[..]);
+            tracing::trace!(b = ?&udp_buf[..len], ?msg, ?tids, "recv stun response");
+
+            if msg.typ().is_none() || msg.tid().is_none() {
+                continue;
+            }
+
+            if matches!(
+                msg.typ().as_ref().unwrap(),
+                stun_format::MsgType::BindingResponse
+            ) && tids.contains(msg.tid().as_ref().unwrap())
+            {
+                return Ok((msg, remote_addr));
+            }
+        }
+
+        Err(Error::Unknown)
+    }
+
+    fn stun_addr(addr: stun_format::SocketAddr) -> SocketAddr {
+        match addr {
+            stun_format::SocketAddr::V4(ip, port) => {
+                SocketAddr::V4(SocketAddrV4::new(Ipv4Addr::from(ip), port))
+            }
+            stun_format::SocketAddr::V6(ip, port) => {
+                SocketAddr::V6(SocketAddrV6::new(Ipv6Addr::from(ip), port, 0, 0))
+            }
+        }
+    }
+
+    fn extrace_mapped_addr(msg: &stun_format::Msg) -> Option<SocketAddr> {
+        let mut mapped_addr = None;
+        for x in msg.attrs_iter() {
+            match x {
+                Attr::MappedAddress(addr) => {
+                    if mapped_addr.is_none() {
+                        let _ = mapped_addr.insert(Self::stun_addr(addr));
+                    }
+                }
+                Attr::XorMappedAddress(addr) => {
+                    if mapped_addr.is_none() {
+                        let _ = mapped_addr.insert(Self::stun_addr(addr));
+                    }
+                }
+                _ => {}
+            }
+        }
+        mapped_addr
+    }
+
+    fn extract_changed_addr(msg: &stun_format::Msg) -> Option<SocketAddr> {
+        let mut changed_addr = None;
+        for x in msg.attrs_iter() {
+            match x {
+                Attr::ChangedAddress(addr) => {
+                    if changed_addr.is_none() {
+                        let _ = changed_addr.insert(Self::stun_addr(addr));
+                    }
+                }
+                _ => {}
+            }
+        }
+        changed_addr
+    }
+
+    pub async fn bind_request(
+        &self,
+        source_port: u16,
+        change_ip: bool,
+        change_port: bool,
+    ) -> Result<BindRequestResponse, Error> {
+        let stun_host = lookup_host(&self.stun_server)
+            .await?
+            .next()
+            .ok_or(Error::NotFound)?;
+        // let udp_socket = socket2::Socket::new(
+        //     match stun_host {
+        //         SocketAddr::V4(..) => socket2::Domain::IPV4,
+        //         SocketAddr::V6(..) => socket2::Domain::IPV6,
+        //     },
+        //     socket2::Type::DGRAM,
+        //     Some(socket2::Protocol::UDP),
+        // )?;
+        // udp_socket.set_reuse_port(true)?;
+        // udp_socket.set_reuse_address(true)?;
+        let udp = UdpSocket::bind(format!("0.0.0.0:{}", source_port)).await?;
+
+        // repeat req in case of packet loss
+        let mut tids = vec![];
+        for _ in 0..self.req_repeat {
+            let mut buf = [0u8; 28];
+            // memset buf
+            unsafe { std::ptr::write_bytes(buf.as_mut_ptr(), 0, buf.len()) };
+            let mut msg = stun_format::MsgBuilder::from(buf.as_mut_slice());
+            msg.typ(stun_format::MsgType::BindingRequest).unwrap();
+            let tid = rand::random::<u32>();
+            msg.tid(tid as u128).unwrap();
+            if change_ip || change_port {
+                msg.add_attr(Attr::ChangeRequest {
+                    change_ip,
+                    change_port,
+                })
+                .unwrap();
+            }
+
+            tids.push(tid as u128);
+            tracing::trace!(b = ?msg.as_bytes(), tid, "send stun request");
+            udp.send_to(msg.as_bytes(), &stun_host).await?;
+        }
+
+        tracing::trace!("waiting stun response");
+        let mut buf = [0; 1620];
+        let (msg, recv_addr) = self.wait_stun_response(&mut buf, &udp, &tids).await?;
+
+        let changed_socket_addr = Self::extract_changed_addr(&msg);
+        let ip_changed = stun_host.ip() != recv_addr.ip();
+        let port_changed = stun_host.port() != recv_addr.port();
+
+        let resp = BindRequestResponse {
+            source_addr: udp.local_addr()?,
+            mapped_socket_addr: Self::extrace_mapped_addr(&msg),
+            changed_socket_addr,
+            ip_changed,
+            port_changed,
+        };
+
+        tracing::info!(
+            ?stun_host,
+            ?recv_addr,
+            ?changed_socket_addr,
+            "finish stun bind request"
+        );
+
+        Ok(resp)
+    }
+}
+
+struct UdpNatTypeDetector {
+    stun_servers: Vec<String>,
+}
+
+impl UdpNatTypeDetector {
+    pub fn new(stun_servers: Vec<String>) -> Self {
+        Self { stun_servers }
+    }
+
+    async fn get_udp_nat_type(&self, mut source_port: u16) -> NatType {
+        // Like classic STUN (rfc3489). Detect NAT behavior for UDP.
+        // Modified from rfc3489. Requires at least two STUN servers.
+        let mut ret_test1_1 = None;
+        let mut ret_test1_2 = None;
+        let mut ret_test2 = None;
+        let mut ret_test3 = None;
+
+        if source_port == 0 {
+            let udp = UdpSocket::bind("0.0.0.0:0").await.unwrap();
+            source_port = udp.local_addr().unwrap().port();
+        }
+
+        let mut succ = false;
+        for server_ip in &self.stun_servers {
+            let stun = Stun::new(server_ip.clone());
+            let ret = stun.bind_request(source_port, false, false).await;
+            if ret.is_err() {
+                // Try another STUN server
+                continue;
+            }
+            if ret_test1_1.is_none() {
+                ret_test1_1 = ret.ok();
+                continue;
+            }
+            ret_test1_2 = ret.ok();
+            let ret = stun.bind_request(source_port, true, true).await;
+            if let Ok(resp) = ret {
+                if !resp.ip_changed || !resp.port_changed {
+                    // Try another STUN server
+                    continue;
+                }
+            }
+            ret_test2 = ret.ok();
+            ret_test3 = stun.bind_request(source_port, false, true).await.ok();
+            succ = true;
+            break;
+        }
+
+        if !succ {
+            return NatType::Unknown;
+        }
+
+        tracing::info!(
+            ?ret_test1_1,
+            ?ret_test1_2,
+            ?ret_test2,
+            ?ret_test3,
+            "finish stun test, try to detect nat type"
+        );
+
+        let ret_test1_1 = ret_test1_1.unwrap();
+        let ret_test1_2 = ret_test1_2.unwrap();
+
+        if ret_test1_1.mapped_socket_addr != ret_test1_2.mapped_socket_addr {
+            return NatType::Symmetric;
+        }
+
+        if ret_test1_1.mapped_socket_addr.is_some()
+            && ret_test1_1.source_addr == ret_test1_1.mapped_socket_addr.unwrap()
+        {
+            if !ret_test2.is_none() {
+                return NatType::OpenInternet;
+            } else {
+                return NatType::SymUdpFirewall;
+            }
+        } else {
+            if let Some(ret_test2) = ret_test2 {
+                if source_port == ret_test2.get_mapped_addr_no_check().port()
+                    && source_port == ret_test1_1.get_mapped_addr_no_check().port()
+                {
+                    return NatType::NoPat;
+                } else {
+                    return NatType::FullCone;
+                }
+            } else {
+                if !ret_test3.is_none() {
+                    return NatType::Restricted;
+                } else {
+                    return NatType::PortRestricted;
+                }
+            }
+        }
+    }
+}
+
+#[async_trait::async_trait]
+#[auto_impl::auto_impl(&, Arc, Box)]
+pub trait StunInfoCollectorTrait: Send + Sync {
+    fn get_stun_info(&self) -> StunInfo;
+    async fn get_udp_port_mapping(&self, local_port: u16) -> Result<SocketAddr, Error>;
+}
+
+pub struct StunInfoCollector {
+    stun_servers: Arc<RwLock<Vec<String>>>,
+    udp_nat_type: Arc<AtomicCell<(NatType, std::time::Instant)>>,
+    redetect_notify: Arc<tokio::sync::Notify>,
+    tasks: JoinSet<()>,
+}
+
+#[async_trait::async_trait]
+impl StunInfoCollectorTrait for StunInfoCollector {
+    fn get_stun_info(&self) -> StunInfo {
+        let (typ, time) = self.udp_nat_type.load();
+        StunInfo {
+            udp_nat_type: typ as i32,
+            tcp_nat_type: 0,
+            last_update_time: time.elapsed().as_secs() as i64,
+        }
+    }
+
+    async fn get_udp_port_mapping(&self, local_port: u16) -> Result<SocketAddr, Error> {
+        let stun_servers = self.stun_servers.read().await.clone();
+        for server in stun_servers.iter() {
+            let stun = Stun::new(server.clone());
+            let Ok(ret) = stun.bind_request(local_port, false, false).await else {
+                tracing::warn!(?server, "stun bind request failed");
+                continue;
+            };
+            if let Some(mapped_addr) = ret.mapped_socket_addr {
+                return Ok(mapped_addr);
+            }
+        }
+        Err(Error::NotFound)
+    }
+}
+
+impl StunInfoCollector {
+    pub fn new(stun_servers: Vec<String>) -> Self {
+        let mut ret = Self {
+            stun_servers: Arc::new(RwLock::new(stun_servers)),
+            udp_nat_type: Arc::new(AtomicCell::new((
+                NatType::Unknown,
+                std::time::Instant::now(),
+            ))),
+            redetect_notify: Arc::new(tokio::sync::Notify::new()),
+            tasks: JoinSet::new(),
+        };
+
+        ret.start_stun_routine();
+
+        ret
+    }
+
+    fn start_stun_routine(&mut self) {
+        let stun_servers = self.stun_servers.clone();
+        let udp_nat_type = self.udp_nat_type.clone();
+        let redetect_notify = self.redetect_notify.clone();
+        self.tasks.spawn(async move {
+            loop {
+                let detector = UdpNatTypeDetector::new(stun_servers.read().await.clone());
+                let ret = detector.get_udp_nat_type(0).await;
+                udp_nat_type.store((ret, std::time::Instant::now()));
+
+                let sleep_sec = match ret {
+                    NatType::Unknown => 15,
+                    _ => 60,
+                };
+                tracing::info!(?ret, ?sleep_sec, "finish udp nat type detect");
+
+                tokio::select! {
+                    _ = redetect_notify.notified() => {}
+                    _ = tokio::time::sleep(Duration::from_secs(sleep_sec)) => {}
+                }
+            }
+        });
+    }
+
+    pub fn update_stun_info(&self) {
+        self.redetect_notify.notify_one();
+    }
+
+    pub async fn set_stun_servers(&self, stun_servers: Vec<String>) {
+        *self.stun_servers.write().await = stun_servers;
+        self.update_stun_info();
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[tokio::test]
+    async fn test_stun_bind_request() {
+        // miwifi / qq seems not correctly responde to change_ip and change_port, they always try to change the src ip and port.
+        // let stun = Stun::new("stun.counterpath.com:3478".to_string());
+        let stun = Stun::new("180.235.108.91:3478".to_string());
+        // let stun = Stun::new("193.22.2.248:3478".to_string());
+
+        // let stun = Stun::new("stun.chat.bilibili.com:3478".to_string());
+        // let stun = Stun::new("stun.miwifi.com:3478".to_string());
+
+        let rs = stun.bind_request(12345, true, true).await.unwrap();
+        assert!(rs.ip_changed);
+        assert!(rs.port_changed);
+
+        let rs = stun.bind_request(12345, true, false).await.unwrap();
+        assert!(rs.ip_changed);
+        assert!(!rs.port_changed);
+
+        let rs = stun.bind_request(12345, false, true).await.unwrap();
+        assert!(!rs.ip_changed);
+        assert!(rs.port_changed);
+
+        let rs = stun.bind_request(12345, false, false).await.unwrap();
+        assert!(!rs.ip_changed);
+        assert!(!rs.port_changed);
+    }
+
+    #[tokio::test]
+    async fn test_udp_nat_type_detect() {
+        let detector = UdpNatTypeDetector::new(vec![
+            "stun.counterpath.com:3478".to_string(),
+            "180.235.108.91:3478".to_string(),
+        ]);
+        let ret = detector.get_udp_nat_type(0).await;
+
+        assert_eq!(ret, NatType::FullCone);
+    }
+}

easytier-core/src/connector/direct.rs

@@ -0,0 +1,325 @@
+// try connect peers directly, with either its public ip or lan ip
+
+use std::sync::Arc;
+
+use crate::{
+    common::{
+        constants::{self, DIRECT_CONNECTOR_BLACKLIST_TIMEOUT_SEC},
+        error::Error,
+        global_ctx::ArcGlobalCtx,
+        network::IPCollector,
+    },
+    peers::{peer_manager::PeerManager, peer_rpc::PeerRpcManager, PeerId},
+};
+
+use easytier_rpc::{peer::GetIpListResponse, PeerConnInfo};
+use tokio::{task::JoinSet, time::timeout};
+use tracing::Instrument;
+
+use super::create_connector_by_url;
+
+#[tarpc::service]
+pub trait DirectConnectorRpc {
+    async fn get_ip_list() -> GetIpListResponse;
+}
+
+#[async_trait::async_trait]
+pub trait PeerManagerForDirectConnector {
+    async fn list_peers(&self) -> Vec<PeerId>;
+    async fn list_peer_conns(&self, peer_id: &PeerId) -> Option<Vec<PeerConnInfo>>;
+    fn get_peer_rpc_mgr(&self) -> Arc<PeerRpcManager>;
+}
+
+#[async_trait::async_trait]
+impl PeerManagerForDirectConnector for PeerManager {
+    async fn list_peers(&self) -> Vec<PeerId> {
+        let mut ret = vec![];
+
+        let routes = self.list_routes().await;
+        for r in routes.iter() {
+            ret.push(r.peer_id.parse().unwrap());
+        }
+
+        ret
+    }
+
+    async fn list_peer_conns(&self, peer_id: &PeerId) -> Option<Vec<PeerConnInfo>> {
+        self.get_peer_map().list_peer_conns(peer_id).await
+    }
+
+    fn get_peer_rpc_mgr(&self) -> Arc<PeerRpcManager> {
+        self.get_peer_rpc_mgr()
+    }
+}
+
+#[derive(Clone)]
+struct DirectConnectorManagerRpcServer {
+    // TODO: this only cache for one src peer, should make it global
+    ip_list_collector: Arc<IPCollector>,
+}
+
+#[tarpc::server]
+impl DirectConnectorRpc for DirectConnectorManagerRpcServer {
+    async fn get_ip_list(self, _: tarpc::context::Context) -> GetIpListResponse {
+        return self.ip_list_collector.collect_ip_addrs().await;
+    }
+}
+
+impl DirectConnectorManagerRpcServer {
+    pub fn new(ip_collector: Arc<IPCollector>) -> Self {
+        Self {
+            ip_list_collector: ip_collector,
+        }
+    }
+}
+
+#[derive(Hash, Eq, PartialEq, Clone)]
+struct DstBlackListItem(PeerId, String);
+
+struct DirectConnectorManagerData {
+    global_ctx: ArcGlobalCtx,
+    peer_manager: Arc<PeerManager>,
+    dst_blacklist: timedmap::TimedMap<DstBlackListItem, ()>,
+}
+
+impl std::fmt::Debug for DirectConnectorManagerData {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("DirectConnectorManagerData")
+            .field("peer_manager", &self.peer_manager)
+            .finish()
+    }
+}
+
+pub struct DirectConnectorManager {
+    my_node_id: uuid::Uuid,
+    global_ctx: ArcGlobalCtx,
+    data: Arc<DirectConnectorManagerData>,
+
+    tasks: JoinSet<()>,
+}
+
+impl DirectConnectorManager {
+    pub fn new(
+        my_node_id: uuid::Uuid,
+        global_ctx: ArcGlobalCtx,
+        peer_manager: Arc<PeerManager>,
+    ) -> Self {
+        Self {
+            my_node_id,
+            global_ctx: global_ctx.clone(),
+            data: Arc::new(DirectConnectorManagerData {
+                global_ctx,
+                peer_manager,
+                dst_blacklist: timedmap::TimedMap::new(),
+            }),
+            tasks: JoinSet::new(),
+        }
+    }
+
+    pub fn run(&mut self) {
+        self.run_as_server();
+        self.run_as_client();
+    }
+
+    pub fn run_as_server(&mut self) {
+        self.data.peer_manager.get_peer_rpc_mgr().run_service(
+            constants::DIRECT_CONNECTOR_SERVICE_ID,
+            DirectConnectorManagerRpcServer::new(self.global_ctx.get_ip_collector()).serve(),
+        );
+    }
+
+    pub fn run_as_client(&mut self) {
+        let data = self.data.clone();
+        let my_node_id = self.my_node_id.clone();
+        self.tasks.spawn(
+            async move {
+                loop {
+                    let peers = data.peer_manager.list_peers().await;
+                    let mut tasks = JoinSet::new();
+                    for peer_id in peers {
+                        if peer_id == my_node_id {
+                            continue;
+                        }
+                        tasks.spawn(Self::do_try_direct_connect(data.clone(), peer_id));
+                    }
+
+                    while let Some(task_ret) = tasks.join_next().await {
+                        tracing::trace!(?task_ret, "direct connect task ret");
+                    }
+                    tokio::time::sleep(std::time::Duration::from_secs(5)).await;
+                }
+            }
+            .instrument(tracing::info_span!("direct_connector_client", my_id = ?self.my_node_id)),
+        );
+    }
+
+    async fn do_try_connect_to_ip(
+        data: Arc<DirectConnectorManagerData>,
+        dst_peer_id: PeerId,
+        addr: String,
+    ) -> Result<(), Error> {
+        data.dst_blacklist.cleanup();
+        if data
+            .dst_blacklist
+            .contains(&DstBlackListItem(dst_peer_id.clone(), addr.clone()))
+        {
+            tracing::trace!("try_connect_to_ip failed, addr in blacklist: {}", addr);
+            return Err(Error::UrlInBlacklist);
+        }
+
+        let connector = create_connector_by_url(&addr, data.global_ctx.get_ip_collector()).await?;
+        let (peer_id, conn_id) = timeout(
+            std::time::Duration::from_secs(5),
+            data.peer_manager.try_connect(connector),
+        )
+        .await??;
+
+        // let (peer_id, conn_id) = data.peer_manager.try_connect(connector).await?;
+
+        if peer_id != dst_peer_id {
+            tracing::info!(
+                "connect to ip succ: {}, but peer id mismatch, expect: {}, actual: {}",
+                addr,
+                dst_peer_id,
+                peer_id
+            );
+            data.peer_manager
+                .get_peer_map()
+                .close_peer_conn(&peer_id, &conn_id)
+                .await?;
+            return Err(Error::InvalidUrl(addr));
+        }
+        Ok(())
+    }
+
+    #[tracing::instrument]
+    async fn try_connect_to_ip(
+        data: Arc<DirectConnectorManagerData>,
+        dst_peer_id: PeerId,
+        addr: String,
+    ) {
+        let ret = Self::do_try_connect_to_ip(data.clone(), dst_peer_id, addr.clone()).await;
+        if let Err(e) = ret {
+            if !matches!(e, Error::UrlInBlacklist) {
+                tracing::info!(
+                    "try_connect_to_ip failed: {:?}, peer_id: {}",
+                    e,
+                    dst_peer_id
+                );
+                data.dst_blacklist.insert(
+                    DstBlackListItem(dst_peer_id.clone(), addr.clone()),
+                    (),
+                    std::time::Duration::from_secs(DIRECT_CONNECTOR_BLACKLIST_TIMEOUT_SEC),
+                );
+            }
+        } else {
+            log::info!("try_connect_to_ip success, peer_id: {}", dst_peer_id);
+        }
+    }
+
+    #[tracing::instrument]
+    async fn do_try_direct_connect(
+        data: Arc<DirectConnectorManagerData>,
+        dst_peer_id: PeerId,
+    ) -> Result<(), Error> {
+        let peer_manager = data.peer_manager.clone();
+        // check if we have direct connection with dst_peer_id
+        if let Some(c) = peer_manager.list_peer_conns(&dst_peer_id).await {
+            // currently if we have any type of direct connection (udp or tcp), we will not try to connect
+            if !c.is_empty() {
+                return Ok(());
+            }
+        }
+
+        log::trace!("try direct connect to peer: {}", dst_peer_id);
+
+        let ip_list = peer_manager
+            .get_peer_rpc_mgr()
+            .do_client_rpc_scoped(1, dst_peer_id, |c| async {
+                let client =
+                    DirectConnectorRpcClient::new(tarpc::client::Config::default(), c).spawn();
+                let ip_list = client.get_ip_list(tarpc::context::current()).await;
+                tracing::info!(ip_list = ?ip_list, dst_peer_id = ?dst_peer_id, "got ip list");
+                ip_list
+            })
+            .await?;
+
+        let mut tasks = JoinSet::new();
+        ip_list.interface_ipv4s.iter().for_each(|ip| {
+            let addr = format!("{}://{}:{}", "tcp", ip, 11010);
+            tasks.spawn(Self::try_connect_to_ip(
+                data.clone(),
+                dst_peer_id.clone(),
+                addr,
+            ));
+        });
+
+        let addr = format!("{}://{}:{}", "tcp", ip_list.public_ipv4.clone(), 11010);
+        tasks.spawn(Self::try_connect_to_ip(
+            data.clone(),
+            dst_peer_id.clone(),
+            addr,
+        ));
+
+        while let Some(ret) = tasks.join_next().await {
+            if let Err(e) = ret {
+                log::error!("join direct connect task failed: {:?}", e);
+            }
+        }
+
+        Ok(())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use crate::{
+        connector::direct::DirectConnectorManager,
+        instance::listeners::ListenerManager,
+        peers::tests::{
+            connect_peer_manager, create_mock_peer_manager, wait_route_appear,
+            wait_route_appear_with_cost,
+        },
+        tunnels::tcp_tunnel::TcpTunnelListener,
+    };
+
+    #[tokio::test]
+    async fn direct_connector_basic_test() {
+        let p_a = create_mock_peer_manager().await;
+        let p_b = create_mock_peer_manager().await;
+        let p_c = create_mock_peer_manager().await;
+        connect_peer_manager(p_a.clone(), p_b.clone()).await;
+        connect_peer_manager(p_b.clone(), p_c.clone()).await;
+
+        wait_route_appear(p_a.clone(), p_c.my_node_id())
+            .await
+            .unwrap();
+
+        let mut dm_a =
+            DirectConnectorManager::new(p_a.my_node_id(), p_a.get_global_ctx(), p_a.clone());
+        let mut dm_c =
+            DirectConnectorManager::new(p_c.my_node_id(), p_c.get_global_ctx(), p_c.clone());
+
+        dm_a.run_as_client();
+        dm_c.run_as_server();
+
+        let mut lis_c = ListenerManager::new(
+            p_c.my_node_id(),
+            p_c.get_global_ctx().net_ns.clone(),
+            p_c.clone(),
+        );
+
+        lis_c
+            .add_listener(TcpTunnelListener::new(
+                "tcp://0.0.0.0:11010".parse().unwrap(),
+            ))
+            .await
+            .unwrap();
+
+        lis_c.run().await.unwrap();
+
+        wait_route_appear_with_cost(p_a.clone(), p_c.my_node_id(), Some(1))
+            .await
+            .unwrap();
+    }
+}

easytier-core/src/connector/manual.rs

@@ -0,0 +1,384 @@
+use std::{collections::BTreeSet, sync::Arc};
+
+use dashmap::{DashMap, DashSet};
+use easytier_rpc::{
+    connector_manage_rpc_server::ConnectorManageRpc, Connector, ConnectorStatus,
+    ListConnectorRequest, ManageConnectorRequest,
+};
+use tokio::{
+    sync::{broadcast::Receiver, mpsc, Mutex},
+    task::JoinSet,
+    time::timeout,
+};
+
+use crate::{
+    common::{
+        error::Error,
+        global_ctx::{ArcGlobalCtx, GlobalCtxEvent},
+        netns::NetNS,
+    },
+    connector::set_bind_addr_for_peer_connector,
+    peers::peer_manager::PeerManager,
+    tunnels::{Tunnel, TunnelConnector},
+    use_global_var,
+};
+
+use super::create_connector_by_url;
+
+type ConnectorMap = Arc<DashMap<String, Box<dyn TunnelConnector + Send + Sync>>>;
+
+#[derive(Debug, Clone)]
+struct ReconnResult {
+    dead_url: String,
+    peer_id: uuid::Uuid,
+    conn_id: uuid::Uuid,
+}
+
+struct ConnectorManagerData {
+    connectors: ConnectorMap,
+    reconnecting: DashSet<String>,
+    peer_manager: Arc<PeerManager>,
+    alive_conn_urls: Arc<Mutex<BTreeSet<String>>>,
+    // user removed connector urls
+    removed_conn_urls: Arc<DashSet<String>>,
+    net_ns: NetNS,
+    global_ctx: ArcGlobalCtx,
+}
+
+pub struct ManualConnectorManager {
+    my_node_id: uuid::Uuid,
+    global_ctx: ArcGlobalCtx,
+    data: Arc<ConnectorManagerData>,
+    tasks: JoinSet<()>,
+}
+
+impl ManualConnectorManager {
+    pub fn new(
+        my_node_id: uuid::Uuid,
+        global_ctx: ArcGlobalCtx,
+        peer_manager: Arc<PeerManager>,
+    ) -> Self {
+        let connectors = Arc::new(DashMap::new());
+        let tasks = JoinSet::new();
+        let event_subscriber = global_ctx.subscribe();
+
+        let mut ret = Self {
+            my_node_id,
+            global_ctx: global_ctx.clone(),
+            data: Arc::new(ConnectorManagerData {
+                connectors,
+                reconnecting: DashSet::new(),
+                peer_manager,
+                alive_conn_urls: Arc::new(Mutex::new(BTreeSet::new())),
+                removed_conn_urls: Arc::new(DashSet::new()),
+                net_ns: global_ctx.net_ns.clone(),
+                global_ctx,
+            }),
+            tasks,
+        };
+
+        ret.tasks
+            .spawn(Self::conn_mgr_routine(ret.data.clone(), event_subscriber));
+
+        ret
+    }
+
+    pub fn add_connector<T>(&self, connector: T)
+    where
+        T: TunnelConnector + Send + Sync + 'static,
+    {
+        log::info!("add_connector: {}", connector.remote_url());
+        self.data
+            .connectors
+            .insert(connector.remote_url().into(), Box::new(connector));
+    }
+
+    pub async fn add_connector_by_url(&self, url: &str) -> Result<(), Error> {
+        self.add_connector(create_connector_by_url(url, self.global_ctx.get_ip_collector()).await?);
+        Ok(())
+    }
+
+    pub async fn remove_connector(&self, url: &str) -> Result<(), Error> {
+        log::info!("remove_connector: {}", url);
+        if !self.list_connectors().await.iter().any(|x| x.url == url) {
+            return Err(Error::NotFound);
+        }
+        self.data.removed_conn_urls.insert(url.into());
+        Ok(())
+    }
+
+    pub async fn list_connectors(&self) -> Vec<Connector> {
+        let conn_urls: BTreeSet<String> = self
+            .data
+            .connectors
+            .iter()
+            .map(|x| x.key().clone().into())
+            .collect();
+
+        let dead_urls: BTreeSet<String> = Self::collect_dead_conns(self.data.clone())
+            .await
+            .into_iter()
+            .collect();
+
+        let mut ret = Vec::new();
+
+        for conn_url in conn_urls {
+            let mut status = ConnectorStatus::Connected;
+            if dead_urls.contains(&conn_url) {
+                status = ConnectorStatus::Disconnected;
+            }
+            ret.insert(
+                0,
+                Connector {
+                    url: conn_url,
+                    status: status.into(),
+                },
+            );
+        }
+
+        let reconnecting_urls: BTreeSet<String> = self
+            .data
+            .reconnecting
+            .iter()
+            .map(|x| x.clone().into())
+            .collect();
+
+        for conn_url in reconnecting_urls {
+            ret.insert(
+                0,
+                Connector {
+                    url: conn_url,
+                    status: ConnectorStatus::Connecting.into(),
+                },
+            );
+        }
+
+        ret
+    }
+
+    async fn conn_mgr_routine(
+        data: Arc<ConnectorManagerData>,
+        mut event_recv: Receiver<GlobalCtxEvent>,
+    ) {
+        log::warn!("conn_mgr_routine started");
+        let mut reconn_interval = tokio::time::interval(std::time::Duration::from_millis(
+            use_global_var!(MANUAL_CONNECTOR_RECONNECT_INTERVAL_MS),
+        ));
+        let mut reconn_tasks = JoinSet::new();
+        let (reconn_result_send, mut reconn_result_recv) = mpsc::channel(100);
+
+        loop {
+            tokio::select! {
+                event = event_recv.recv() => {
+                    if let Ok(event) = event {
+                        Self::handle_event(&event, data.clone()).await;
+                    } else {
+                        log::warn!("event_recv closed");
+                        panic!("event_recv closed");
+                    }
+                }
+
+                _ = reconn_interval.tick() => {
+                    let dead_urls = Self::collect_dead_conns(data.clone()).await;
+                    if dead_urls.is_empty() {
+                        continue;
+                    }
+                    for dead_url in dead_urls {
+                        let data_clone = data.clone();
+                        let sender = reconn_result_send.clone();
+                        let (_, connector) = data.connectors.remove(&dead_url).unwrap();
+                        let insert_succ = data.reconnecting.insert(dead_url.clone());
+                        assert!(insert_succ);
+                        reconn_tasks.spawn(async move {
+                            sender.send(Self::conn_reconnect(data_clone.clone(), dead_url, connector).await).await.unwrap();
+                        });
+                    }
+                    log::info!("reconn_interval tick, done");
+                }
+
+                ret = reconn_result_recv.recv() => {
+                    log::warn!("reconn_tasks done, out: {:?}", ret);
+                    let _ = reconn_tasks.join_next().await.unwrap();
+                }
+            }
+        }
+    }
+
+    async fn handle_event(event: &GlobalCtxEvent, data: Arc<ConnectorManagerData>) {
+        match event {
+            GlobalCtxEvent::PeerConnAdded(conn_info) => {
+                let addr = conn_info.tunnel.as_ref().unwrap().remote_addr.clone();
+                data.alive_conn_urls.lock().await.insert(addr);
+                log::warn!("peer conn added: {:?}", conn_info);
+            }
+
+            GlobalCtxEvent::PeerConnRemoved(conn_info) => {
+                let addr = conn_info.tunnel.as_ref().unwrap().remote_addr.clone();
+                data.alive_conn_urls.lock().await.remove(&addr);
+                log::warn!("peer conn removed: {:?}", conn_info);
+            }
+
+            GlobalCtxEvent::PeerAdded => todo!(),
+            GlobalCtxEvent::PeerRemoved => todo!(),
+        }
+    }
+
+    fn handle_remove_connector(data: Arc<ConnectorManagerData>) {
+        let remove_later = DashSet::new();
+        for it in data.removed_conn_urls.iter() {
+            let url = it.key();
+            if let Some(_) = data.connectors.remove(url) {
+                log::warn!("connector: {}, removed", url);
+                continue;
+            } else if data.reconnecting.contains(url) {
+                log::warn!("connector: {}, reconnecting, remove later.", url);
+                remove_later.insert(url.clone());
+                continue;
+            } else {
+                log::warn!("connector: {}, not found", url);
+            }
+        }
+        data.removed_conn_urls.clear();
+        for it in remove_later.iter() {
+            data.removed_conn_urls.insert(it.key().clone());
+        }
+    }
+
+    async fn collect_dead_conns(data: Arc<ConnectorManagerData>) -> BTreeSet<String> {
+        Self::handle_remove_connector(data.clone());
+
+        let curr_alive = data.alive_conn_urls.lock().await.clone();
+        let all_urls: BTreeSet<String> = data
+            .connectors
+            .iter()
+            .map(|x| x.key().clone().into())
+            .collect();
+        &all_urls - &curr_alive
+    }
+
+    async fn conn_reconnect(
+        data: Arc<ConnectorManagerData>,
+        dead_url: String,
+        connector: Box<dyn TunnelConnector + Send + Sync>,
+    ) -> Result<ReconnResult, Error> {
+        let connector = Arc::new(Mutex::new(Some(connector)));
+        let net_ns = data.net_ns.clone();
+
+        log::info!("reconnect: {}", dead_url);
+
+        let connector_clone = connector.clone();
+        let data_clone = data.clone();
+        let url_clone = dead_url.clone();
+        let ip_collector = data.global_ctx.get_ip_collector();
+        let reconn_task = async move {
+            let mut locked = connector_clone.lock().await;
+            let conn = locked.as_mut().unwrap();
+            // TODO: should support set v6 here, use url in connector array
+            set_bind_addr_for_peer_connector(conn, true, &ip_collector).await;
+            let _g = net_ns.guard();
+            log::info!("reconnect try connect... conn: {:?}", conn);
+            let tunnel = conn.connect().await?;
+            log::info!("reconnect get tunnel succ: {:?}", tunnel);
+            assert_eq!(
+                url_clone,
+                tunnel.info().unwrap().remote_addr,
+                "info: {:?}",
+                tunnel.info()
+            );
+            let (peer_id, conn_id) = data_clone.peer_manager.add_client_tunnel(tunnel).await?;
+            log::info!("reconnect succ: {} {} {}", peer_id, conn_id, url_clone);
+            Ok(ReconnResult {
+                dead_url: url_clone,
+                peer_id,
+                conn_id,
+            })
+        };
+
+        let ret = timeout(std::time::Duration::from_secs(1), reconn_task).await;
+        log::info!("reconnect: {} done, ret: {:?}", dead_url, ret);
+
+        let conn = connector.lock().await.take().unwrap();
+        data.reconnecting.remove(&dead_url).unwrap();
+        data.connectors.insert(dead_url.clone(), conn);
+
+        ret?
+    }
+}
+
+pub struct ConnectorManagerRpcService(pub Arc<ManualConnectorManager>);
+
+#[tonic::async_trait]
+impl ConnectorManageRpc for ConnectorManagerRpcService {
+    async fn list_connector(
+        &self,
+        _request: tonic::Request<ListConnectorRequest>,
+    ) -> Result<tonic::Response<easytier_rpc::ListConnectorResponse>, tonic::Status> {
+        let mut ret = easytier_rpc::ListConnectorResponse::default();
+        let connectors = self.0.list_connectors().await;
+        ret.connectors = connectors;
+        Ok(tonic::Response::new(ret))
+    }
+
+    async fn manage_connector(
+        &self,
+        request: tonic::Request<ManageConnectorRequest>,
+    ) -> Result<tonic::Response<easytier_rpc::ManageConnectorResponse>, tonic::Status> {
+        let req = request.into_inner();
+        let url = url::Url::parse(&req.url)
+            .map_err(|_| tonic::Status::invalid_argument("invalid url"))?;
+        if req.action == easytier_rpc::ConnectorManageAction::Remove as i32 {
+            self.0.remove_connector(url.path()).await.map_err(|e| {
+                tonic::Status::invalid_argument(format!("remove connector failed: {:?}", e))
+            })?;
+            return Ok(tonic::Response::new(
+                easytier_rpc::ManageConnectorResponse::default(),
+            ));
+        } else {
+            self.0
+                .add_connector_by_url(url.as_str())
+                .await
+                .map_err(|e| {
+                    tonic::Status::invalid_argument(format!("add connector failed: {:?}", e))
+                })?;
+        }
+        Ok(tonic::Response::new(
+            easytier_rpc::ManageConnectorResponse::default(),
+        ))
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use crate::{
+        peers::tests::create_mock_peer_manager,
+        set_global_var,
+        tunnels::{Tunnel, TunnelError},
+    };
+
+    use super::*;
+
+    #[tokio::test]
+    async fn test_reconnect_with_connecting_addr() {
+        set_global_var!(MANUAL_CONNECTOR_RECONNECT_INTERVAL_MS, 1);
+
+        let peer_mgr = create_mock_peer_manager().await;
+        let my_node_id = uuid::Uuid::new_v4();
+        let mgr = ManualConnectorManager::new(my_node_id, peer_mgr.get_global_ctx(), peer_mgr);
+
+        struct MockConnector {}
+        #[async_trait::async_trait]
+        impl TunnelConnector for MockConnector {
+            fn remote_url(&self) -> url::Url {
+                url::Url::parse("tcp://aa.com").unwrap()
+            }
+            async fn connect(&mut self) -> Result<Box<dyn Tunnel>, TunnelError> {
+                tokio::time::sleep(std::time::Duration::from_millis(10)).await;
+                Err(TunnelError::CommonError("fake error".into()))
+            }
+        }
+
+        mgr.add_connector(MockConnector {});
+
+        tokio::time::sleep(std::time::Duration::from_secs(5)).await;
+    }
+}

easytier-core/src/connector/mod.rs

@@ -0,0 +1,73 @@
+use std::{
+    net::{SocketAddr, SocketAddrV4, SocketAddrV6},
+    sync::Arc,
+};
+
+use crate::{
+    common::{error::Error, network::IPCollector},
+    tunnels::{
+        ring_tunnel::RingTunnelConnector, tcp_tunnel::TcpTunnelConnector,
+        udp_tunnel::UdpTunnelConnector, TunnelConnector,
+    },
+};
+
+pub mod direct;
+pub mod manual;
+pub mod udp_hole_punch;
+
+async fn set_bind_addr_for_peer_connector(
+    connector: &mut impl TunnelConnector,
+    is_ipv4: bool,
+    ip_collector: &Arc<IPCollector>,
+) {
+    let ips = ip_collector.collect_ip_addrs().await;
+    if is_ipv4 {
+        let mut bind_addrs = vec![];
+        for ipv4 in ips.interface_ipv4s {
+            let socket_addr = SocketAddrV4::new(ipv4.parse().unwrap(), 0).into();
+            bind_addrs.push(socket_addr);
+        }
+        connector.set_bind_addrs(bind_addrs);
+    } else {
+        let mut bind_addrs = vec![];
+        for ipv6 in ips.interface_ipv6s {
+            let socket_addr = SocketAddrV6::new(ipv6.parse().unwrap(), 0, 0, 0).into();
+            bind_addrs.push(socket_addr);
+        }
+        connector.set_bind_addrs(bind_addrs);
+    }
+    let _ = connector;
+}
+
+pub async fn create_connector_by_url(
+    url: &str,
+    ip_collector: Arc<IPCollector>,
+) -> Result<Box<dyn TunnelConnector + Send + Sync + 'static>, Error> {
+    let url = url::Url::parse(url).map_err(|_| Error::InvalidUrl(url.to_owned()))?;
+    match url.scheme() {
+        "tcp" => {
+            let dst_addr =
+                crate::tunnels::check_scheme_and_get_socket_addr::<SocketAddr>(&url, "tcp")?;
+            let mut connector = TcpTunnelConnector::new(url);
+            set_bind_addr_for_peer_connector(&mut connector, dst_addr.is_ipv4(), &ip_collector)
+                .await;
+            return Ok(Box::new(connector));
+        }
+        "udp" => {
+            let dst_addr =
+                crate::tunnels::check_scheme_and_get_socket_addr::<SocketAddr>(&url, "udp")?;
+            let mut connector = UdpTunnelConnector::new(url);
+            set_bind_addr_for_peer_connector(&mut connector, dst_addr.is_ipv4(), &ip_collector)
+                .await;
+            return Ok(Box::new(connector));
+        }
+        "ring" => {
+            crate::tunnels::check_scheme_and_get_socket_addr::<uuid::Uuid>(&url, "ring")?;
+            let connector = RingTunnelConnector::new(url);
+            return Ok(Box::new(connector));
+        }
+        _ => {
+            return Err(Error::InvalidUrl(url.into()));
+        }
+    }
+}

easytier-core/src/connector/udp_hole_punch.rs

@@ -0,0 +1,523 @@
+use std::{net::SocketAddr, sync::Arc};
+
+use anyhow::Context;
+use crossbeam::atomic::AtomicCell;
+use easytier_rpc::NatType;
+use rand::{seq::SliceRandom, Rng, SeedableRng};
+use tokio::{net::UdpSocket, sync::Mutex, task::JoinSet};
+use tracing::Instrument;
+
+use crate::{
+    common::{
+        constants, error::Error, global_ctx::ArcGlobalCtx, rkyv_util::encode_to_bytes,
+        stun::StunInfoCollectorTrait,
+    },
+    peers::{peer_manager::PeerManager, PeerId},
+    tunnels::{
+        udp_tunnel::{UdpPacket, UdpTunnelConnector, UdpTunnelListener},
+        Tunnel, TunnelConnCounter, TunnelListener,
+    },
+};
+
+use super::direct::PeerManagerForDirectConnector;
+
+#[tarpc::service]
+pub trait UdpHolePunchService {
+    async fn try_punch_hole(local_mapped_addr: SocketAddr) -> Option<SocketAddr>;
+}
+
+#[derive(Debug)]
+struct UdpHolePunchListener {
+    socket: Arc<UdpSocket>,
+    tasks: JoinSet<()>,
+    running: Arc<AtomicCell<bool>>,
+    mapped_addr: SocketAddr,
+    conn_counter: Arc<Box<dyn TunnelConnCounter>>,
+
+    listen_time: std::time::Instant,
+    last_select_time: AtomicCell<std::time::Instant>,
+    last_connected_time: Arc<AtomicCell<std::time::Instant>>,
+}
+
+impl UdpHolePunchListener {
+    async fn get_avail_port() -> Result<u16, Error> {
+        let socket = UdpSocket::bind("0.0.0.0:0").await?;
+        Ok(socket.local_addr()?.port())
+    }
+
+    pub async fn new(peer_mgr: Arc<PeerManager>) -> Result<Self, Error> {
+        let port = Self::get_avail_port().await?;
+        let listen_url = format!("udp://0.0.0.0:{}", port);
+
+        let gctx = peer_mgr.get_global_ctx();
+        let stun_info_collect = gctx.get_stun_info_collector();
+        let mapped_addr = stun_info_collect.get_udp_port_mapping(port).await?;
+
+        let mut listener = UdpTunnelListener::new(listen_url.parse().unwrap());
+
+        listener.listen().await?;
+        let socket = listener.get_socket().unwrap();
+
+        let running = Arc::new(AtomicCell::new(true));
+        let running_clone = running.clone();
+
+        let last_connected_time = Arc::new(AtomicCell::new(std::time::Instant::now()));
+        let last_connected_time_clone = last_connected_time.clone();
+
+        let conn_counter = listener.get_conn_counter();
+        let mut tasks = JoinSet::new();
+
+        tasks.spawn(async move {
+            while let Ok(conn) = listener.accept().await {
+                last_connected_time_clone.store(std::time::Instant::now());
+                tracing::warn!(?conn, "udp hole punching listener got peer connection");
+                if let Err(e) = peer_mgr.add_tunnel_as_server(conn).await {
+                    tracing::error!(?e, "failed to add tunnel as server in hole punch listener");
+                }
+            }
+
+            running_clone.store(false);
+        });
+
+        tracing::warn!(?mapped_addr, ?socket, "udp hole punching listener started");
+
+        Ok(Self {
+            tasks,
+            socket,
+            running,
+            mapped_addr,
+            conn_counter,
+
+            listen_time: std::time::Instant::now(),
+            last_select_time: AtomicCell::new(std::time::Instant::now()),
+            last_connected_time,
+        })
+    }
+
+    pub async fn get_socket(&self) -> Arc<UdpSocket> {
+        self.last_select_time.store(std::time::Instant::now());
+        self.socket.clone()
+    }
+}
+
+#[derive(Debug)]
+struct UdpHolePunchConnectorData {
+    global_ctx: ArcGlobalCtx,
+    peer_mgr: Arc<PeerManager>,
+    listeners: Arc<Mutex<Vec<UdpHolePunchListener>>>,
+}
+
+#[derive(Clone)]
+struct UdpHolePunchRpcServer {
+    data: Arc<UdpHolePunchConnectorData>,
+
+    tasks: Arc<Mutex<JoinSet<()>>>,
+}
+
+#[tarpc::server]
+impl UdpHolePunchService for UdpHolePunchRpcServer {
+    async fn try_punch_hole(
+        self,
+        _: tarpc::context::Context,
+        local_mapped_addr: SocketAddr,
+    ) -> Option<SocketAddr> {
+        let (socket, mapped_addr) = self.select_listener().await?;
+        tracing::warn!(?local_mapped_addr, ?mapped_addr, "start hole punching");
+
+        let my_udp_nat_type = self
+            .data
+            .global_ctx
+            .get_stun_info_collector()
+            .get_stun_info()
+            .udp_nat_type;
+
+        // if we are restricted, we need to send hole punching resp to client
+        if my_udp_nat_type == NatType::PortRestricted as i32
+            || my_udp_nat_type == NatType::Restricted as i32
+        {
+            // send punch msg to local_mapped_addr for 3 seconds, 3.3 packet per second
+            self.tasks.lock().await.spawn(async move {
+                for _ in 0..10 {
+                    tracing::info!(?local_mapped_addr, "sending hole punching packet");
+                    // generate a 128 bytes vec with random data
+                    let mut rng = rand::rngs::StdRng::from_entropy();
+                    let mut buf = vec![0u8; 128];
+                    rng.fill(&mut buf[..]);
+
+                    let udp_packet = UdpPacket::new_hole_punch_packet(buf);
+                    let udp_packet_bytes = encode_to_bytes::<_, 256>(&udp_packet);
+                    let _ = socket
+                        .send_to(udp_packet_bytes.as_ref(), local_mapped_addr)
+                        .await;
+                    tokio::time::sleep(std::time::Duration::from_millis(300)).await;
+                }
+            });
+        }
+
+        Some(mapped_addr)
+    }
+}
+
+impl UdpHolePunchRpcServer {
+    pub fn new(data: Arc<UdpHolePunchConnectorData>) -> Self {
+        Self {
+            data,
+            tasks: Arc::new(Mutex::new(JoinSet::new())),
+        }
+    }
+
+    async fn select_listener(&self) -> Option<(Arc<UdpSocket>, SocketAddr)> {
+        let all_listener_sockets = &self.data.listeners;
+
+        // remove listener that not have connection in for 20 seconds
+        all_listener_sockets.lock().await.retain(|listener| {
+            listener.last_connected_time.load().elapsed().as_secs() < 20
+                && listener.conn_counter.get() > 0
+        });
+
+        let mut use_last = false;
+        if all_listener_sockets.lock().await.len() < 4 {
+            tracing::warn!("creating new udp hole punching listener");
+            all_listener_sockets.lock().await.push(
+                UdpHolePunchListener::new(self.data.peer_mgr.clone())
+                    .await
+                    .ok()?,
+            );
+            use_last = true;
+        }
+
+        let locked = all_listener_sockets.lock().await;
+
+        let listener = if use_last {
+            locked.last()?
+        } else {
+            locked.choose(&mut rand::rngs::StdRng::from_entropy())?
+        };
+
+        Some((listener.get_socket().await, listener.mapped_addr))
+    }
+}
+
+pub struct UdpHolePunchConnector {
+    data: Arc<UdpHolePunchConnectorData>,
+    tasks: JoinSet<()>,
+}
+
+// Currently support:
+// Symmetric -> Full Cone
+// Any Type of Full Cone -> Any Type of Full Cone
+
+// if same level of full cone, node with smaller peer_id will be the initiator
+// if different level of full cone, node with more strict level will be the initiator
+
+impl UdpHolePunchConnector {
+    pub fn new(global_ctx: ArcGlobalCtx, peer_mgr: Arc<PeerManager>) -> Self {
+        Self {
+            data: Arc::new(UdpHolePunchConnectorData {
+                global_ctx,
+                peer_mgr,
+                listeners: Arc::new(Mutex::new(Vec::new())),
+            }),
+            tasks: JoinSet::new(),
+        }
+    }
+
+    pub async fn run_as_client(&mut self) -> Result<(), Error> {
+        let data = self.data.clone();
+        self.tasks.spawn(async move {
+            Self::main_loop(data).await;
+        });
+
+        Ok(())
+    }
+
+    pub async fn run_as_server(&mut self) -> Result<(), Error> {
+        self.data.peer_mgr.get_peer_rpc_mgr().run_service(
+            constants::UDP_HOLE_PUNCH_CONNECTOR_SERVICE_ID,
+            UdpHolePunchRpcServer::new(self.data.clone()).serve(),
+        );
+
+        Ok(())
+    }
+
+    pub async fn run(&mut self) -> Result<(), Error> {
+        self.run_as_client().await?;
+        self.run_as_server().await?;
+
+        Ok(())
+    }
+
+    async fn collect_peer_to_connect(data: Arc<UdpHolePunchConnectorData>) -> Vec<PeerId> {
+        let mut peers_to_connect = Vec::new();
+
+        // do not do anything if:
+        // 1. our stun test has not finished
+        // 2. our nat type is OpenInternet or NoPat, which means we can wait other peers to connect us
+        let my_nat_type = data
+            .global_ctx
+            .get_stun_info_collector()
+            .get_stun_info()
+            .udp_nat_type;
+
+        let my_nat_type = NatType::try_from(my_nat_type).unwrap();
+
+        if my_nat_type == NatType::Unknown
+            || my_nat_type == NatType::OpenInternet
+            || my_nat_type == NatType::NoPat
+        {
+            return peers_to_connect;
+        }
+
+        // collect peer list from peer manager and do some filter:
+        // 1. peers without direct conns;
+        // 2. peers is full cone (any restricted type);
+        for route in data.peer_mgr.list_routes().await.iter() {
+            let Some(peer_stun_info) = route.stun_info.as_ref() else {
+                continue;
+            };
+            let Ok(peer_nat_type) = NatType::try_from(peer_stun_info.udp_nat_type) else {
+                continue;
+            };
+
+            let peer_id: PeerId = route.peer_id.parse().unwrap();
+            let conns = data.peer_mgr.list_peer_conns(&peer_id).await;
+            if conns.is_some() && conns.unwrap().len() > 0 {
+                continue;
+            }
+
+            // if peer is symmetric ignore it because we cannot connect to it
+            // if peer is open internet or no pat, direct connector will connecto to it
+            if peer_nat_type == NatType::Unknown
+                || peer_nat_type == NatType::OpenInternet
+                || peer_nat_type == NatType::NoPat
+                || peer_nat_type == NatType::Symmetric
+                || peer_nat_type == NatType::SymUdpFirewall
+            {
+                continue;
+            }
+
+            // if we are symmetric, we can only connect to full cone
+            // TODO: can also connect to restricted full cone, with some extra work
+            if (my_nat_type == NatType::Symmetric || my_nat_type == NatType::SymUdpFirewall)
+                && peer_nat_type != NatType::FullCone
+            {
+                continue;
+            }
+
+            // if we have smae level of full cone, node with smaller peer_id will be the initiator
+            if my_nat_type == peer_nat_type {
+                if data.global_ctx.id > peer_id {
+                    continue;
+                }
+            } else {
+                // if we have different level of full cone
+                // we will be the initiator if we have more strict level
+                if my_nat_type < peer_nat_type {
+                    continue;
+                }
+            }
+
+            tracing::info!(
+                ?peer_id,
+                ?peer_nat_type,
+                ?my_nat_type,
+                ?data.global_ctx.id,
+                "found peer to do hole punching"
+            );
+
+            peers_to_connect.push(peer_id);
+        }
+
+        peers_to_connect
+    }
+
+    #[tracing::instrument]
+    async fn do_hole_punching(
+        data: Arc<UdpHolePunchConnectorData>,
+        dst_peer_id: PeerId,
+    ) -> Result<Box<dyn Tunnel>, anyhow::Error> {
+        tracing::info!(?dst_peer_id, "start hole punching");
+        // client: choose a local udp port, and get the pubic mapped port from stun server
+        let socket = UdpSocket::bind("0.0.0.0:0").await.with_context(|| "")?;
+        let local_socket_addr = socket.local_addr()?;
+        let local_port = socket.local_addr()?.port();
+        drop(socket); // drop the socket to release the port
+
+        let local_mapped_addr = data
+            .global_ctx
+            .get_stun_info_collector()
+            .get_udp_port_mapping(local_port)
+            .await
+            .with_context(|| "failed to get udp port mapping")?;
+
+        // client -> server: tell server the mapped port, server will return the mapped address of listening port.
+        let Some(remote_mapped_addr) = data
+            .peer_mgr
+            .get_peer_rpc_mgr()
+            .do_client_rpc_scoped(
+                constants::UDP_HOLE_PUNCH_CONNECTOR_SERVICE_ID,
+                dst_peer_id,
+                |c| async {
+                    let client =
+                        UdpHolePunchServiceClient::new(tarpc::client::Config::default(), c).spawn();
+                    let remote_mapped_addr = client
+                        .try_punch_hole(tarpc::context::current(), local_mapped_addr)
+                        .await;
+                    tracing::info!(?remote_mapped_addr, ?dst_peer_id, "got remote mapped addr");
+                    remote_mapped_addr
+                },
+            )
+            .await?
+        else {
+            return Err(anyhow::anyhow!("failed to get remote mapped addr"));
+        };
+
+        // server: will send some punching resps, total 10 packets.
+        // client: use the socket to create UdpTunnel with UdpTunnelConnector
+        // NOTICE: UdpTunnelConnector will ignore the punching resp packet sent by remote.
+
+        let connector = UdpTunnelConnector::new(
+            format!(
+                "udp://{}:{}",
+                remote_mapped_addr.ip(),
+                remote_mapped_addr.port()
+            )
+            .to_string()
+            .parse()
+            .unwrap(),
+        );
+
+        let socket = UdpSocket::bind(local_socket_addr)
+            .await
+            .with_context(|| "")?;
+        Ok(connector
+            .try_connect_with_socket(socket)
+            .await
+            .with_context(|| "UdpTunnelConnector failed to connect remote")?)
+    }
+
+    async fn main_loop(data: Arc<UdpHolePunchConnectorData>) {
+        loop {
+            let peers_to_connect = Self::collect_peer_to_connect(data.clone()).await;
+            tracing::trace!(?peers_to_connect, "peers to connect");
+            if peers_to_connect.len() == 0 {
+                tokio::time::sleep(std::time::Duration::from_secs(5)).await;
+                continue;
+            }
+
+            let mut tasks: JoinSet<Result<(), anyhow::Error>> = JoinSet::new();
+            for peer_id in peers_to_connect {
+                let data = data.clone();
+                tasks.spawn(
+                    async move {
+                        let tunnel = Self::do_hole_punching(data.clone(), peer_id)
+                            .await
+                            .with_context(|| "failed to do hole punching")?;
+
+                        let _ =
+                            data.peer_mgr
+                                .add_client_tunnel(tunnel)
+                                .await
+                                .with_context(|| {
+                                    "failed to add tunnel as client in hole punch connector"
+                                })?;
+
+                        Ok(())
+                    }
+                    .instrument(tracing::info_span!("doing hole punching client", ?peer_id)),
+                );
+            }
+
+            while let Some(res) = tasks.join_next().await {
+                if let Err(e) = res {
+                    tracing::error!(?e, "failed to join hole punching job");
+                    continue;
+                }
+
+                match res.unwrap() {
+                    Err(e) => {
+                        tracing::error!(?e, "failed to do hole punching job");
+                    }
+                    Ok(_) => {
+                        tracing::info!("hole punching job succeed");
+                    }
+                }
+            }
+
+            tokio::time::sleep(std::time::Duration::from_secs(10)).await;
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::sync::Arc;
+
+    use easytier_rpc::{NatType, StunInfo};
+
+    use crate::{
+        common::{error::Error, stun::StunInfoCollectorTrait},
+        connector::udp_hole_punch::UdpHolePunchConnector,
+        peers::{
+            peer_manager::PeerManager,
+            tests::{
+                connect_peer_manager, create_mock_peer_manager, wait_route_appear,
+                wait_route_appear_with_cost,
+            },
+        },
+        tests::enable_log,
+    };
+
+    struct MockStunInfoCollector {
+        udp_nat_type: NatType,
+    }
+
+    #[async_trait::async_trait]
+    impl StunInfoCollectorTrait for MockStunInfoCollector {
+        fn get_stun_info(&self) -> StunInfo {
+            StunInfo {
+                udp_nat_type: self.udp_nat_type as i32,
+                tcp_nat_type: NatType::Unknown as i32,
+                last_update_time: std::time::Instant::now().elapsed().as_secs() as i64,
+            }
+        }
+
+        async fn get_udp_port_mapping(&self, port: u16) -> Result<std::net::SocketAddr, Error> {
+            Ok(format!("127.0.0.1:{}", port).parse().unwrap())
+        }
+    }
+
+    async fn create_mock_peer_manager_with_mock_stun(udp_nat_type: NatType) -> Arc<PeerManager> {
+        let p_a = create_mock_peer_manager().await;
+        let collector = Box::new(MockStunInfoCollector { udp_nat_type });
+        p_a.get_global_ctx().replace_stun_info_collector(collector);
+        p_a
+    }
+
+    #[tokio::test]
+    async fn hole_punching() {
+        enable_log();
+        let p_a = create_mock_peer_manager_with_mock_stun(NatType::PortRestricted).await;
+        let p_b = create_mock_peer_manager_with_mock_stun(NatType::Symmetric).await;
+        let p_c = create_mock_peer_manager_with_mock_stun(NatType::PortRestricted).await;
+        connect_peer_manager(p_a.clone(), p_b.clone()).await;
+        connect_peer_manager(p_b.clone(), p_c.clone()).await;
+
+        wait_route_appear(p_a.clone(), p_c.my_node_id())
+            .await
+            .unwrap();
+
+        println!("{:?}", p_a.list_routes().await);
+
+        let mut hole_punching_a = UdpHolePunchConnector::new(p_a.get_global_ctx(), p_a.clone());
+        let mut hole_punching_c = UdpHolePunchConnector::new(p_c.get_global_ctx(), p_c.clone());
+
+        hole_punching_a.run().await.unwrap();
+        hole_punching_c.run().await.unwrap();
+
+        wait_route_appear_with_cost(p_a.clone(), p_c.my_node_id(), Some(1))
+            .await
+            .unwrap();
+        println!("{:?}", p_a.list_routes().await);
+    }
+}

easytier-core/src/gateway/icmp_proxy.rs

@@ -0,0 +1,301 @@
+use std::{
+    mem::MaybeUninit,
+    net::{IpAddr, Ipv4Addr, SocketAddrV4},
+    sync::Arc,
+    thread,
+};
+
+use pnet::packet::{
+    icmp::{self, IcmpTypes},
+    ip::IpNextHeaderProtocols,
+    ipv4::{self, Ipv4Packet, MutableIpv4Packet},
+    Packet,
+};
+use socket2::Socket;
+use tokio::{
+    sync::{mpsc::UnboundedSender, Mutex},
+    task::JoinSet,
+};
+use tokio_util::bytes::Bytes;
+use tracing::Instrument;
+
+use crate::{
+    common::{error::Error, global_ctx::ArcGlobalCtx},
+    peers::{
+        packet,
+        peer_manager::{PeerManager, PeerPacketFilter},
+        PeerId,
+    },
+};
+
+use super::CidrSet;
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+struct IcmpNatKey {
+    dst_ip: std::net::IpAddr,
+    icmp_id: u16,
+    icmp_seq: u16,
+}
+
+#[derive(Debug)]
+struct IcmpNatEntry {
+    src_peer_id: PeerId,
+    my_peer_id: PeerId,
+    src_ip: IpAddr,
+    start_time: std::time::Instant,
+}
+
+impl IcmpNatEntry {
+    fn new(src_peer_id: PeerId, my_peer_id: PeerId, src_ip: IpAddr) -> Result<Self, Error> {
+        Ok(Self {
+            src_peer_id,
+            my_peer_id,
+            src_ip,
+            start_time: std::time::Instant::now(),
+        })
+    }
+}
+
+type IcmpNatTable = Arc<dashmap::DashMap<IcmpNatKey, IcmpNatEntry>>;
+type NewPacketSender = tokio::sync::mpsc::UnboundedSender<IcmpNatKey>;
+type NewPacketReceiver = tokio::sync::mpsc::UnboundedReceiver<IcmpNatKey>;
+
+#[derive(Debug)]
+pub struct IcmpProxy {
+    global_ctx: ArcGlobalCtx,
+    peer_manager: Arc<PeerManager>,
+
+    cidr_set: CidrSet,
+    socket: socket2::Socket,
+
+    nat_table: IcmpNatTable,
+
+    tasks: Mutex<JoinSet<()>>,
+}
+
+fn socket_recv(socket: &Socket, buf: &mut [MaybeUninit<u8>]) -> Result<(usize, IpAddr), Error> {
+    let (size, addr) = socket.recv_from(buf)?;
+    let addr = match addr.as_socket() {
+        None => IpAddr::V4(Ipv4Addr::UNSPECIFIED),
+        Some(add) => add.ip(),
+    };
+    Ok((size, addr))
+}
+
+fn socket_recv_loop(
+    socket: Socket,
+    nat_table: IcmpNatTable,
+    sender: UnboundedSender<packet::Packet>,
+) {
+    let mut buf = [0u8; 4096];
+    let data: &mut [MaybeUninit<u8>] = unsafe { std::mem::transmute(&mut buf[12..]) };
+
+    loop {
+        let Ok((len, peer_ip)) = socket_recv(&socket, data) else {
+            continue;
+        };
+
+        if !peer_ip.is_ipv4() {
+            continue;
+        }
+
+        let Some(mut ipv4_packet) = MutableIpv4Packet::new(&mut buf[12..12 + len]) else {
+            continue;
+        };
+
+        let Some(icmp_packet) = icmp::echo_reply::EchoReplyPacket::new(ipv4_packet.payload())
+        else {
+            continue;
+        };
+
+        if icmp_packet.get_icmp_type() != IcmpTypes::EchoReply {
+            continue;
+        }
+
+        let key = IcmpNatKey {
+            dst_ip: peer_ip,
+            icmp_id: icmp_packet.get_identifier(),
+            icmp_seq: icmp_packet.get_sequence_number(),
+        };
+
+        let Some((_, v)) = nat_table.remove(&key) else {
+            continue;
+        };
+
+        // send packet back to the peer where this request origin.
+        let IpAddr::V4(dest_ip) = v.src_ip else {
+            continue;
+        };
+
+        ipv4_packet.set_destination(dest_ip);
+        ipv4_packet.set_checksum(ipv4::checksum(&ipv4_packet.to_immutable()));
+
+        let peer_packet = packet::Packet::new_data_packet(
+            v.my_peer_id,
+            v.src_peer_id,
+            &ipv4_packet.to_immutable().packet(),
+        );
+
+        if let Err(e) = sender.send(peer_packet) {
+            tracing::error!("send icmp packet to peer failed: {:?}, may exiting..", e);
+            break;
+        }
+    }
+}
+
+#[async_trait::async_trait]
+impl PeerPacketFilter for IcmpProxy {
+    async fn try_process_packet_from_peer(
+        &self,
+        packet: &packet::ArchivedPacket,
+        _: &Bytes,
+    ) -> Option<()> {
+        let _ = self.global_ctx.get_ipv4()?;
+
+        let packet::ArchivedPacketBody::Data(x) = &packet.body else {
+            return None;
+        };
+
+        let ipv4 = Ipv4Packet::new(&x.data)?;
+
+        if ipv4.get_version() != 4 || ipv4.get_next_level_protocol() != IpNextHeaderProtocols::Icmp
+        {
+            return None;
+        }
+
+        if !self.cidr_set.contains_v4(ipv4.get_destination()) {
+            return None;
+        }
+
+        let icmp_packet = icmp::echo_request::EchoRequestPacket::new(&ipv4.payload())?;
+
+        if icmp_packet.get_icmp_type() != IcmpTypes::EchoRequest {
+            // drop it because we do not support other icmp types
+            tracing::trace!("unsupported icmp type: {:?}", icmp_packet.get_icmp_type());
+            return Some(());
+        }
+
+        let icmp_id = icmp_packet.get_identifier();
+        let icmp_seq = icmp_packet.get_sequence_number();
+
+        let key = IcmpNatKey {
+            dst_ip: ipv4.get_destination().into(),
+            icmp_id,
+            icmp_seq,
+        };
+
+        if packet.to_peer.is_none() {
+            return None;
+        }
+
+        let value = IcmpNatEntry::new(
+            packet.from_peer.to_uuid(),
+            packet.to_peer.as_ref().unwrap().to_uuid(),
+            ipv4.get_source().into(),
+        )
+        .ok()?;
+
+        if let Some(old) = self.nat_table.insert(key, value) {
+            tracing::info!("icmp nat table entry replaced: {:?}", old);
+        }
+
+        if let Err(e) = self.send_icmp_packet(ipv4.get_destination(), &icmp_packet) {
+            tracing::error!("send icmp packet failed: {:?}", e);
+        }
+
+        Some(())
+    }
+}
+
+impl IcmpProxy {
+    pub fn new(
+        global_ctx: ArcGlobalCtx,
+        peer_manager: Arc<PeerManager>,
+    ) -> Result<Arc<Self>, Error> {
+        let cidr_set = CidrSet::new(global_ctx.clone());
+
+        let _g = global_ctx.net_ns.guard();
+        let socket = socket2::Socket::new(
+            socket2::Domain::IPV4,
+            socket2::Type::RAW,
+            Some(socket2::Protocol::ICMPV4),
+        )?;
+        socket.bind(&socket2::SockAddr::from(SocketAddrV4::new(
+            std::net::Ipv4Addr::UNSPECIFIED,
+            0,
+        )))?;
+
+        let ret = Self {
+            global_ctx,
+            peer_manager,
+            cidr_set,
+            socket,
+
+            nat_table: Arc::new(dashmap::DashMap::new()),
+            tasks: Mutex::new(JoinSet::new()),
+        };
+
+        Ok(Arc::new(ret))
+    }
+
+    pub async fn start(self: &Arc<Self>) -> Result<(), Error> {
+        self.start_icmp_proxy().await?;
+        self.start_nat_table_cleaner().await?;
+        Ok(())
+    }
+
+    async fn start_nat_table_cleaner(self: &Arc<Self>) -> Result<(), Error> {
+        let nat_table = self.nat_table.clone();
+        self.tasks.lock().await.spawn(
+            async move {
+                loop {
+                    tokio::time::sleep(std::time::Duration::from_secs(1)).await;
+                    nat_table.retain(|_, v| v.start_time.elapsed().as_secs() < 20);
+                }
+            }
+            .instrument(tracing::info_span!("icmp proxy nat table cleaner")),
+        );
+        Ok(())
+    }
+
+    async fn start_icmp_proxy(self: &Arc<Self>) -> Result<(), Error> {
+        let socket = self.socket.try_clone()?;
+        let (sender, mut receiver) = tokio::sync::mpsc::unbounded_channel();
+        let nat_table = self.nat_table.clone();
+        thread::spawn(|| {
+            socket_recv_loop(socket, nat_table, sender);
+        });
+
+        let peer_manager = self.peer_manager.clone();
+        self.tasks.lock().await.spawn(
+            async move {
+                while let Some(msg) = receiver.recv().await {
+                    let to_peer_id: uuid::Uuid = msg.to_peer.as_ref().unwrap().clone().into();
+                    let ret = peer_manager.send_msg(msg.into(), &to_peer_id).await;
+                    if ret.is_err() {
+                        tracing::error!("send icmp packet to peer failed: {:?}", ret);
+                    }
+                }
+            }
+            .instrument(tracing::info_span!("icmp proxy send loop")),
+        );
+
+        self.peer_manager
+            .add_packet_process_pipeline(Box::new(self.clone()))
+            .await;
+        Ok(())
+    }
+
+    fn send_icmp_packet(
+        &self,
+        dst_ip: Ipv4Addr,
+        icmp_packet: &icmp::echo_request::EchoRequestPacket,
+    ) -> Result<(), Error> {
+        self.socket.send_to(
+            icmp_packet.packet(),
+            &SocketAddrV4::new(dst_ip.into(), 0).into(),
+        )?;
+
+        Ok(())
+    }
+}

easytier-core/src/gateway/mod.rs

@@ -0,0 +1,51 @@
+use dashmap::DashSet;
+use std::sync::Arc;
+use tokio::task::JoinSet;
+
+use crate::common::global_ctx::ArcGlobalCtx;
+
+pub mod icmp_proxy;
+pub mod tcp_proxy;
+
+#[derive(Debug)]
+struct CidrSet {
+    global_ctx: ArcGlobalCtx,
+    cidr_set: Arc<DashSet<cidr::IpCidr>>,
+    tasks: JoinSet<()>,
+}
+
+impl CidrSet {
+    pub fn new(global_ctx: ArcGlobalCtx) -> Self {
+        let mut ret = Self {
+            global_ctx,
+            cidr_set: Arc::new(DashSet::new()),
+            tasks: JoinSet::new(),
+        };
+        ret.run_cidr_updater();
+        ret
+    }
+
+    fn run_cidr_updater(&mut self) {
+        let global_ctx = self.global_ctx.clone();
+        let cidr_set = self.cidr_set.clone();
+        self.tasks.spawn(async move {
+            let mut last_cidrs = vec![];
+            loop {
+                let cidrs = global_ctx.get_proxy_cidrs();
+                if cidrs != last_cidrs {
+                    last_cidrs = cidrs.clone();
+                    cidr_set.clear();
+                    for cidr in cidrs.iter() {
+                        cidr_set.insert(cidr.clone());
+                    }
+                }
+                tokio::time::sleep(std::time::Duration::from_secs(1)).await;
+            }
+        });
+    }
+
+    pub fn contains_v4(&self, ip: std::net::Ipv4Addr) -> bool {
+        let ip = ip.into();
+        return self.cidr_set.iter().any(|cidr| cidr.contains(&ip));
+    }
+}

easytier-core/src/gateway/tcp_proxy.rs

@@ -0,0 +1,402 @@
+use crossbeam::atomic::AtomicCell;
+use dashmap::DashMap;
+use pnet::packet::ip::IpNextHeaderProtocols;
+use pnet::packet::ipv4::{Ipv4Packet, MutableIpv4Packet};
+use pnet::packet::tcp::{ipv4_checksum, MutableTcpPacket};
+use std::net::{IpAddr, Ipv4Addr, SocketAddr, SocketAddrV4};
+use std::sync::atomic::AtomicU16;
+use std::sync::Arc;
+use std::time::{Duration, Instant};
+use tokio::io::copy_bidirectional;
+use tokio::net::{TcpListener, TcpSocket, TcpStream};
+use tokio::sync::Mutex;
+use tokio::task::JoinSet;
+use tokio_util::bytes::{Bytes, BytesMut};
+use tracing::Instrument;
+
+use crate::common::error::Result;
+use crate::common::global_ctx::GlobalCtx;
+use crate::common::netns::NetNS;
+use crate::peers::packet::{self, ArchivedPacket};
+use crate::peers::peer_manager::{NicPacketFilter, PeerManager, PeerPacketFilter};
+
+use super::CidrSet;
+
+#[derive(Debug, Clone, Copy, PartialEq)]
+enum NatDstEntryState {
+    // receive syn packet but not start connecting to dst
+    SynReceived,
+    // connecting to dst
+    ConnectingDst,
+    // connected to dst
+    Connected,
+    // connection closed
+    Closed,
+}
+
+#[derive(Debug)]
+pub struct NatDstEntry {
+    id: uuid::Uuid,
+    src: SocketAddr,
+    dst: SocketAddr,
+    start_time: Instant,
+    tasks: Mutex<JoinSet<()>>,
+    state: AtomicCell<NatDstEntryState>,
+}
+
+impl NatDstEntry {
+    pub fn new(src: SocketAddr, dst: SocketAddr) -> Self {
+        Self {
+            id: uuid::Uuid::new_v4(),
+            src,
+            dst,
+            start_time: Instant::now(),
+            tasks: Mutex::new(JoinSet::new()),
+            state: AtomicCell::new(NatDstEntryState::SynReceived),
+        }
+    }
+}
+
+type ArcNatDstEntry = Arc<NatDstEntry>;
+
+type SynSockMap = Arc<DashMap<SocketAddr, ArcNatDstEntry>>;
+type ConnSockMap = Arc<DashMap<uuid::Uuid, ArcNatDstEntry>>;
+// peer src addr to nat entry, when respond tcp packet, should modify the tcp src addr to the nat entry's dst addr
+type AddrConnSockMap = Arc<DashMap<SocketAddr, ArcNatDstEntry>>;
+
+#[derive(Debug)]
+pub struct TcpProxy {
+    global_ctx: Arc<GlobalCtx>,
+    peer_manager: Arc<PeerManager>,
+    local_port: AtomicU16,
+
+    tasks: Arc<Mutex<JoinSet<()>>>,
+
+    syn_map: SynSockMap,
+    conn_map: ConnSockMap,
+    addr_conn_map: AddrConnSockMap,
+
+    cidr_set: CidrSet,
+}
+
+#[async_trait::async_trait]
+impl PeerPacketFilter for TcpProxy {
+    async fn try_process_packet_from_peer(&self, packet: &ArchivedPacket, _: &Bytes) -> Option<()> {
+        let ipv4_addr = self.global_ctx.get_ipv4()?;
+
+        let packet::ArchivedPacketBody::Data(x) = &packet.body else {
+            return None;
+        };
+
+        let ipv4 = Ipv4Packet::new(&x.data)?;
+        if ipv4.get_version() != 4 || ipv4.get_next_level_protocol() != IpNextHeaderProtocols::Tcp {
+            return None;
+        }
+
+        if !self.cidr_set.contains_v4(ipv4.get_destination()) {
+            return None;
+        }
+
+        tracing::trace!(ipv4 = ?ipv4, cidr_set = ?self.cidr_set, "proxy tcp packet received");
+
+        let mut packet_buffer = BytesMut::with_capacity(x.data.len());
+        packet_buffer.extend_from_slice(&x.data.to_vec());
+
+        let (ip_buffer, tcp_buffer) =
+            packet_buffer.split_at_mut(ipv4.get_header_length() as usize * 4);
+
+        let mut ip_packet = MutableIpv4Packet::new(ip_buffer).unwrap();
+        let mut tcp_packet = MutableTcpPacket::new(tcp_buffer).unwrap();
+
+        let is_tcp_syn = tcp_packet.get_flags() & pnet::packet::tcp::TcpFlags::SYN != 0;
+        if is_tcp_syn {
+            let source_ip = ip_packet.get_source();
+            let source_port = tcp_packet.get_source();
+            let src = SocketAddr::V4(SocketAddrV4::new(source_ip, source_port));
+
+            let dest_ip = ip_packet.get_destination();
+            let dest_port = tcp_packet.get_destination();
+            let dst = SocketAddr::V4(SocketAddrV4::new(dest_ip, dest_port));
+
+            let old_val = self
+                .syn_map
+                .insert(src, Arc::new(NatDstEntry::new(src, dst)));
+            tracing::trace!(src = ?src, dst = ?dst, old_entry = ?old_val, "tcp syn received");
+        }
+
+        ip_packet.set_destination(ipv4_addr);
+        tcp_packet.set_destination(self.get_local_port());
+        Self::update_ipv4_packet_checksum(&mut ip_packet, &mut tcp_packet);
+
+        tracing::trace!(ip_packet = ?ip_packet, tcp_packet = ?tcp_packet, "tcp packet forwarded");
+
+        if let Err(e) = self
+            .peer_manager
+            .get_nic_channel()
+            .send(packet_buffer.freeze())
+            .await
+        {
+            tracing::error!("send to nic failed: {:?}", e);
+        }
+
+        Some(())
+    }
+}
+
+#[async_trait::async_trait]
+impl NicPacketFilter for TcpProxy {
+    async fn try_process_packet_from_nic(&self, mut data: BytesMut) -> BytesMut {
+        let Some(my_ipv4) = self.global_ctx.get_ipv4() else {
+            return data;
+        };
+
+        let header_len = {
+            let Some(ipv4) = &Ipv4Packet::new(&data[..]) else {
+                return data;
+            };
+
+            if ipv4.get_version() != 4
+                || ipv4.get_source() != my_ipv4
+                || ipv4.get_next_level_protocol() != IpNextHeaderProtocols::Tcp
+            {
+                return data;
+            }
+
+            ipv4.get_header_length() as usize * 4
+        };
+
+        let (ip_buffer, tcp_buffer) = data.split_at_mut(header_len);
+        let mut ip_packet = MutableIpv4Packet::new(ip_buffer).unwrap();
+        let mut tcp_packet = MutableTcpPacket::new(tcp_buffer).unwrap();
+
+        if tcp_packet.get_source() != self.get_local_port() {
+            return data;
+        }
+
+        let dst_addr = SocketAddr::V4(SocketAddrV4::new(
+            ip_packet.get_destination(),
+            tcp_packet.get_destination(),
+        ));
+
+        tracing::trace!(dst_addr = ?dst_addr, "tcp packet try find entry");
+        let entry = if let Some(entry) = self.addr_conn_map.get(&dst_addr) {
+            entry
+        } else {
+            let Some(syn_entry) = self.syn_map.get(&dst_addr) else {
+                return data;
+            };
+            syn_entry
+        };
+        let nat_entry = entry.clone();
+        drop(entry);
+        assert_eq!(nat_entry.src, dst_addr);
+
+        let IpAddr::V4(ip) = nat_entry.dst.ip() else {
+            panic!("v4 nat entry src ip is not v4");
+        };
+
+        ip_packet.set_source(ip);
+        tcp_packet.set_source(nat_entry.dst.port());
+        Self::update_ipv4_packet_checksum(&mut ip_packet, &mut tcp_packet);
+
+        tracing::trace!(dst_addr = ?dst_addr, nat_entry = ?nat_entry, packet = ?ip_packet, "tcp packet after modified");
+
+        data
+    }
+}
+
+impl TcpProxy {
+    pub fn new(global_ctx: Arc<GlobalCtx>, peer_manager: Arc<PeerManager>) -> Arc<Self> {
+        Arc::new(Self {
+            global_ctx: global_ctx.clone(),
+            peer_manager,
+
+            local_port: AtomicU16::new(0),
+            tasks: Arc::new(Mutex::new(JoinSet::new())),
+
+            syn_map: Arc::new(DashMap::new()),
+            conn_map: Arc::new(DashMap::new()),
+            addr_conn_map: Arc::new(DashMap::new()),
+
+            cidr_set: CidrSet::new(global_ctx),
+        })
+    }
+
+    fn update_ipv4_packet_checksum(
+        ipv4_packet: &mut MutableIpv4Packet,
+        tcp_packet: &mut MutableTcpPacket,
+    ) {
+        tcp_packet.set_checksum(ipv4_checksum(
+            &tcp_packet.to_immutable(),
+            &ipv4_packet.get_source(),
+            &ipv4_packet.get_destination(),
+        ));
+
+        ipv4_packet.set_checksum(pnet::packet::ipv4::checksum(&ipv4_packet.to_immutable()));
+    }
+
+    pub async fn start(self: &Arc<Self>) -> Result<()> {
+        self.run_syn_map_cleaner().await?;
+        self.run_listener().await?;
+        self.peer_manager
+            .add_packet_process_pipeline(Box::new(self.clone()))
+            .await;
+        self.peer_manager
+            .add_nic_packet_process_pipeline(Box::new(self.clone()))
+            .await;
+
+        Ok(())
+    }
+
+    async fn run_syn_map_cleaner(&self) -> Result<()> {
+        let syn_map = self.syn_map.clone();
+        let tasks = self.tasks.clone();
+        let syn_map_cleaner_task = async move {
+            loop {
+                syn_map.retain(|_, entry| {
+                    if entry.start_time.elapsed() > Duration::from_secs(30) {
+                        tracing::warn!(entry = ?entry, "syn nat entry expired");
+                        entry.state.store(NatDstEntryState::Closed);
+                        false
+                    } else {
+                        true
+                    }
+                });
+                tokio::time::sleep(Duration::from_secs(10)).await;
+            }
+        };
+        tasks.lock().await.spawn(syn_map_cleaner_task);
+
+        Ok(())
+    }
+
+    async fn run_listener(&self) -> Result<()> {
+        // bind on both v4 & v6
+        let listen_addr = SocketAddr::new(Ipv4Addr::UNSPECIFIED.into(), 0);
+
+        let net_ns = self.global_ctx.net_ns.clone();
+        let tcp_listener = net_ns
+            .run_async(|| async { TcpListener::bind(&listen_addr).await })
+            .await?;
+
+        self.local_port.store(
+            tcp_listener.local_addr()?.port(),
+            std::sync::atomic::Ordering::Relaxed,
+        );
+
+        let tasks = self.tasks.clone();
+        let syn_map = self.syn_map.clone();
+        let conn_map = self.conn_map.clone();
+        let addr_conn_map = self.addr_conn_map.clone();
+        let accept_task = async move {
+            tracing::info!(listener = ?tcp_listener, "tcp connection start accepting");
+
+            let conn_map = conn_map.clone();
+            while let Ok((tcp_stream, socket_addr)) = tcp_listener.accept().await {
+                let Some(entry) = syn_map.get(&socket_addr) else {
+                    tracing::error!("tcp connection from unknown source: {:?}", socket_addr);
+                    continue;
+                };
+                assert_eq!(entry.state.load(), NatDstEntryState::SynReceived);
+
+                let entry_clone = entry.clone();
+                drop(entry);
+                syn_map.remove_if(&socket_addr, |_, entry| entry.id == entry_clone.id);
+
+                entry_clone.state.store(NatDstEntryState::ConnectingDst);
+
+                let _ = addr_conn_map.insert(entry_clone.src, entry_clone.clone());
+                let old_nat_val = conn_map.insert(entry_clone.id, entry_clone.clone());
+                assert!(old_nat_val.is_none());
+
+                tasks.lock().await.spawn(Self::connect_to_nat_dst(
+                    net_ns.clone(),
+                    tcp_stream,
+                    conn_map.clone(),
+                    addr_conn_map.clone(),
+                    entry_clone,
+                ));
+            }
+            tracing::error!("nat tcp listener exited");
+            panic!("nat tcp listener exited");
+        };
+        self.tasks
+            .lock()
+            .await
+            .spawn(accept_task.instrument(tracing::info_span!("tcp_proxy_listener")));
+
+        Ok(())
+    }
+
+    fn remove_entry_from_all_conn_map(
+        conn_map: ConnSockMap,
+        addr_conn_map: AddrConnSockMap,
+        nat_entry: ArcNatDstEntry,
+    ) {
+        conn_map.remove(&nat_entry.id);
+        addr_conn_map.remove_if(&nat_entry.src, |_, entry| entry.id == nat_entry.id);
+    }
+
+    async fn connect_to_nat_dst(
+        net_ns: NetNS,
+        src_tcp_stream: TcpStream,
+        conn_map: ConnSockMap,
+        addr_conn_map: AddrConnSockMap,
+        nat_entry: ArcNatDstEntry,
+    ) {
+        if let Err(e) = src_tcp_stream.set_nodelay(true) {
+            tracing::warn!("set_nodelay failed, ignore it: {:?}", e);
+        }
+
+        let _guard = net_ns.guard();
+        let socket = TcpSocket::new_v4().unwrap();
+        if let Err(e) = socket.set_nodelay(true) {
+            tracing::warn!("set_nodelay failed, ignore it: {:?}", e);
+        }
+        let Ok(Ok(dst_tcp_stream)) = tokio::time::timeout(
+            Duration::from_secs(10),
+            TcpSocket::new_v4().unwrap().connect(nat_entry.dst),
+        )
+        .await
+        else {
+            tracing::error!("connect to dst failed: {:?}", nat_entry);
+            nat_entry.state.store(NatDstEntryState::Closed);
+            Self::remove_entry_from_all_conn_map(conn_map, addr_conn_map, nat_entry);
+            return;
+        };
+        drop(_guard);
+
+        assert_eq!(nat_entry.state.load(), NatDstEntryState::ConnectingDst);
+        nat_entry.state.store(NatDstEntryState::Connected);
+
+        Self::handle_nat_connection(
+            src_tcp_stream,
+            dst_tcp_stream,
+            conn_map,
+            addr_conn_map,
+            nat_entry,
+        )
+        .await;
+    }
+
+    async fn handle_nat_connection(
+        mut src_tcp_stream: TcpStream,
+        mut dst_tcp_stream: TcpStream,
+        conn_map: ConnSockMap,
+        addr_conn_map: AddrConnSockMap,
+        nat_entry: ArcNatDstEntry,
+    ) {
+        let nat_entry_clone = nat_entry.clone();
+        nat_entry.tasks.lock().await.spawn(async move {
+            let ret = copy_bidirectional(&mut src_tcp_stream, &mut dst_tcp_stream).await;
+            tracing::trace!(nat_entry = ?nat_entry_clone, ret = ?ret, "nat tcp connection closed");
+            nat_entry_clone.state.store(NatDstEntryState::Closed);
+
+            Self::remove_entry_from_all_conn_map(conn_map, addr_conn_map, nat_entry_clone);
+        });
+    }
+
+    pub fn get_local_port(&self) -> u16 {
+        self.local_port.load(std::sync::atomic::Ordering::Relaxed)
+    }
+}

easytier-core/src/instance/instance.rs

@@ -0,0 +1,413 @@
+use std::borrow::BorrowMut;
+use std::io::Write;
+use std::sync::Arc;
+
+use futures::StreamExt;
+use pnet::packet::ethernet::EthernetPacket;
+use pnet::packet::ipv4::Ipv4Packet;
+
+use tokio::{sync::Mutex, task::JoinSet};
+use tokio_util::bytes::{Bytes, BytesMut};
+use tonic::transport::Server;
+use uuid::Uuid;
+
+use crate::common::config_fs::ConfigFs;
+use crate::common::error::Error;
+use crate::common::global_ctx::{ArcGlobalCtx, GlobalCtx};
+use crate::common::netns::NetNS;
+use crate::connector::direct::DirectConnectorManager;
+use crate::connector::manual::{ConnectorManagerRpcService, ManualConnectorManager};
+use crate::connector::udp_hole_punch::UdpHolePunchConnector;
+use crate::gateway::icmp_proxy::IcmpProxy;
+use crate::gateway::tcp_proxy::TcpProxy;
+use crate::peers::peer_manager::PeerManager;
+use crate::peers::rip_route::BasicRoute;
+use crate::peers::rpc_service::PeerManagerRpcService;
+use crate::tunnels::SinkItem;
+
+use tokio_stream::wrappers::ReceiverStream;
+
+use super::listeners::ListenerManager;
+use super::virtual_nic;
+
+pub struct InstanceConfigWriter {
+    config: ConfigFs,
+}
+
+impl InstanceConfigWriter {
+    pub fn new(inst_name: &str) -> Self {
+        InstanceConfigWriter {
+            config: ConfigFs::new(inst_name),
+        }
+    }
+
+    pub fn set_ns(self, net_ns: Option<String>) -> Self {
+        let net_ns_in_conf = if let Some(net_ns) = net_ns {
+            net_ns
+        } else {
+            "".to_string()
+        };
+
+        self.config
+            .add_file("net_ns")
+            .unwrap()
+            .write_all(net_ns_in_conf.as_bytes())
+            .unwrap();
+
+        self
+    }
+
+    pub fn set_addr(self, addr: String) -> Self {
+        self.config
+            .add_file("ipv4")
+            .unwrap()
+            .write_all(addr.as_bytes())
+            .unwrap();
+        self
+    }
+}
+
+pub struct Instance {
+    inst_name: String,
+
+    id: uuid::Uuid,
+
+    virtual_nic: Option<Arc<virtual_nic::VirtualNic>>,
+    peer_packet_receiver: Option<ReceiverStream<SinkItem>>,
+
+    tasks: JoinSet<()>,
+
+    peer_manager: Arc<PeerManager>,
+    listener_manager: Arc<Mutex<ListenerManager<PeerManager>>>,
+    conn_manager: Arc<ManualConnectorManager>,
+    direct_conn_manager: Arc<DirectConnectorManager>,
+    udp_hole_puncher: Arc<Mutex<UdpHolePunchConnector>>,
+
+    tcp_proxy: Arc<TcpProxy>,
+    icmp_proxy: Arc<IcmpProxy>,
+
+    global_ctx: ArcGlobalCtx,
+}
+
+impl Instance {
+    pub fn new(inst_name: &str) -> Self {
+        let config = ConfigFs::new(inst_name);
+        let net_ns_in_conf = config.get_or_default("net_ns", || "".to_string()).unwrap();
+        let net_ns = NetNS::new(if net_ns_in_conf.is_empty() {
+            None
+        } else {
+            Some(net_ns_in_conf.clone())
+        });
+
+        let addr = config
+            .get_or_default("ipv4", || "10.144.144.10".to_string())
+            .unwrap();
+
+        log::info!(
+            "[INIT] instance creating. inst_name: {}, addr: {}, netns: {}",
+            inst_name,
+            addr,
+            net_ns_in_conf
+        );
+
+        let (peer_packet_sender, peer_packet_receiver) = tokio::sync::mpsc::channel(100);
+
+        let global_ctx = Arc::new(GlobalCtx::new(inst_name, config, net_ns.clone()));
+
+        let id = global_ctx.get_id();
+
+        let peer_manager = Arc::new(PeerManager::new(
+            global_ctx.clone(),
+            peer_packet_sender.clone(),
+        ));
+
+        let listener_manager = Arc::new(Mutex::new(ListenerManager::new(
+            id,
+            net_ns.clone(),
+            peer_manager.clone(),
+        )));
+
+        let conn_manager = Arc::new(ManualConnectorManager::new(
+            id,
+            global_ctx.clone(),
+            peer_manager.clone(),
+        ));
+
+        let mut direct_conn_manager =
+            DirectConnectorManager::new(id, global_ctx.clone(), peer_manager.clone());
+        direct_conn_manager.run();
+
+        let udp_hole_puncher = UdpHolePunchConnector::new(global_ctx.clone(), peer_manager.clone());
+
+        let arc_tcp_proxy = TcpProxy::new(global_ctx.clone(), peer_manager.clone());
+        let arc_icmp_proxy = IcmpProxy::new(global_ctx.clone(), peer_manager.clone()).unwrap();
+
+        Instance {
+            inst_name: inst_name.to_string(),
+            id,
+
+            virtual_nic: None,
+            peer_packet_receiver: Some(ReceiverStream::new(peer_packet_receiver)),
+
+            tasks: JoinSet::new(),
+            peer_manager,
+            listener_manager,
+            conn_manager,
+            direct_conn_manager: Arc::new(direct_conn_manager),
+            udp_hole_puncher: Arc::new(Mutex::new(udp_hole_puncher)),
+
+            tcp_proxy: arc_tcp_proxy,
+            icmp_proxy: arc_icmp_proxy,
+
+            global_ctx,
+        }
+    }
+
+    pub fn get_conn_manager(&self) -> Arc<ManualConnectorManager> {
+        self.conn_manager.clone()
+    }
+
+    async fn do_forward_nic_to_peers_ipv4(ret: BytesMut, mgr: &PeerManager) {
+        if let Some(ipv4) = Ipv4Packet::new(&ret) {
+            if ipv4.get_version() != 4 {
+                tracing::info!("[USER_PACKET] not ipv4 packet: {:?}", ipv4);
+            }
+            let dst_ipv4 = ipv4.get_destination();
+            tracing::trace!(
+                ?ret,
+                "[USER_PACKET] recv new packet from tun device and forward to peers."
+            );
+            let send_ret = mgr.send_msg_ipv4(ret, dst_ipv4).await;
+            if send_ret.is_err() {
+                tracing::trace!(?send_ret, "[USER_PACKET] send_msg_ipv4 failed")
+            }
+        } else {
+            tracing::warn!(?ret, "[USER_PACKET] not ipv4 packet");
+        }
+    }
+
+    async fn do_forward_nic_to_peers_ethernet(mut ret: BytesMut, mgr: &PeerManager) {
+        if let Some(eth) = EthernetPacket::new(&ret) {
+            log::warn!("begin to forward: {:?}, type: {}", eth, eth.get_ethertype());
+            Self::do_forward_nic_to_peers_ipv4(ret.split_off(14), mgr).await;
+        } else {
+            log::warn!("not ipv4 packet: {:?}", ret);
+        }
+    }
+
+    fn do_forward_nic_to_peers(&mut self) -> Result<(), Error> {
+        // read from nic and write to corresponding tunnel
+        let nic = self.virtual_nic.as_ref().unwrap();
+        let nic = nic.clone();
+        let mgr = self.peer_manager.clone();
+
+        self.tasks.spawn(async move {
+            let mut stream = nic.pin_recv_stream();
+            while let Some(ret) = stream.next().await {
+                if ret.is_err() {
+                    log::error!("read from nic failed: {:?}", ret);
+                    break;
+                }
+                Self::do_forward_nic_to_peers_ipv4(ret.unwrap(), mgr.as_ref()).await;
+                // Self::do_forward_nic_to_peers_ethernet(ret.into(), mgr.as_ref()).await;
+            }
+        });
+
+        Ok(())
+    }
+
+    fn do_forward_peers_to_nic(
+        tasks: &mut JoinSet<()>,
+        nic: Arc<virtual_nic::VirtualNic>,
+        channel: Option<ReceiverStream<Bytes>>,
+    ) {
+        tasks.spawn(async move {
+            let send = nic.pin_send_stream();
+            let channel = channel.unwrap();
+            let ret = channel
+                .map(|packet| {
+                    log::trace!(
+                        "[USER_PACKET] forward packet from peers to nic. packet: {:?}",
+                        packet
+                    );
+                    Ok(packet)
+                })
+                .forward(send)
+                .await;
+            if ret.is_err() {
+                panic!("do_forward_tunnel_to_nic");
+            }
+        });
+    }
+
+    pub async fn run(&mut self) -> Result<(), Error> {
+        let ipv4_addr = self.global_ctx.get_ipv4().unwrap();
+
+        let mut nic = virtual_nic::VirtualNic::new(self.get_global_ctx())
+            .create_dev()
+            .await?
+            .link_up()
+            .await?
+            .remove_ip(None)
+            .await?
+            .add_ip(ipv4_addr, 24)
+            .await?;
+
+        if cfg!(target_os = "macos") {
+            nic = nic.add_route(ipv4_addr, 24).await?;
+        }
+
+        self.virtual_nic = Some(Arc::new(nic));
+
+        self.do_forward_nic_to_peers().unwrap();
+        Self::do_forward_peers_to_nic(
+            self.tasks.borrow_mut(),
+            self.virtual_nic.as_ref().unwrap().clone(),
+            self.peer_packet_receiver.take(),
+        );
+
+        self.listener_manager
+            .lock()
+            .await
+            .prepare_listeners()
+            .await?;
+        self.listener_manager.lock().await.run().await?;
+        self.peer_manager.run().await?;
+
+        let route = BasicRoute::new(self.id(), self.global_ctx.clone());
+        self.peer_manager.set_route(route).await;
+
+        self.run_rpc_server().unwrap();
+
+        self.tcp_proxy.start().await.unwrap();
+        self.icmp_proxy.start().await.unwrap();
+        self.run_proxy_cidrs_route_updater();
+
+        self.udp_hole_puncher.lock().await.run().await?;
+
+        Ok(())
+    }
+
+    pub fn get_peer_manager(&self) -> Arc<PeerManager> {
+        self.peer_manager.clone()
+    }
+
+    pub async fn close_peer_conn(&mut self, peer_id: &Uuid, conn_id: &Uuid) -> Result<(), Error> {
+        self.peer_manager
+            .get_peer_map()
+            .close_peer_conn(peer_id, conn_id)
+            .await?;
+        Ok(())
+    }
+
+    pub async fn wait(&mut self) {
+        while let Some(ret) = self.tasks.join_next().await {
+            log::info!("task finished: {:?}", ret);
+            ret.unwrap();
+        }
+    }
+
+    pub fn id(&self) -> uuid::Uuid {
+        self.id
+    }
+
+    fn run_rpc_server(&mut self) -> Result<(), Box<dyn std::error::Error>> {
+        let addr = "0.0.0.0:15888".parse()?;
+        let peer_mgr = self.peer_manager.clone();
+        let conn_manager = self.conn_manager.clone();
+        let net_ns = self.global_ctx.net_ns.clone();
+
+        self.tasks.spawn(async move {
+            let _g = net_ns.guard();
+            log::info!("[INIT RPC] start rpc server. addr: {}", addr);
+            Server::builder()
+                .add_service(
+                    easytier_rpc::peer_manage_rpc_server::PeerManageRpcServer::new(
+                        PeerManagerRpcService::new(peer_mgr),
+                    ),
+                )
+                .add_service(
+                    easytier_rpc::connector_manage_rpc_server::ConnectorManageRpcServer::new(
+                        ConnectorManagerRpcService(conn_manager.clone()),
+                    ),
+                )
+                .serve(addr)
+                .await
+                .unwrap();
+        });
+        Ok(())
+    }
+
+    fn run_proxy_cidrs_route_updater(&mut self) {
+        let peer_mgr = self.peer_manager.clone();
+        let net_ns = self.global_ctx.net_ns.clone();
+        let nic = self.virtual_nic.as_ref().unwrap().clone();
+
+        self.tasks.spawn(async move {
+            let mut cur_proxy_cidrs = vec![];
+            loop {
+                let mut proxy_cidrs = vec![];
+                let routes = peer_mgr.list_routes().await;
+                for r in routes {
+                    for cidr in r.proxy_cidrs {
+                        let Ok(cidr) = cidr.parse::<cidr::Ipv4Cidr>() else {
+                            continue;
+                        };
+                        proxy_cidrs.push(cidr);
+                    }
+                }
+
+                // if route is in cur_proxy_cidrs but not in proxy_cidrs, delete it.
+                for cidr in cur_proxy_cidrs.iter() {
+                    if proxy_cidrs.contains(cidr) {
+                        continue;
+                    }
+
+                    let _g = net_ns.guard();
+                    let ret = nic
+                        .get_ifcfg()
+                        .remove_ipv4_route(
+                            nic.ifname(),
+                            cidr.first_address(),
+                            cidr.network_length(),
+                        )
+                        .await;
+
+                    if ret.is_err() {
+                        tracing::trace!(
+                            cidr = ?cidr,
+                            err = ?ret,
+                            "remove route failed.",
+                        );
+                    }
+                }
+
+                for cidr in proxy_cidrs.iter() {
+                    if cur_proxy_cidrs.contains(cidr) {
+                        continue;
+                    }
+                    let _g = net_ns.guard();
+                    let ret = nic
+                        .get_ifcfg()
+                        .add_ipv4_route(nic.ifname(), cidr.first_address(), cidr.network_length())
+                        .await;
+
+                    if ret.is_err() {
+                        tracing::trace!(
+                            cidr = ?cidr,
+                            err = ?ret,
+                            "add route failed.",
+                        );
+                    }
+                }
+
+                cur_proxy_cidrs = proxy_cidrs;
+                tokio::time::sleep(std::time::Duration::from_secs(1)).await;
+            }
+        });
+    }
+
+    pub fn get_global_ctx(&self) -> ArcGlobalCtx {
+        self.global_ctx.clone()
+    }
+}

easytier-core/src/instance/listeners.rs

@@ -0,0 +1,150 @@
+use std::{fmt::Debug, sync::Arc};
+
+use async_trait::async_trait;
+use tokio::{sync::Mutex, task::JoinSet};
+
+use crate::{
+    common::{error::Error, netns::NetNS},
+    peers::peer_manager::PeerManager,
+    tunnels::{
+        ring_tunnel::RingTunnelListener, tcp_tunnel::TcpTunnelListener,
+        udp_tunnel::UdpTunnelListener, Tunnel, TunnelListener,
+    },
+};
+
+#[async_trait]
+pub trait TunnelHandlerForListener {
+    async fn handle_tunnel(&self, tunnel: Box<dyn Tunnel>) -> Result<(), Error>;
+}
+
+#[async_trait]
+impl TunnelHandlerForListener for PeerManager {
+    #[tracing::instrument]
+    async fn handle_tunnel(&self, tunnel: Box<dyn Tunnel>) -> Result<(), Error> {
+        self.add_tunnel_as_server(tunnel).await
+    }
+}
+
+pub struct ListenerManager<H> {
+    my_node_id: uuid::Uuid,
+    net_ns: NetNS,
+    listeners: Vec<Arc<Mutex<dyn TunnelListener>>>,
+    peer_manager: Arc<H>,
+
+    tasks: JoinSet<()>,
+}
+
+impl<H: TunnelHandlerForListener + Send + Sync + 'static + Debug> ListenerManager<H> {
+    pub fn new(my_node_id: uuid::Uuid, net_ns: NetNS, peer_manager: Arc<H>) -> Self {
+        Self {
+            my_node_id,
+            net_ns,
+            listeners: Vec::new(),
+            peer_manager,
+            tasks: JoinSet::new(),
+        }
+    }
+
+    pub async fn prepare_listeners(&mut self) -> Result<(), Error> {
+        self.add_listener(UdpTunnelListener::new(
+            "udp://0.0.0.0:11010".parse().unwrap(),
+        ))
+        .await?;
+        self.add_listener(TcpTunnelListener::new(
+            "tcp://0.0.0.0:11010".parse().unwrap(),
+        ))
+        .await?;
+        self.add_listener(RingTunnelListener::new(
+            format!("ring://{}", self.my_node_id).parse().unwrap(),
+        ))
+        .await?;
+        Ok(())
+    }
+
+    pub async fn add_listener<Listener>(&mut self, listener: Listener) -> Result<(), Error>
+    where
+        Listener: TunnelListener + 'static,
+    {
+        let listener = Arc::new(Mutex::new(listener));
+        self.listeners.push(listener);
+        Ok(())
+    }
+
+    #[tracing::instrument]
+    async fn run_listener(listener: Arc<Mutex<dyn TunnelListener>>, peer_manager: Arc<H>) {
+        let mut l = listener.lock().await;
+        while let Ok(ret) = l.accept().await {
+            tracing::info!(ret = ?ret, "conn accepted");
+            let server_ret = peer_manager.handle_tunnel(ret).await;
+            if let Err(e) = &server_ret {
+                tracing::error!(error = ?e, "handle conn error");
+            }
+        }
+    }
+
+    pub async fn run(&mut self) -> Result<(), Error> {
+        for listener in &self.listeners {
+            let _guard = self.net_ns.guard();
+            log::warn!("run listener: {:?}", listener);
+            listener.lock().await.listen().await?;
+            self.tasks.spawn(Self::run_listener(
+                listener.clone(),
+                self.peer_manager.clone(),
+            ));
+        }
+
+        Ok(())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use futures::{SinkExt, StreamExt};
+    use tokio::time::timeout;
+
+    use crate::tunnels::{ring_tunnel::RingTunnelConnector, TunnelConnector};
+
+    use super::*;
+
+    #[derive(Debug)]
+    struct MockListenerHandler {}
+
+    #[async_trait]
+    impl TunnelHandlerForListener for MockListenerHandler {
+        async fn handle_tunnel(&self, _tunnel: Box<dyn Tunnel>) -> Result<(), Error> {
+            let data = "abc";
+            _tunnel.pin_sink().send(data.into()).await.unwrap();
+            Err(Error::Unknown)
+        }
+    }
+
+    #[tokio::test]
+    async fn handle_error_in_accept() {
+        let net_ns = NetNS::new(None);
+        let handler = Arc::new(MockListenerHandler {});
+        let mut listener_mgr =
+            ListenerManager::new(uuid::Uuid::new_v4(), net_ns.clone(), handler.clone());
+
+        let ring_id = format!("ring://{}", uuid::Uuid::new_v4());
+
+        listener_mgr
+            .add_listener(RingTunnelListener::new(ring_id.parse().unwrap()))
+            .await
+            .unwrap();
+        listener_mgr.run().await.unwrap();
+
+        let connect_once = |ring_id| async move {
+            let tunnel = RingTunnelConnector::new(ring_id).connect().await.unwrap();
+            assert_eq!(tunnel.pin_stream().next().await.unwrap().unwrap(), "abc");
+            tunnel
+        };
+
+        timeout(std::time::Duration::from_secs(1), async move {
+            connect_once(ring_id.parse().unwrap()).await;
+            // handle tunnel fail should not impact the second connect
+            connect_once(ring_id.parse().unwrap()).await;
+        })
+        .await
+        .unwrap();
+    }
+}

easytier-core/src/instance/mod.rs

@@ -0,0 +1,4 @@
+pub mod instance;
+pub mod listeners;
+pub mod tun_codec;
+pub mod virtual_nic;

easytier-core/src/instance/tun_codec.rs

@@ -0,0 +1,179 @@
+use std::io;
+
+use byteorder::{NativeEndian, NetworkEndian, WriteBytesExt};
+use tokio_util::bytes::{BufMut, Bytes, BytesMut};
+use tokio_util::codec::{Decoder, Encoder};
+
+/// A packet protocol IP version
+#[derive(Debug, Clone, Copy, Default)]
+enum PacketProtocol {
+    #[default]
+    IPv4,
+    IPv6,
+    Other(u8),
+}
+
+// Note: the protocol in the packet information header is platform dependent.
+impl PacketProtocol {
+    #[cfg(any(target_os = "linux", target_os = "android"))]
+    fn into_pi_field(self) -> Result<u16, io::Error> {
+        use nix::libc;
+        match self {
+            PacketProtocol::IPv4 => Ok(libc::ETH_P_IP as u16),
+            PacketProtocol::IPv6 => Ok(libc::ETH_P_IPV6 as u16),
+            PacketProtocol::Other(_) => Err(io::Error::new(
+                io::ErrorKind::Other,
+                "neither an IPv4 nor IPv6 packet",
+            )),
+        }
+    }
+
+    #[cfg(any(target_os = "macos", target_os = "ios"))]
+    fn into_pi_field(self) -> Result<u16, io::Error> {
+        use nix::libc;
+        match self {
+            PacketProtocol::IPv4 => Ok(libc::PF_INET as u16),
+            PacketProtocol::IPv6 => Ok(libc::PF_INET6 as u16),
+            PacketProtocol::Other(_) => Err(io::Error::new(
+                io::ErrorKind::Other,
+                "neither an IPv4 nor IPv6 packet",
+            )),
+        }
+    }
+
+    #[cfg(target_os = "windows")]
+    fn into_pi_field(self) -> Result<u16, io::Error> {
+        unimplemented!()
+    }
+}
+
+#[derive(Debug)]
+pub enum TunPacketBuffer {
+    Bytes(Bytes),
+    BytesMut(BytesMut),
+}
+
+impl From<TunPacketBuffer> for Bytes {
+    fn from(buf: TunPacketBuffer) -> Self {
+        match buf {
+            TunPacketBuffer::Bytes(bytes) => bytes,
+            TunPacketBuffer::BytesMut(bytes) => bytes.freeze(),
+        }
+    }
+}
+
+impl AsRef<[u8]> for TunPacketBuffer {
+    fn as_ref(&self) -> &[u8] {
+        match self {
+            TunPacketBuffer::Bytes(bytes) => bytes.as_ref(),
+            TunPacketBuffer::BytesMut(bytes) => bytes.as_ref(),
+        }
+    }
+}
+
+/// A Tun Packet to be sent or received on the TUN interface.
+#[derive(Debug)]
+pub struct TunPacket(PacketProtocol, TunPacketBuffer);
+
+/// Infer the protocol based on the first nibble in the packet buffer.
+fn infer_proto(buf: &[u8]) -> PacketProtocol {
+    match buf[0] >> 4 {
+        4 => PacketProtocol::IPv4,
+        6 => PacketProtocol::IPv6,
+        p => PacketProtocol::Other(p),
+    }
+}
+
+impl TunPacket {
+    /// Create a new `TunPacket` based on a byte slice.
+    pub fn new(buffer: TunPacketBuffer) -> TunPacket {
+        let proto = infer_proto(buffer.as_ref());
+        TunPacket(proto, buffer)
+    }
+
+    /// Return this packet's bytes.
+    pub fn get_bytes(&self) -> &[u8] {
+        match &self.1 {
+            TunPacketBuffer::Bytes(bytes) => bytes.as_ref(),
+            TunPacketBuffer::BytesMut(bytes) => bytes.as_ref(),
+        }
+    }
+
+    pub fn into_bytes(self) -> Bytes {
+        match self.1 {
+            TunPacketBuffer::Bytes(bytes) => bytes,
+            TunPacketBuffer::BytesMut(bytes) => bytes.freeze(),
+        }
+    }
+
+    pub fn into_bytes_mut(self) -> BytesMut {
+        match self.1 {
+            TunPacketBuffer::Bytes(_) => panic!("cannot into_bytes_mut from bytes"),
+            TunPacketBuffer::BytesMut(bytes) => bytes,
+        }
+    }
+}
+
+/// A TunPacket Encoder/Decoder.
+pub struct TunPacketCodec(bool, i32);
+
+impl TunPacketCodec {
+    /// Create a new `TunPacketCodec` specifying whether the underlying
+    ///  tunnel Device has enabled the packet information header.
+    pub fn new(pi: bool, mtu: i32) -> TunPacketCodec {
+        TunPacketCodec(pi, mtu)
+    }
+}
+
+impl Decoder for TunPacketCodec {
+    type Item = TunPacket;
+    type Error = io::Error;
+
+    fn decode(&mut self, buf: &mut BytesMut) -> Result<Option<Self::Item>, Self::Error> {
+        if buf.is_empty() {
+            return Ok(None);
+        }
+
+        let mut pkt = buf.split_to(buf.len());
+
+        // reserve enough space for the next packet
+        if self.0 {
+            buf.reserve(self.1 as usize + 4);
+        } else {
+            buf.reserve(self.1 as usize);
+        }
+
+        // if the packet information is enabled we have to ignore the first 4 bytes
+        if self.0 {
+            let _ = pkt.split_to(4);
+        }
+
+        let proto = infer_proto(pkt.as_ref());
+        Ok(Some(TunPacket(proto, TunPacketBuffer::BytesMut(pkt))))
+    }
+}
+
+impl Encoder<TunPacket> for TunPacketCodec {
+    type Error = io::Error;
+
+    fn encode(&mut self, item: TunPacket, dst: &mut BytesMut) -> Result<(), Self::Error> {
+        dst.reserve(item.get_bytes().len() + 4);
+        match item {
+            TunPacket(proto, bytes) if self.0 => {
+                // build the packet information header comprising of 2 u16
+                // fields: flags and protocol.
+                let mut buf = Vec::<u8>::with_capacity(4);
+
+                // flags is always 0
+                buf.write_u16::<NativeEndian>(0)?;
+                // write the protocol as network byte order
+                buf.write_u16::<NetworkEndian>(proto.into_pi_field()?)?;
+
+                dst.put_slice(&buf);
+                dst.put(Bytes::from(bytes));
+            }
+            TunPacket(_, bytes) => dst.put(Bytes::from(bytes)),
+        }
+        Ok(())
+    }
+}

easytier-core/src/instance/virtual_nic.rs

@@ -0,0 +1,203 @@
+use std::{net::Ipv4Addr, pin::Pin};
+
+use crate::{
+    common::{
+        error::Result,
+        global_ctx::ArcGlobalCtx,
+        ifcfg::{IfConfiger, IfConfiguerTrait},
+    },
+    tunnels::{
+        codec::BytesCodec, common::FramedTunnel, DatagramSink, DatagramStream, Tunnel, TunnelError,
+    },
+};
+
+use futures::{SinkExt, StreamExt};
+use tokio_util::{bytes::Bytes, codec::Framed};
+use tun::Device;
+
+use super::tun_codec::{TunPacket, TunPacketCodec};
+
+pub struct VirtualNic {
+    dev_name: String,
+    queue_num: usize,
+
+    global_ctx: ArcGlobalCtx,
+
+    ifname: Option<String>,
+    tun: Option<Box<dyn Tunnel>>,
+    ifcfg: Box<dyn IfConfiguerTrait + Send + Sync + 'static>,
+}
+
+impl VirtualNic {
+    pub fn new(global_ctx: ArcGlobalCtx) -> Self {
+        Self {
+            dev_name: "".to_owned(),
+            queue_num: 1,
+            global_ctx,
+            ifname: None,
+            tun: None,
+            ifcfg: Box::new(IfConfiger {}),
+        }
+    }
+
+    pub fn set_dev_name(mut self, dev_name: &str) -> Result<Self> {
+        self.dev_name = dev_name.to_owned();
+        Ok(self)
+    }
+
+    pub fn set_queue_num(mut self, queue_num: usize) -> Result<Self> {
+        self.queue_num = queue_num;
+        Ok(self)
+    }
+
+    async fn create_dev_ret_err(&mut self) -> Result<()> {
+        let mut config = tun::Configuration::default();
+        let has_packet_info = cfg!(target_os = "macos");
+        config.layer(tun::Layer::L3);
+
+        #[cfg(target_os = "linux")]
+        {
+            config.platform(|config| {
+                // detect protocol by ourselves for cross platform
+                config.packet_information(false);
+            });
+            config.name(self.dev_name.clone());
+        }
+
+        if self.queue_num != 1 {
+            todo!("queue_num != 1")
+        }
+        config.queues(self.queue_num);
+        config.up();
+
+        let dev = {
+            let _g = self.global_ctx.net_ns.guard();
+            tun::create_as_async(&config)?
+        };
+        let ifname = dev.get_ref().name()?;
+        self.ifcfg.wait_interface_show(ifname.as_str()).await?;
+
+        let ft: Box<dyn Tunnel> = if has_packet_info {
+            let framed = Framed::new(dev, TunPacketCodec::new(true, 2500));
+            let (sink, stream) = framed.split();
+
+            let new_stream = stream.map(|item| match item {
+                Ok(item) => Ok(item.into_bytes_mut()),
+                Err(err) => {
+                    println!("tun stream error: {:?}", err);
+                    Err(TunnelError::TunError(err.to_string()))
+                }
+            });
+
+            let new_sink = Box::pin(sink.with(|item: Bytes| async move {
+                if false {
+                    return Err(TunnelError::TunError("tun sink error".to_owned()));
+                }
+                Ok(TunPacket::new(super::tun_codec::TunPacketBuffer::Bytes(
+                    item,
+                )))
+            }));
+
+            Box::new(FramedTunnel::new(new_stream, new_sink, None))
+        } else {
+            let framed = Framed::new(dev, BytesCodec::new(2500));
+            let (sink, stream) = framed.split();
+            Box::new(FramedTunnel::new(stream, sink, None))
+        };
+
+        self.ifname = Some(ifname.to_owned());
+        self.tun = Some(ft);
+
+        Ok(())
+    }
+
+    pub async fn create_dev(mut self) -> Result<Self> {
+        self.create_dev_ret_err().await?;
+        Ok(self)
+    }
+
+    pub fn ifname(&self) -> &str {
+        self.ifname.as_ref().unwrap().as_str()
+    }
+
+    pub async fn link_up(self) -> Result<Self> {
+        let _g = self.global_ctx.net_ns.guard();
+        self.ifcfg.set_link_status(self.ifname(), true).await?;
+        Ok(self)
+    }
+
+    pub async fn add_route(self, address: Ipv4Addr, cidr: u8) -> Result<Self> {
+        let _g = self.global_ctx.net_ns.guard();
+        self.ifcfg
+            .add_ipv4_route(self.ifname(), address, cidr)
+            .await?;
+        Ok(self)
+    }
+
+    pub async fn remove_ip(self, ip: Option<Ipv4Addr>) -> Result<Self> {
+        let _g = self.global_ctx.net_ns.guard();
+        self.ifcfg.remove_ip(self.ifname(), ip).await?;
+        Ok(self)
+    }
+
+    pub async fn add_ip(self, ip: Ipv4Addr, cidr: i32) -> Result<Self> {
+        let _g = self.global_ctx.net_ns.guard();
+        self.ifcfg
+            .add_ipv4_ip(self.ifname(), ip, cidr as u8)
+            .await?;
+        Ok(self)
+    }
+
+    pub fn pin_recv_stream(&self) -> Pin<Box<dyn DatagramStream>> {
+        self.tun.as_ref().unwrap().pin_stream()
+    }
+
+    pub fn pin_send_stream(&self) -> Pin<Box<dyn DatagramSink>> {
+        self.tun.as_ref().unwrap().pin_sink()
+    }
+
+    pub fn get_ifcfg(&self) -> &dyn IfConfiguerTrait {
+        self.ifcfg.as_ref()
+    }
+}
+#[cfg(test)]
+mod tests {
+    use crate::{
+        common::{error::Error, global_ctx::tests::get_mock_global_ctx},
+        tests::enable_log,
+    };
+
+    use super::VirtualNic;
+
+    async fn run_test_helper() -> Result<VirtualNic, Error> {
+        let dev = VirtualNic::new(get_mock_global_ctx()).create_dev().await?;
+
+        tokio::time::sleep(tokio::time::Duration::from_secs(1)).await;
+
+        dev.link_up()
+            .await?
+            .remove_ip(None)
+            .await?
+            .add_ip("10.144.111.1".parse().unwrap(), 24)
+            .await
+    }
+
+    #[tokio::test]
+    async fn tun_test() {
+        enable_log();
+        let _dev = run_test_helper().await.unwrap();
+
+        // let mut stream = nic.pin_recv_stream();
+        // while let Some(item) = stream.next().await {
+        //     println!("item: {:?}", item);
+        // }
+
+        // let framed = dev.into_framed();
+        // let (mut s, mut b) = framed.split();
+        // loop {
+        //     let tmp = b.next().await.unwrap().unwrap();
+        //     let tmp = EthernetPacket::new(tmp.get_bytes());
+        //     println!("ret: {:?}", tmp.unwrap());
+        // }
+    }
+}

easytier-core/src/main.rs

@@ -0,0 +1,103 @@
+#![allow(dead_code)]
+
+#[cfg(test)]
+mod tests;
+
+use clap::Parser;
+
+mod common;
+mod connector;
+mod gateway;
+mod instance;
+mod peers;
+mod tunnels;
+
+use instance::instance::{Instance, InstanceConfigWriter};
+use tracing::level_filters::LevelFilter;
+use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt, EnvFilter, Layer};
+
+#[derive(Parser, Debug)]
+#[command(author, version, about, long_about = None)]
+struct Cli {
+    /// the instance name
+    #[arg(short = 'n', long, default_value = "default")]
+    instance_name: String,
+
+    /// specify the network namespace, default is the root namespace
+    #[arg(long)]
+    net_ns: Option<String>,
+
+    #[arg(short, long)]
+    ipv4: Option<String>,
+
+    #[arg(short, long)]
+    peers: Vec<String>,
+}
+
+fn init_logger() {
+    // logger to rolling file
+    let file_filter = EnvFilter::builder()
+        .with_default_directive(LevelFilter::INFO.into())
+        .from_env()
+        .unwrap();
+    let file_appender = tracing_appender::rolling::Builder::new()
+        .rotation(tracing_appender::rolling::Rotation::DAILY)
+        .max_log_files(5)
+        .filename_prefix("core.log")
+        .build("/var/log/easytier")
+        .expect("failed to initialize rolling file appender");
+    let mut file_layer = tracing_subscriber::fmt::layer();
+    file_layer.set_ansi(false);
+    let file_layer = file_layer
+        .with_writer(file_appender)
+        .with_filter(file_filter);
+
+    // logger to console
+    let console_filter = EnvFilter::builder()
+        .with_default_directive(LevelFilter::WARN.into())
+        .from_env()
+        .unwrap();
+    let console_layer = tracing_subscriber::fmt::layer()
+        .pretty()
+        .with_writer(std::io::stderr)
+        .with_filter(console_filter);
+
+    tracing_subscriber::Registry::default()
+        .with(console_layer)
+        .with(file_layer)
+        .init();
+}
+
+#[tokio::main(flavor = "current_thread")]
+#[tracing::instrument]
+pub async fn main() {
+    init_logger();
+
+    let cli = Cli::parse();
+    tracing::info!(cli = ?cli, "cli args parsed");
+
+    let cfg = InstanceConfigWriter::new(cli.instance_name.as_str()).set_ns(cli.net_ns.clone());
+    if let Some(ipv4) = &cli.ipv4 {
+        cfg.set_addr(ipv4.clone());
+    }
+
+    let mut inst = Instance::new(cli.instance_name.as_str());
+
+    let mut events = inst.get_global_ctx().subscribe();
+    tokio::spawn(async move {
+        while let Ok(e) = events.recv().await {
+            log::warn!("event: {:?}", e);
+        }
+    });
+
+    inst.run().await.unwrap();
+
+    for peer in cli.peers {
+        inst.get_conn_manager()
+            .add_connector_by_url(peer.as_str())
+            .await
+            .unwrap();
+    }
+
+    inst.wait().await;
+}

easytier-core/src/peers/mod.rs

@@ -0,0 +1,14 @@
+pub mod packet;
+pub mod peer;
+pub mod peer_conn;
+pub mod peer_manager;
+pub mod peer_map;
+pub mod peer_rpc;
+pub mod rip_route;
+pub mod route_trait;
+pub mod rpc_service;
+
+#[cfg(test)]
+pub mod tests;
+
+pub type PeerId = uuid::Uuid;

easytier-core/src/peers/packet.rs

@@ -0,0 +1,205 @@
+use rkyv::{Archive, Deserialize, Serialize};
+use tokio_util::bytes::Bytes;
+
+use crate::common::rkyv_util::{decode_from_bytes, encode_to_bytes};
+
+const MAGIC: u32 = 0xd1e1a5e1;
+const VERSION: u32 = 1;
+
+#[derive(Archive, Deserialize, Serialize, PartialEq, Clone)]
+#[archive(compare(PartialEq), check_bytes)]
+// Derives can be passed through to the generated type:
+#[archive_attr(derive(Debug))]
+pub struct UUID(uuid::Bytes);
+
+// impl Debug for UUID
+impl std::fmt::Debug for UUID {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        let uuid = uuid::Uuid::from_bytes(self.0);
+        write!(f, "{}", uuid)
+    }
+}
+
+impl From<uuid::Uuid> for UUID {
+    fn from(uuid: uuid::Uuid) -> Self {
+        UUID(*uuid.as_bytes())
+    }
+}
+
+impl From<UUID> for uuid::Uuid {
+    fn from(uuid: UUID) -> Self {
+        uuid::Uuid::from_bytes(uuid.0)
+    }
+}
+
+impl ArchivedUUID {
+    pub fn to_uuid(&self) -> uuid::Uuid {
+        uuid::Uuid::from_bytes(self.0)
+    }
+}
+
+impl From<&ArchivedUUID> for UUID {
+    fn from(uuid: &ArchivedUUID) -> Self {
+        UUID(uuid.0)
+    }
+}
+
+#[derive(Archive, Deserialize, Serialize, Debug)]
+#[archive(compare(PartialEq), check_bytes)]
+// Derives can be passed through to the generated type:
+#[archive_attr(derive(Debug))]
+pub struct HandShake {
+    pub magic: u32,
+    pub my_peer_id: UUID,
+    pub version: u32,
+    pub features: Vec<String>,
+    // pub interfaces: Vec<String>,
+}
+
+#[derive(Archive, Deserialize, Serialize, Debug)]
+#[archive(compare(PartialEq), check_bytes)]
+#[archive_attr(derive(Debug))]
+pub struct RoutePacket {
+    pub route_id: u8,
+    pub body: Vec<u8>,
+}
+
+#[derive(Archive, Deserialize, Serialize, Debug)]
+#[archive(compare(PartialEq), check_bytes)]
+// Derives can be passed through to the generated type:
+#[archive_attr(derive(Debug))]
+pub enum CtrlPacketBody {
+    HandShake(HandShake),
+    RoutePacket(RoutePacket),
+    Ping,
+    Pong,
+    TaRpc(u32, bool, Vec<u8>), // u32: service_id, bool: is_req, Vec<u8>: rpc body
+}
+
+#[derive(Archive, Deserialize, Serialize, Debug)]
+#[archive(compare(PartialEq), check_bytes)]
+// Derives can be passed through to the generated type:
+#[archive_attr(derive(Debug))]
+pub struct DataPacketBody {
+    pub data: Vec<u8>,
+}
+
+#[derive(Archive, Deserialize, Serialize, Debug)]
+#[archive(compare(PartialEq), check_bytes)]
+// Derives can be passed through to the generated type:
+#[archive_attr(derive(Debug))]
+pub enum PacketBody {
+    Ctrl(CtrlPacketBody),
+    Data(DataPacketBody),
+}
+
+#[derive(Archive, Deserialize, Serialize, Debug)]
+#[archive(compare(PartialEq), check_bytes)]
+// Derives can be passed through to the generated type:
+#[archive_attr(derive(Debug))]
+pub struct Packet {
+    pub from_peer: UUID,
+    pub to_peer: Option<UUID>,
+    pub body: PacketBody,
+}
+
+impl Packet {
+    pub fn decode(v: &[u8]) -> &ArchivedPacket {
+        decode_from_bytes::<Packet>(v).unwrap()
+    }
+}
+
+impl From<Packet> for Bytes {
+    fn from(val: Packet) -> Self {
+        encode_to_bytes::<_, 4096>(&val)
+    }
+}
+
+impl Packet {
+    pub fn new_handshake(from_peer: uuid::Uuid) -> Self {
+        Packet {
+            from_peer: from_peer.into(),
+            to_peer: None,
+            body: PacketBody::Ctrl(CtrlPacketBody::HandShake(HandShake {
+                magic: MAGIC,
+                my_peer_id: from_peer.into(),
+                version: VERSION,
+                features: Vec::new(),
+            })),
+        }
+    }
+
+    pub fn new_data_packet(from_peer: uuid::Uuid, to_peer: uuid::Uuid, data: &[u8]) -> Self {
+        Packet {
+            from_peer: from_peer.into(),
+            to_peer: Some(to_peer.into()),
+            body: PacketBody::Data(DataPacketBody {
+                data: data.to_vec(),
+            }),
+        }
+    }
+
+    pub fn new_route_packet(
+        from_peer: uuid::Uuid,
+        to_peer: uuid::Uuid,
+        route_id: u8,
+        data: &[u8],
+    ) -> Self {
+        Packet {
+            from_peer: from_peer.into(),
+            to_peer: Some(to_peer.into()),
+            body: PacketBody::Ctrl(CtrlPacketBody::RoutePacket(RoutePacket {
+                route_id,
+                body: data.to_vec(),
+            })),
+        }
+    }
+
+    pub fn new_ping_packet(from_peer: uuid::Uuid, to_peer: uuid::Uuid) -> Self {
+        Packet {
+            from_peer: from_peer.into(),
+            to_peer: Some(to_peer.into()),
+            body: PacketBody::Ctrl(CtrlPacketBody::Ping),
+        }
+    }
+
+    pub fn new_pong_packet(from_peer: uuid::Uuid, to_peer: uuid::Uuid) -> Self {
+        Packet {
+            from_peer: from_peer.into(),
+            to_peer: Some(to_peer.into()),
+            body: PacketBody::Ctrl(CtrlPacketBody::Pong),
+        }
+    }
+
+    pub fn new_tarpc_packet(
+        from_peer: uuid::Uuid,
+        to_peer: uuid::Uuid,
+        service_id: u32,
+        is_req: bool,
+        body: Vec<u8>,
+    ) -> Self {
+        Packet {
+            from_peer: from_peer.into(),
+            to_peer: Some(to_peer.into()),
+            body: PacketBody::Ctrl(CtrlPacketBody::TaRpc(service_id, is_req, body)),
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[tokio::test]
+    async fn serialize() {
+        let a = "abcde";
+        let out = Packet::new_data_packet(uuid::Uuid::new_v4(), uuid::Uuid::new_v4(), a.as_bytes());
+        // let out = T::new(a.as_bytes());
+        let out_bytes: Bytes = out.into();
+        println!("out str: {:?}", a.as_bytes());
+        println!("out bytes: {:?}", out_bytes);
+
+        let archived = Packet::decode(&out_bytes[..]);
+        println!("in packet: {:?}", archived);
+    }
+}

easytier-core/src/peers/peer.rs

@@ -0,0 +1,218 @@
+use std::sync::Arc;
+
+use dashmap::DashMap;
+use easytier_rpc::PeerConnInfo;
+
+use tokio::{
+    select,
+    sync::{mpsc, Mutex},
+    task::JoinHandle,
+};
+use tokio_util::bytes::Bytes;
+use tracing::Instrument;
+use uuid::Uuid;
+
+use crate::common::{
+    error::Error,
+    global_ctx::{ArcGlobalCtx, GlobalCtxEvent},
+};
+
+use super::peer_conn::PeerConn;
+
+type ArcPeerConn = Arc<Mutex<PeerConn>>;
+type ConnMap = Arc<DashMap<Uuid, ArcPeerConn>>;
+
+pub struct Peer {
+    pub peer_node_id: uuid::Uuid,
+    conns: ConnMap,
+    global_ctx: ArcGlobalCtx,
+
+    packet_recv_chan: mpsc::Sender<Bytes>,
+
+    close_event_sender: mpsc::Sender<Uuid>,
+    close_event_listener: JoinHandle<()>,
+
+    shutdown_notifier: Arc<tokio::sync::Notify>,
+}
+
+impl Peer {
+    pub fn new(
+        peer_node_id: uuid::Uuid,
+        packet_recv_chan: mpsc::Sender<Bytes>,
+        global_ctx: ArcGlobalCtx,
+    ) -> Self {
+        let conns: ConnMap = Arc::new(DashMap::new());
+        let (close_event_sender, mut close_event_receiver) = mpsc::channel(10);
+        let shutdown_notifier = Arc::new(tokio::sync::Notify::new());
+
+        let conns_copy = conns.clone();
+        let shutdown_notifier_copy = shutdown_notifier.clone();
+        let global_ctx_copy = global_ctx.clone();
+        let close_event_listener = tokio::spawn(
+            async move {
+                loop {
+                    select! {
+                        ret = close_event_receiver.recv() => {
+                            if ret.is_none() {
+                                break;
+                            }
+                            let ret = ret.unwrap();
+                            tracing::warn!(
+                                ?peer_node_id,
+                                ?ret,
+                                "notified that peer conn is closed",
+                            );
+
+                            if let Some((_, conn)) = conns_copy.remove(&ret) {
+                                global_ctx_copy.issue_event(GlobalCtxEvent::PeerConnRemoved(
+                                    conn.lock().await.get_conn_info(),
+                                ));
+                            }
+                        }
+
+                        _ = shutdown_notifier_copy.notified() => {
+                            close_event_receiver.close();
+                            tracing::warn!(?peer_node_id, "peer close event listener notified");
+                        }
+                    }
+                }
+                tracing::info!("peer {} close event listener exit", peer_node_id);
+            }
+            .instrument(tracing::info_span!(
+                "peer_close_event_listener",
+                ?peer_node_id,
+            )),
+        );
+
+        Peer {
+            peer_node_id,
+            conns: conns.clone(),
+            packet_recv_chan,
+            global_ctx,
+
+            close_event_sender,
+            close_event_listener,
+
+            shutdown_notifier,
+        }
+    }
+
+    pub async fn add_peer_conn(&self, mut conn: PeerConn) {
+        conn.set_close_event_sender(self.close_event_sender.clone());
+        conn.start_recv_loop(self.packet_recv_chan.clone());
+        self.global_ctx
+            .issue_event(GlobalCtxEvent::PeerConnAdded(conn.get_conn_info()));
+        self.conns
+            .insert(conn.get_conn_id(), Arc::new(Mutex::new(conn)));
+    }
+
+    pub async fn send_msg(&self, msg: Bytes) -> Result<(), Error> {
+        let Some(conn) = self.conns.iter().next() else {
+            return Err(Error::PeerNoConnectionError(self.peer_node_id));
+        };
+
+        let conn_clone = conn.clone();
+        drop(conn);
+        conn_clone.lock().await.send_msg(msg).await?;
+
+        Ok(())
+    }
+
+    pub async fn close_peer_conn(&self, conn_id: &Uuid) -> Result<(), Error> {
+        let has_key = self.conns.contains_key(conn_id);
+        if !has_key {
+            return Err(Error::NotFound);
+        }
+        self.close_event_sender.send(conn_id.clone()).await.unwrap();
+        Ok(())
+    }
+
+    pub async fn list_peer_conns(&self) -> Vec<PeerConnInfo> {
+        let mut conns = vec![];
+        for conn in self.conns.iter() {
+            // do not lock here, otherwise it will cause dashmap deadlock
+            conns.push(conn.clone());
+        }
+
+        let mut ret = Vec::new();
+        for conn in conns {
+            ret.push(conn.lock().await.get_conn_info());
+        }
+        ret
+    }
+}
+
+// pritn on drop
+impl Drop for Peer {
+    fn drop(&mut self) {
+        self.shutdown_notifier.notify_one();
+        tracing::info!("peer {} drop", self.peer_node_id);
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::sync::Arc;
+
+    use tokio::{sync::mpsc, time::timeout};
+
+    use crate::{
+        common::{config_fs::ConfigFs, global_ctx::GlobalCtx, netns::NetNS},
+        peers::peer_conn::PeerConn,
+        tunnels::ring_tunnel::create_ring_tunnel_pair,
+    };
+
+    use super::Peer;
+
+    #[tokio::test]
+    async fn close_peer() {
+        let (local_packet_send, _local_packet_recv) = mpsc::channel(10);
+        let (remote_packet_send, _remote_packet_recv) = mpsc::channel(10);
+        let global_ctx = Arc::new(GlobalCtx::new(
+            "test",
+            ConfigFs::new("/tmp/easytier-test"),
+            NetNS::new(None),
+        ));
+        let local_peer = Peer::new(uuid::Uuid::new_v4(), local_packet_send, global_ctx.clone());
+        let remote_peer = Peer::new(uuid::Uuid::new_v4(), remote_packet_send, global_ctx.clone());
+
+        let (local_tunnel, remote_tunnel) = create_ring_tunnel_pair();
+        let mut local_peer_conn =
+            PeerConn::new(local_peer.peer_node_id, global_ctx.clone(), local_tunnel);
+        let mut remote_peer_conn =
+            PeerConn::new(remote_peer.peer_node_id, global_ctx.clone(), remote_tunnel);
+
+        assert!(!local_peer_conn.handshake_done());
+        assert!(!remote_peer_conn.handshake_done());
+
+        let (a, b) = tokio::join!(
+            local_peer_conn.do_handshake_as_client(),
+            remote_peer_conn.do_handshake_as_server()
+        );
+        a.unwrap();
+        b.unwrap();
+
+        let local_conn_id = local_peer_conn.get_conn_id();
+
+        local_peer.add_peer_conn(local_peer_conn).await;
+        remote_peer.add_peer_conn(remote_peer_conn).await;
+
+        assert_eq!(local_peer.list_peer_conns().await.len(), 1);
+        assert_eq!(remote_peer.list_peer_conns().await.len(), 1);
+
+        let close_handler =
+            tokio::spawn(async move { local_peer.close_peer_conn(&local_conn_id).await });
+
+        // wait for remote peer conn close
+        timeout(std::time::Duration::from_secs(5), async {
+            while (&remote_peer).list_peer_conns().await.len() != 0 {
+                tokio::time::sleep(std::time::Duration::from_millis(100)).await;
+            }
+        })
+        .await
+        .unwrap();
+
+        println!("wait for close handler");
+        close_handler.await.unwrap().unwrap();
+    }
+}

easytier-core/src/peers/peer_conn.rs

@@ -0,0 +1,484 @@
+use std::{pin::Pin, sync::Arc};
+
+use easytier_rpc::{PeerConnInfo, PeerConnStats};
+use futures::{SinkExt, StreamExt};
+use pnet::datalink::NetworkInterface;
+
+use tokio::{
+    sync::{broadcast, mpsc},
+    task::JoinSet,
+    time::{timeout, Duration},
+};
+
+use tokio_util::{
+    bytes::{Bytes, BytesMut},
+    sync::PollSender,
+};
+use tracing::Instrument;
+
+use crate::{
+    common::global_ctx::ArcGlobalCtx,
+    define_tunnel_filter_chain,
+    tunnels::{
+        stats::{Throughput, WindowLatency},
+        tunnel_filter::StatsRecorderTunnelFilter,
+        DatagramSink, Tunnel, TunnelError,
+    },
+};
+
+use super::packet::{self, ArchivedCtrlPacketBody, ArchivedHandShake, Packet};
+
+pub type PacketRecvChan = mpsc::Sender<Bytes>;
+
+macro_rules! wait_response {
+    ($stream: ident, $out_var:ident, $pattern:pat_param => $value:expr) => {
+        let rsp_vec = timeout(Duration::from_secs(1), $stream.next()).await;
+        if rsp_vec.is_err() {
+            return Err(TunnelError::WaitRespError(
+                "wait handshake response timeout".to_owned(),
+            ));
+        }
+        let rsp_vec = rsp_vec.unwrap().unwrap()?;
+
+        let $out_var;
+        let rsp_bytes = Packet::decode(&rsp_vec);
+        match &rsp_bytes.body {
+            $pattern => $out_var = $value,
+            _ => {
+                log::error!(
+                    "unexpected packet: {:?}, pattern: {:?}",
+                    rsp_bytes,
+                    stringify!($pattern)
+                );
+                return Err(TunnelError::WaitRespError("unexpected packet".to_owned()));
+            }
+        }
+    };
+}
+
+pub struct PeerInfo {
+    magic: u32,
+    pub my_peer_id: uuid::Uuid,
+    version: u32,
+    pub features: Vec<String>,
+    pub interfaces: Vec<NetworkInterface>,
+}
+
+impl<'a> From<&ArchivedHandShake> for PeerInfo {
+    fn from(hs: &ArchivedHandShake) -> Self {
+        PeerInfo {
+            magic: hs.magic.into(),
+            my_peer_id: hs.my_peer_id.to_uuid(),
+            version: hs.version.into(),
+            features: hs.features.iter().map(|x| x.to_string()).collect(),
+            interfaces: Vec::new(),
+        }
+    }
+}
+
+define_tunnel_filter_chain!(PeerConnTunnel, stats = StatsRecorderTunnelFilter);
+
+pub struct PeerConn {
+    conn_id: uuid::Uuid,
+
+    my_node_id: uuid::Uuid,
+    global_ctx: ArcGlobalCtx,
+
+    sink: Pin<Box<dyn DatagramSink>>,
+    tunnel: Box<dyn Tunnel>,
+
+    tasks: JoinSet<Result<(), TunnelError>>,
+
+    info: Option<PeerInfo>,
+
+    close_event_sender: Option<mpsc::Sender<uuid::Uuid>>,
+
+    ctrl_resp_sender: broadcast::Sender<Bytes>,
+
+    latency_stats: Arc<WindowLatency>,
+    throughput: Arc<Throughput>,
+}
+
+enum PeerConnPacketType {
+    Data(Bytes),
+    CtrlReq(Bytes),
+    CtrlResp(Bytes),
+}
+
+static CTRL_REQ_PACKET_PREFIX: &[u8] = &[0x12, 0x34, 0x56, 0x78, 0x9a, 0xbc, 0xde, 0xf0];
+static CTRL_RESP_PACKET_PREFIX: &[u8] = &[0x12, 0x34, 0x56, 0x78, 0x9a, 0xbc, 0xde, 0xf1];
+
+impl PeerConn {
+    pub fn new(node_id: uuid::Uuid, global_ctx: ArcGlobalCtx, tunnel: Box<dyn Tunnel>) -> Self {
+        let (ctrl_sender, _ctrl_receiver) = broadcast::channel(100);
+        let peer_conn_tunnel = PeerConnTunnel::new();
+        let tunnel = peer_conn_tunnel.wrap_tunnel(tunnel);
+
+        PeerConn {
+            conn_id: uuid::Uuid::new_v4(),
+
+            my_node_id: node_id,
+            global_ctx,
+
+            sink: tunnel.pin_sink(),
+            tunnel: Box::new(tunnel),
+
+            tasks: JoinSet::new(),
+
+            info: None,
+            close_event_sender: None,
+
+            ctrl_resp_sender: ctrl_sender,
+
+            latency_stats: Arc::new(WindowLatency::new(15)),
+            throughput: peer_conn_tunnel.stats.get_throughput().clone(),
+        }
+    }
+
+    pub fn get_conn_id(&self) -> uuid::Uuid {
+        self.conn_id
+    }
+
+    pub async fn do_handshake_as_server(&mut self) -> Result<(), TunnelError> {
+        let mut stream = self.tunnel.pin_stream();
+        let mut sink = self.tunnel.pin_sink();
+
+        wait_response!(stream, hs_req, packet::ArchivedPacketBody::Ctrl(ArchivedCtrlPacketBody::HandShake(x)) => x);
+        self.info = Some(PeerInfo::from(hs_req));
+        log::info!("handshake request: {:?}", hs_req);
+
+        let hs_req = self
+            .global_ctx
+            .net_ns
+            .run(|| packet::Packet::new_handshake(self.my_node_id));
+        sink.send(hs_req.into()).await?;
+
+        Ok(())
+    }
+
+    pub async fn do_handshake_as_client(&mut self) -> Result<(), TunnelError> {
+        let mut stream = self.tunnel.pin_stream();
+        let mut sink = self.tunnel.pin_sink();
+
+        let hs_req = self
+            .global_ctx
+            .net_ns
+            .run(|| packet::Packet::new_handshake(self.my_node_id));
+        sink.send(hs_req.into()).await?;
+
+        wait_response!(stream, hs_rsp, packet::ArchivedPacketBody::Ctrl(ArchivedCtrlPacketBody::HandShake(x)) => x);
+        self.info = Some(PeerInfo::from(hs_rsp));
+        log::info!("handshake response: {:?}", hs_rsp);
+
+        Ok(())
+    }
+
+    pub fn handshake_done(&self) -> bool {
+        self.info.is_some()
+    }
+
+    async fn do_pingpong_once(
+        my_node_id: uuid::Uuid,
+        peer_id: uuid::Uuid,
+        sink: &mut Pin<Box<dyn DatagramSink>>,
+        receiver: &mut broadcast::Receiver<Bytes>,
+    ) -> Result<u128, TunnelError> {
+        // should add seq here. so latency can be calculated more accurately
+        let req = Self::build_ctrl_msg(
+            packet::Packet::new_ping_packet(my_node_id, peer_id).into(),
+            true,
+        );
+        log::trace!("send ping packet: {:?}", req);
+        sink.send(req).await?;
+
+        let now = std::time::Instant::now();
+
+        // wait until we get a pong packet in ctrl_resp_receiver
+        let resp = timeout(Duration::from_secs(4), async {
+            loop {
+                match receiver.recv().await {
+                    Ok(p) => {
+                        if let packet::ArchivedPacketBody::Ctrl(
+                            packet::ArchivedCtrlPacketBody::Pong,
+                        ) = &Packet::decode(&p).body
+                        {
+                            break;
+                        }
+                    }
+                    Err(e) => {
+                        log::warn!("recv pong resp error: {:?}", e);
+                        return Err(TunnelError::WaitRespError(
+                            "recv pong resp error".to_owned(),
+                        ));
+                    }
+                }
+            }
+            Ok(())
+        })
+        .await;
+
+        if resp.is_err() {
+            return Err(TunnelError::WaitRespError(
+                "wait ping response timeout".to_owned(),
+            ));
+        }
+
+        if resp.as_ref().unwrap().is_err() {
+            return Err(resp.unwrap().err().unwrap());
+        }
+
+        Ok(now.elapsed().as_micros())
+    }
+
+    fn start_pingpong(&mut self) {
+        let mut sink = self.tunnel.pin_sink();
+        let my_node_id = self.my_node_id;
+        let peer_id = self.get_peer_id();
+        let receiver = self.ctrl_resp_sender.subscribe();
+        let close_event_sender = self.close_event_sender.clone().unwrap();
+        let conn_id = self.conn_id;
+        let latency_stats = self.latency_stats.clone();
+
+        self.tasks.spawn(async move {
+            //sleep 1s
+            tokio::time::sleep(Duration::from_secs(1)).await;
+            loop {
+                let mut receiver = receiver.resubscribe();
+                if let Ok(lat) =
+                    Self::do_pingpong_once(my_node_id, peer_id, &mut sink, &mut receiver).await
+                {
+                    log::trace!(
+                        "pingpong latency: {}us, my_node_id: {}, peer_id: {}",
+                        lat,
+                        my_node_id,
+                        peer_id
+                    );
+                    latency_stats.record_latency(lat as u64);
+
+                    tokio::time::sleep(Duration::from_secs(1)).await;
+                } else {
+                    break;
+                }
+            }
+
+            log::warn!(
+                "pingpong task exit, my_node_id: {}, peer_id: {}",
+                my_node_id,
+                peer_id,
+            );
+
+            if let Err(e) = close_event_sender.send(conn_id).await {
+                log::warn!("close event sender error: {:?}", e);
+            }
+
+            Ok(())
+        });
+    }
+
+    fn get_packet_type(mut bytes_item: Bytes) -> PeerConnPacketType {
+        if bytes_item.starts_with(CTRL_REQ_PACKET_PREFIX) {
+            PeerConnPacketType::CtrlReq(bytes_item.split_off(CTRL_REQ_PACKET_PREFIX.len()))
+        } else if bytes_item.starts_with(CTRL_RESP_PACKET_PREFIX) {
+            PeerConnPacketType::CtrlResp(bytes_item.split_off(CTRL_RESP_PACKET_PREFIX.len()))
+        } else {
+            PeerConnPacketType::Data(bytes_item)
+        }
+    }
+
+    fn handle_ctrl_req_packet(
+        bytes_item: Bytes,
+        conn_info: &PeerConnInfo,
+    ) -> Result<Bytes, TunnelError> {
+        let packet = Packet::decode(&bytes_item);
+        match packet.body {
+            packet::ArchivedPacketBody::Ctrl(packet::ArchivedCtrlPacketBody::Ping) => {
+                log::trace!("recv ping packet: {:?}", packet);
+                Ok(Self::build_ctrl_msg(
+                    packet::Packet::new_pong_packet(
+                        conn_info.my_node_id.parse().unwrap(),
+                        conn_info.peer_id.parse().unwrap(),
+                    )
+                    .into(),
+                    false,
+                ))
+            }
+            _ => {
+                log::error!("unexpected packet: {:?}", packet);
+                Err(TunnelError::CommonError("unexpected packet".to_owned()))
+            }
+        }
+    }
+
+    pub fn start_recv_loop(&mut self, packet_recv_chan: PacketRecvChan) {
+        let mut stream = self.tunnel.pin_stream();
+        let mut sink = self.tunnel.pin_sink();
+        let mut sender = PollSender::new(packet_recv_chan.clone());
+        let close_event_sender = self.close_event_sender.clone().unwrap();
+        let conn_id = self.conn_id;
+        let ctrl_sender = self.ctrl_resp_sender.clone();
+        let conn_info = self.get_conn_info();
+        let conn_info_for_instrument = self.get_conn_info();
+
+        self.tasks.spawn(
+            async move {
+                tracing::info!("start recving peer conn packet");
+                while let Some(ret) = stream.next().await {
+                    if ret.is_err() {
+                        tracing::error!(error = ?ret, "peer conn recv error");
+                        if let Err(close_ret) = sink.close().await {
+                            tracing::error!(error = ?close_ret, "peer conn sink close error, ignore it");
+                        }
+                        if let Err(e) = close_event_sender.send(conn_id).await {
+                            tracing::error!(error = ?e, "peer conn close event send error");
+                        }
+                        return Err(ret.err().unwrap());
+                    }
+
+                    match Self::get_packet_type(ret.unwrap().into()) {
+                        PeerConnPacketType::Data(item) => sender.send(item).await.unwrap(),
+                        PeerConnPacketType::CtrlReq(item) => {
+                            let ret = Self::handle_ctrl_req_packet(item, &conn_info).unwrap();
+                            if let Err(e) = sink.send(ret).await {
+                                tracing::error!(?e, "peer conn send req error");
+                            }
+                        }
+                        PeerConnPacketType::CtrlResp(item) => {
+                            if let Err(e) = ctrl_sender.send(item) {
+                                tracing::error!(?e, "peer conn send ctrl resp error");
+                            }
+                        }
+                    }
+                }
+                tracing::info!("end recving peer conn packet");
+                Ok(())
+            }
+            .instrument(
+                tracing::info_span!("peer conn recv loop", conn_info = ?conn_info_for_instrument),
+            ),
+        );
+
+        self.start_pingpong();
+    }
+
+    pub async fn send_msg(&mut self, msg: Bytes) -> Result<(), TunnelError> {
+        self.sink.send(msg).await
+    }
+
+    fn build_ctrl_msg(msg: Bytes, is_req: bool) -> Bytes {
+        let prefix: &'static [u8] = if is_req {
+            CTRL_REQ_PACKET_PREFIX
+        } else {
+            CTRL_RESP_PACKET_PREFIX
+        };
+        let mut new_msg = BytesMut::new();
+        new_msg.reserve(prefix.len() + msg.len());
+        new_msg.extend_from_slice(prefix);
+        new_msg.extend_from_slice(&msg);
+        new_msg.into()
+    }
+
+    pub fn get_peer_id(&self) -> uuid::Uuid {
+        self.info.as_ref().unwrap().my_peer_id
+    }
+
+    pub fn set_close_event_sender(&mut self, sender: mpsc::Sender<uuid::Uuid>) {
+        self.close_event_sender = Some(sender);
+    }
+
+    pub fn get_stats(&self) -> PeerConnStats {
+        PeerConnStats {
+            latency_us: self.latency_stats.get_latency_us(),
+
+            tx_bytes: self.throughput.tx_bytes(),
+            rx_bytes: self.throughput.rx_bytes(),
+
+            tx_packets: self.throughput.tx_packets(),
+            rx_packets: self.throughput.rx_packets(),
+        }
+    }
+
+    pub fn get_conn_info(&self) -> PeerConnInfo {
+        PeerConnInfo {
+            conn_id: self.conn_id.to_string(),
+            my_node_id: self.my_node_id.to_string(),
+            peer_id: self.get_peer_id().to_string(),
+            features: self.info.as_ref().unwrap().features.clone(),
+            tunnel: self.tunnel.info(),
+            stats: Some(self.get_stats()),
+        }
+    }
+}
+
+impl Drop for PeerConn {
+    fn drop(&mut self) {
+        let mut sink = self.tunnel.pin_sink();
+        tokio::spawn(async move {
+            let ret = sink.close().await;
+            tracing::info!(error = ?ret, "peer conn tunnel closed.");
+        });
+        log::info!("peer conn {:?} drop", self.conn_id);
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::sync::Arc;
+
+    use super::*;
+    use crate::common::config_fs::ConfigFs;
+    use crate::common::global_ctx::GlobalCtx;
+    use crate::common::netns::NetNS;
+    use crate::tunnels::tunnel_filter::{PacketRecorderTunnelFilter, TunnelWithFilter};
+
+    #[tokio::test]
+    async fn peer_conn_handshake() {
+        use crate::tunnels::ring_tunnel::create_ring_tunnel_pair;
+        let (c, s) = create_ring_tunnel_pair();
+
+        let c_recorder = Arc::new(PacketRecorderTunnelFilter::new());
+        let s_recorder = Arc::new(PacketRecorderTunnelFilter::new());
+
+        let c = TunnelWithFilter::new(c, c_recorder.clone());
+        let s = TunnelWithFilter::new(s, s_recorder.clone());
+
+        let c_uuid = uuid::Uuid::new_v4();
+        let s_uuid = uuid::Uuid::new_v4();
+
+        let mut c_peer = PeerConn::new(
+            c_uuid,
+            Arc::new(GlobalCtx::new(
+                "c",
+                ConfigFs::new_with_dir("c", "/tmp"),
+                NetNS::new(None),
+            )),
+            Box::new(c),
+        );
+
+        let mut s_peer = PeerConn::new(
+            s_uuid,
+            Arc::new(GlobalCtx::new(
+                "c",
+                ConfigFs::new_with_dir("c", "/tmp"),
+                NetNS::new(None),
+            )),
+            Box::new(s),
+        );
+
+        let (c_ret, s_ret) = tokio::join!(
+            c_peer.do_handshake_as_client(),
+            s_peer.do_handshake_as_server()
+        );
+
+        c_ret.unwrap();
+        s_ret.unwrap();
+
+        assert_eq!(c_recorder.sent.lock().unwrap().len(), 1);
+        assert_eq!(c_recorder.received.lock().unwrap().len(), 1);
+
+        assert_eq!(s_recorder.sent.lock().unwrap().len(), 1);
+        assert_eq!(s_recorder.received.lock().unwrap().len(), 1);
+
+        assert_eq!(c_peer.get_peer_id(), s_uuid);
+        assert_eq!(s_peer.get_peer_id(), c_uuid);
+    }
+}

easytier-core/src/peers/peer_manager.rs

@@ -0,0 +1,539 @@
+use std::{
+    fmt::Debug,
+    net::Ipv4Addr,
+    sync::{atomic::AtomicU8, Arc},
+};
+
+use async_trait::async_trait;
+use futures::{StreamExt, TryFutureExt};
+
+use tokio::{
+    sync::{
+        mpsc::{self, UnboundedReceiver, UnboundedSender},
+        Mutex, RwLock,
+    },
+    task::JoinSet,
+};
+use tokio_stream::wrappers::ReceiverStream;
+use tokio_util::bytes::{Bytes, BytesMut};
+
+use uuid::Uuid;
+
+use crate::{
+    common::{error::Error, global_ctx::ArcGlobalCtx, rkyv_util::extract_bytes_from_archived_vec},
+    peers::{
+        packet::{self},
+        peer_conn::PeerConn,
+        peer_rpc::PeerRpcManagerTransport,
+        route_trait::RouteInterface,
+    },
+    tunnels::{SinkItem, Tunnel, TunnelConnector},
+};
+
+use super::{
+    peer_map::PeerMap,
+    peer_rpc::PeerRpcManager,
+    route_trait::{ArcRoute, Route},
+    PeerId,
+};
+
+struct RpcTransport {
+    my_peer_id: uuid::Uuid,
+    peers: Arc<PeerMap>,
+
+    packet_recv: Mutex<UnboundedReceiver<Bytes>>,
+    peer_rpc_tspt_sender: UnboundedSender<Bytes>,
+
+    route: Arc<Mutex<Option<ArcRoute>>>,
+}
+
+#[async_trait::async_trait]
+impl PeerRpcManagerTransport for RpcTransport {
+    fn my_peer_id(&self) -> Uuid {
+        self.my_peer_id
+    }
+
+    async fn send(&self, msg: Bytes, dst_peer_id: &uuid::Uuid) -> Result<(), Error> {
+        let route = self.route.lock().await;
+        if route.is_none() {
+            log::error!("no route info when send rpc msg");
+            return Err(Error::RouteError("No route info".to_string()));
+        }
+
+        self.peers
+            .send_msg(msg, dst_peer_id, route.as_ref().unwrap().clone())
+            .map_err(|e| e.into())
+            .await
+    }
+
+    async fn recv(&self) -> Result<Bytes, Error> {
+        if let Some(o) = self.packet_recv.lock().await.recv().await {
+            Ok(o)
+        } else {
+            Err(Error::Unknown)
+        }
+    }
+}
+
+#[async_trait::async_trait]
+#[auto_impl::auto_impl(Arc)]
+pub trait PeerPacketFilter {
+    async fn try_process_packet_from_peer(
+        &self,
+        packet: &packet::ArchivedPacket,
+        data: &Bytes,
+    ) -> Option<()>;
+}
+
+#[async_trait::async_trait]
+#[auto_impl::auto_impl(Arc)]
+pub trait NicPacketFilter {
+    async fn try_process_packet_from_nic(&self, data: BytesMut) -> BytesMut;
+}
+
+type BoxPeerPacketFilter = Box<dyn PeerPacketFilter + Send + Sync>;
+type BoxNicPacketFilter = Box<dyn NicPacketFilter + Send + Sync>;
+
+pub struct PeerManager {
+    my_node_id: uuid::Uuid,
+    global_ctx: ArcGlobalCtx,
+    nic_channel: mpsc::Sender<SinkItem>,
+
+    tasks: Arc<Mutex<JoinSet<()>>>,
+
+    packet_recv: Arc<Mutex<Option<mpsc::Receiver<Bytes>>>>,
+
+    peers: Arc<PeerMap>,
+    route: Arc<Mutex<Option<ArcRoute>>>,
+    cur_route_id: AtomicU8,
+
+    peer_rpc_mgr: Arc<PeerRpcManager>,
+    peer_rpc_tspt: Arc<RpcTransport>,
+
+    peer_packet_process_pipeline: Arc<RwLock<Vec<BoxPeerPacketFilter>>>,
+    nic_packet_process_pipeline: Arc<RwLock<Vec<BoxNicPacketFilter>>>,
+}
+
+impl Debug for PeerManager {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("PeerManager")
+            .field("my_node_id", &self.my_node_id)
+            .field("instance_name", &self.global_ctx.inst_name)
+            .field("net_ns", &self.global_ctx.net_ns.name())
+            .field("cur_route_id", &self.cur_route_id)
+            .finish()
+    }
+}
+
+impl PeerManager {
+    pub fn new(global_ctx: ArcGlobalCtx, nic_channel: mpsc::Sender<SinkItem>) -> Self {
+        let (packet_send, packet_recv) = mpsc::channel(100);
+        let peers = Arc::new(PeerMap::new(packet_send.clone()));
+
+        // TODO: remove these because we have impl pipeline processor.
+        let (peer_rpc_tspt_sender, peer_rpc_tspt_recv) = mpsc::unbounded_channel();
+        let rpc_tspt = Arc::new(RpcTransport {
+            my_peer_id: global_ctx.get_id(),
+            peers: peers.clone(),
+            packet_recv: Mutex::new(peer_rpc_tspt_recv),
+            peer_rpc_tspt_sender,
+            route: Arc::new(Mutex::new(None)),
+        });
+
+        PeerManager {
+            my_node_id: global_ctx.get_id(),
+            global_ctx,
+            nic_channel,
+
+            tasks: Arc::new(Mutex::new(JoinSet::new())),
+
+            packet_recv: Arc::new(Mutex::new(Some(packet_recv))),
+
+            peers: peers.clone(),
+            route: Arc::new(Mutex::new(None)),
+            cur_route_id: AtomicU8::new(0),
+
+            peer_rpc_mgr: Arc::new(PeerRpcManager::new(rpc_tspt.clone())),
+            peer_rpc_tspt: rpc_tspt,
+
+            peer_packet_process_pipeline: Arc::new(RwLock::new(Vec::new())),
+            nic_packet_process_pipeline: Arc::new(RwLock::new(Vec::new())),
+        }
+    }
+
+    pub async fn add_client_tunnel(&self, tunnel: Box<dyn Tunnel>) -> Result<(Uuid, Uuid), Error> {
+        let mut peer = PeerConn::new(self.my_node_id, self.global_ctx.clone(), tunnel);
+        peer.do_handshake_as_client().await?;
+        let conn_id = peer.get_conn_id();
+        let peer_id = peer.get_peer_id();
+        self.peers
+            .add_new_peer_conn(peer, self.global_ctx.clone())
+            .await;
+        Ok((peer_id, conn_id))
+    }
+
+    #[tracing::instrument]
+    pub async fn try_connect<C>(&self, mut connector: C) -> Result<(Uuid, Uuid), Error>
+    where
+        C: TunnelConnector + Debug,
+    {
+        let ns = self.global_ctx.net_ns.clone();
+        let t = ns
+            .run_async(|| async move { connector.connect().await })
+            .await?;
+        self.add_client_tunnel(t).await
+    }
+
+    #[tracing::instrument]
+    pub async fn add_tunnel_as_server(&self, tunnel: Box<dyn Tunnel>) -> Result<(), Error> {
+        tracing::info!("add tunnel as server start");
+        let mut peer = PeerConn::new(self.my_node_id, self.global_ctx.clone(), tunnel);
+        peer.do_handshake_as_server().await?;
+        self.peers
+            .add_new_peer_conn(peer, self.global_ctx.clone())
+            .await;
+        tracing::info!("add tunnel as server done");
+        Ok(())
+    }
+
+    async fn start_peer_recv(&self) {
+        let mut recv = ReceiverStream::new(self.packet_recv.lock().await.take().unwrap());
+        let my_node_id = self.my_node_id;
+        let peers = self.peers.clone();
+        let arc_route = self.route.clone();
+        let pipe_line = self.peer_packet_process_pipeline.clone();
+        self.tasks.lock().await.spawn(async move {
+            log::trace!("start_peer_recv");
+            while let Some(ret) = recv.next().await {
+                log::trace!("peer recv a packet...: {:?}", ret);
+                let packet = packet::Packet::decode(&ret);
+                let from_peer_uuid = packet.from_peer.to_uuid();
+                let to_peer_uuid = packet.to_peer.as_ref().unwrap().to_uuid();
+                if to_peer_uuid != my_node_id {
+                    let locked_arc_route = arc_route.lock().await;
+                    if locked_arc_route.is_none() {
+                        log::error!("no route info after recv a packet");
+                        continue;
+                    }
+
+                    let route = locked_arc_route.as_ref().unwrap().clone();
+                    drop(locked_arc_route);
+                    log::trace!(
+                        "need forward: to_peer_uuid: {:?}, my_uuid: {:?}",
+                        to_peer_uuid,
+                        my_node_id
+                    );
+                    let ret = peers
+                        .send_msg(ret.clone(), &to_peer_uuid, route.clone())
+                        .await;
+                    if ret.is_err() {
+                        log::error!(
+                            "forward packet error: {:?}, dst: {:?}, from: {:?}",
+                            ret,
+                            to_peer_uuid,
+                            from_peer_uuid
+                        );
+                    }
+                } else {
+                    let mut processed = false;
+                    for pipeline in pipe_line.read().await.iter().rev() {
+                        if let Some(_) = pipeline.try_process_packet_from_peer(&packet, &ret).await
+                        {
+                            processed = true;
+                            break;
+                        }
+                    }
+                    if !processed {
+                        tracing::error!("unexpected packet: {:?}", ret);
+                    }
+                }
+            }
+            panic!("done_peer_recv");
+        });
+    }
+
+    pub async fn add_packet_process_pipeline(&self, pipeline: BoxPeerPacketFilter) {
+        // newest pipeline will be executed first
+        self.peer_packet_process_pipeline
+            .write()
+            .await
+            .push(pipeline);
+    }
+
+    pub async fn add_nic_packet_process_pipeline(&self, pipeline: BoxNicPacketFilter) {
+        // newest pipeline will be executed first
+        self.nic_packet_process_pipeline
+            .write()
+            .await
+            .push(pipeline);
+    }
+
+    async fn init_packet_process_pipeline(&self) {
+        use packet::ArchivedPacketBody;
+
+        // for tun/tap ip/eth packet.
+        struct NicPacketProcessor {
+            nic_channel: mpsc::Sender<SinkItem>,
+        }
+        #[async_trait::async_trait]
+        impl PeerPacketFilter for NicPacketProcessor {
+            async fn try_process_packet_from_peer(
+                &self,
+                packet: &packet::ArchivedPacket,
+                data: &Bytes,
+            ) -> Option<()> {
+                if let packet::ArchivedPacketBody::Data(x) = &packet.body {
+                    // TODO: use a function to get the body ref directly for zero copy
+                    self.nic_channel
+                        .send(extract_bytes_from_archived_vec(&data, &x.data))
+                        .await
+                        .unwrap();
+                    Some(())
+                } else {
+                    None
+                }
+            }
+        }
+        self.add_packet_process_pipeline(Box::new(NicPacketProcessor {
+            nic_channel: self.nic_channel.clone(),
+        }))
+        .await;
+
+        // for peer manager router packet
+        struct RoutePacketProcessor {
+            route: Arc<Mutex<Option<ArcRoute>>>,
+        }
+        #[async_trait::async_trait]
+        impl PeerPacketFilter for RoutePacketProcessor {
+            async fn try_process_packet_from_peer(
+                &self,
+                packet: &packet::ArchivedPacket,
+                data: &Bytes,
+            ) -> Option<()> {
+                if let ArchivedPacketBody::Ctrl(packet::ArchivedCtrlPacketBody::RoutePacket(
+                    route_packet,
+                )) = &packet.body
+                {
+                    let r = self.route.lock().await;
+                    match r.as_ref() {
+                        Some(x) => {
+                            let x = x.clone();
+                            drop(r);
+                            x.handle_route_packet(
+                                packet.from_peer.to_uuid(),
+                                extract_bytes_from_archived_vec(&data, &route_packet.body),
+                            )
+                            .await;
+                        }
+                        None => {
+                            log::error!("no route info when handle route packet");
+                        }
+                    }
+                    Some(())
+                } else {
+                    None
+                }
+            }
+        }
+        self.add_packet_process_pipeline(Box::new(RoutePacketProcessor {
+            route: self.route.clone(),
+        }))
+        .await;
+
+        // for peer rpc packet
+        struct PeerRpcPacketProcessor {
+            peer_rpc_tspt_sender: UnboundedSender<Bytes>,
+        }
+
+        #[async_trait::async_trait]
+        impl PeerPacketFilter for PeerRpcPacketProcessor {
+            async fn try_process_packet_from_peer(
+                &self,
+                packet: &packet::ArchivedPacket,
+                data: &Bytes,
+            ) -> Option<()> {
+                if let ArchivedPacketBody::Ctrl(packet::ArchivedCtrlPacketBody::TaRpc(..)) =
+                    &packet.body
+                {
+                    self.peer_rpc_tspt_sender.send(data.clone()).unwrap();
+                    Some(())
+                } else {
+                    None
+                }
+            }
+        }
+        self.add_packet_process_pipeline(Box::new(PeerRpcPacketProcessor {
+            peer_rpc_tspt_sender: self.peer_rpc_tspt.peer_rpc_tspt_sender.clone(),
+        }))
+        .await;
+    }
+
+    pub async fn set_route<T>(&self, route: T)
+    where
+        T: Route + Send + Sync + 'static,
+    {
+        struct Interface {
+            my_node_id: uuid::Uuid,
+            peers: Arc<PeerMap>,
+        }
+
+        #[async_trait]
+        impl RouteInterface for Interface {
+            async fn list_peers(&self) -> Vec<PeerId> {
+                self.peers.list_peers_with_conn().await
+            }
+            async fn send_route_packet(
+                &self,
+                msg: Bytes,
+                route_id: u8,
+                dst_peer_id: &PeerId,
+            ) -> Result<(), Error> {
+                self.peers
+                    .send_msg_directly(
+                        packet::Packet::new_route_packet(
+                            self.my_node_id,
+                            *dst_peer_id,
+                            route_id,
+                            &msg,
+                        )
+                        .into(),
+                        dst_peer_id,
+                    )
+                    .await
+            }
+        }
+
+        let my_node_id = self.my_node_id;
+        let route_id = route
+            .open(Box::new(Interface {
+                my_node_id,
+                peers: self.peers.clone(),
+            }))
+            .await
+            .unwrap();
+
+        self.cur_route_id
+            .store(route_id, std::sync::atomic::Ordering::Relaxed);
+        let arc_route: ArcRoute = Arc::new(Box::new(route));
+
+        self.route.lock().await.replace(arc_route.clone());
+
+        self.peer_rpc_tspt
+            .route
+            .lock()
+            .await
+            .replace(arc_route.clone());
+    }
+
+    pub async fn list_routes(&self) -> Vec<easytier_rpc::Route> {
+        let route_info = self.route.lock().await;
+        if route_info.is_none() {
+            return Vec::new();
+        }
+
+        let route = route_info.as_ref().unwrap().clone();
+        drop(route_info);
+        route.list_routes().await
+    }
+
+    async fn run_nic_packet_process_pipeline(&self, mut data: BytesMut) -> BytesMut {
+        for pipeline in self.nic_packet_process_pipeline.read().await.iter().rev() {
+            data = pipeline.try_process_packet_from_nic(data).await;
+        }
+        data
+    }
+
+    pub async fn send_msg(&self, msg: Bytes, dst_peer_id: &PeerId) -> Result<(), Error> {
+        self.peer_rpc_tspt.send(msg, dst_peer_id).await
+    }
+
+    pub async fn send_msg_ipv4(&self, msg: BytesMut, ipv4_addr: Ipv4Addr) -> Result<(), Error> {
+        let route_info = self.route.lock().await;
+        if route_info.is_none() {
+            log::error!("no route info");
+            return Err(Error::RouteError("No route info".to_string()));
+        }
+
+        let route = route_info.as_ref().unwrap().clone();
+        drop(route_info);
+
+        log::trace!(
+            "do send_msg in peer manager, msg: {:?}, ipv4_addr: {}",
+            msg,
+            ipv4_addr
+        );
+
+        match route.get_peer_id_by_ipv4(&ipv4_addr).await {
+            Some(peer_id) => {
+                let msg = self.run_nic_packet_process_pipeline(msg).await;
+                self.peers
+                    .send_msg(
+                        packet::Packet::new_data_packet(self.my_node_id, peer_id, &msg).into(),
+                        &peer_id,
+                        route.clone(),
+                    )
+                    .await?;
+                log::trace!(
+                    "do send_msg in peer manager done, dst_peer_id: {:?}",
+                    peer_id
+                );
+            }
+            None => {
+                log::trace!("no peer id for ipv4: {}", ipv4_addr);
+                return Ok(());
+            }
+        }
+
+        Ok(())
+    }
+
+    async fn run_clean_peer_without_conn_routine(&self) {
+        let peer_map = self.peers.clone();
+        self.tasks.lock().await.spawn(async move {
+            loop {
+                let mut to_remove = vec![];
+
+                for peer_id in peer_map.list_peers().await {
+                    let conns = peer_map.list_peer_conns(&peer_id).await;
+                    if conns.is_none() || conns.as_ref().unwrap().is_empty() {
+                        to_remove.push(peer_id);
+                    }
+                }
+
+                for peer_id in to_remove {
+                    peer_map.close_peer(&peer_id).await.unwrap();
+                }
+
+                tokio::time::sleep(std::time::Duration::from_secs(10)).await;
+            }
+        });
+    }
+
+    pub async fn run(&self) -> Result<(), Error> {
+        self.init_packet_process_pipeline().await;
+        self.start_peer_recv().await;
+        self.peer_rpc_mgr.run();
+        self.run_clean_peer_without_conn_routine().await;
+        Ok(())
+    }
+
+    pub fn get_peer_map(&self) -> Arc<PeerMap> {
+        self.peers.clone()
+    }
+
+    pub fn get_peer_rpc_mgr(&self) -> Arc<PeerRpcManager> {
+        self.peer_rpc_mgr.clone()
+    }
+
+    pub fn my_node_id(&self) -> uuid::Uuid {
+        self.my_node_id
+    }
+
+    pub fn get_global_ctx(&self) -> ArcGlobalCtx {
+        self.global_ctx.clone()
+    }
+
+    pub fn get_nic_channel(&self) -> mpsc::Sender<SinkItem> {
+        self.nic_channel.clone()
+    }
+}

easytier-core/src/peers/peer_map.rs

@@ -0,0 +1,140 @@
+use std::sync::Arc;
+
+use dashmap::DashMap;
+use easytier_rpc::PeerConnInfo;
+use tokio::sync::mpsc;
+use tokio_util::bytes::Bytes;
+
+use crate::{
+    common::{error::Error, global_ctx::ArcGlobalCtx},
+    tunnels::TunnelError,
+};
+
+use super::{peer::Peer, peer_conn::PeerConn, route_trait::ArcRoute, PeerId};
+
+pub struct PeerMap {
+    peer_map: DashMap<PeerId, Arc<Peer>>,
+    packet_send: mpsc::Sender<Bytes>,
+}
+
+impl PeerMap {
+    pub fn new(packet_send: mpsc::Sender<Bytes>) -> Self {
+        PeerMap {
+            peer_map: DashMap::new(),
+            packet_send,
+        }
+    }
+
+    async fn add_new_peer(&self, peer: Peer) {
+        self.peer_map.insert(peer.peer_node_id, Arc::new(peer));
+    }
+
+    pub async fn add_new_peer_conn(&self, peer_conn: PeerConn, global_ctx: ArcGlobalCtx) {
+        let peer_id = peer_conn.get_peer_id();
+        let no_entry = self.peer_map.get(&peer_id).is_none();
+        if no_entry {
+            let new_peer = Peer::new(peer_id, self.packet_send.clone(), global_ctx);
+            new_peer.add_peer_conn(peer_conn).await;
+            self.add_new_peer(new_peer).await;
+        } else {
+            let peer = self.peer_map.get(&peer_id).unwrap().clone();
+            peer.add_peer_conn(peer_conn).await;
+        }
+    }
+
+    fn get_peer_by_id(&self, peer_id: &PeerId) -> Option<Arc<Peer>> {
+        self.peer_map.get(peer_id).map(|v| v.clone())
+    }
+
+    pub async fn send_msg_directly(
+        &self,
+        msg: Bytes,
+        dst_peer_id: &uuid::Uuid,
+    ) -> Result<(), Error> {
+        match self.get_peer_by_id(dst_peer_id) {
+            Some(peer) => {
+                peer.send_msg(msg).await?;
+            }
+            None => {
+                log::error!("no peer for dst_peer_id: {}", dst_peer_id);
+                return Ok(());
+            }
+        }
+
+        Ok(())
+    }
+
+    pub async fn send_msg(
+        &self,
+        msg: Bytes,
+        dst_peer_id: &uuid::Uuid,
+        route: ArcRoute,
+    ) -> Result<(), Error> {
+        // get route info
+        let gateway_peer_id = route.get_next_hop(dst_peer_id).await;
+
+        if gateway_peer_id.is_none() {
+            log::error!("no gateway for dst_peer_id: {}", dst_peer_id);
+            return Ok(());
+        }
+
+        let gateway_peer_id = gateway_peer_id.unwrap();
+        self.send_msg_directly(msg, &gateway_peer_id).await?;
+
+        Ok(())
+    }
+
+    pub async fn list_peers(&self) -> Vec<PeerId> {
+        let mut ret = Vec::new();
+        for item in self.peer_map.iter() {
+            let peer_id = item.key();
+            ret.push(*peer_id);
+        }
+        ret
+    }
+
+    pub async fn list_peers_with_conn(&self) -> Vec<PeerId> {
+        let mut ret = Vec::new();
+        let peers = self.list_peers().await;
+        for peer_id in peers.iter() {
+            let Some(peer) = self.get_peer_by_id(peer_id) else {
+                continue;
+            };
+            if peer.list_peer_conns().await.len() > 0 {
+                ret.push(*peer_id);
+            }
+        }
+        ret
+    }
+
+    pub async fn list_peer_conns(&self, peer_id: &PeerId) -> Option<Vec<PeerConnInfo>> {
+        if let Some(p) = self.get_peer_by_id(peer_id) {
+            Some(p.list_peer_conns().await)
+        } else {
+            return None;
+        }
+    }
+
+    pub async fn close_peer_conn(
+        &self,
+        peer_id: &PeerId,
+        conn_id: &uuid::Uuid,
+    ) -> Result<(), Error> {
+        if let Some(p) = self.get_peer_by_id(peer_id) {
+            p.close_peer_conn(conn_id).await
+        } else {
+            return Err(Error::NotFound);
+        }
+    }
+
+    pub async fn close_peer(&self, peer_id: &PeerId) -> Result<(), TunnelError> {
+        let remove_ret = self.peer_map.remove(peer_id);
+        tracing::info!(
+            ?peer_id,
+            has_old_value = ?remove_ret.is_some(),
+            peer_ref_counter = ?remove_ret.map(|v| Arc::strong_count(&v.1)),
+            "peer is closed"
+        );
+        Ok(())
+    }
+}

easytier-core/src/peers/peer_rpc.rs

@@ -0,0 +1,509 @@
+use std::sync::Arc;
+
+use dashmap::DashMap;
+use futures::{SinkExt, StreamExt};
+use rkyv::Deserialize;
+use tarpc::{server::Channel, transport::channel::UnboundedChannel};
+use tokio::{
+    sync::mpsc::{self, UnboundedSender},
+    task::JoinSet,
+};
+use tokio_util::bytes::Bytes;
+use tracing::Instrument;
+
+use crate::{common::error::Error, peers::packet::Packet};
+
+use super::packet::{CtrlPacketBody, PacketBody};
+
+type PeerRpcServiceId = u32;
+
+#[async_trait::async_trait]
+#[auto_impl::auto_impl(Arc)]
+pub trait PeerRpcManagerTransport: Send + Sync + 'static {
+    fn my_peer_id(&self) -> uuid::Uuid;
+    async fn send(&self, msg: Bytes, dst_peer_id: &uuid::Uuid) -> Result<(), Error>;
+    async fn recv(&self) -> Result<Bytes, Error>;
+}
+
+type PacketSender = UnboundedSender<Packet>;
+
+struct PeerRpcEndPoint {
+    peer_id: uuid::Uuid,
+    packet_sender: PacketSender,
+    tasks: JoinSet<()>,
+}
+
+type PeerRpcEndPointCreator = Box<dyn Fn(uuid::Uuid) -> PeerRpcEndPoint + Send + Sync + 'static>;
+#[derive(Hash, Eq, PartialEq, Clone)]
+struct PeerRpcClientCtxKey(uuid::Uuid, PeerRpcServiceId);
+
+// handle rpc request from one peer
+pub struct PeerRpcManager {
+    service_map: Arc<DashMap<PeerRpcServiceId, PacketSender>>,
+    tasks: JoinSet<()>,
+    tspt: Arc<Box<dyn PeerRpcManagerTransport>>,
+
+    service_registry: Arc<DashMap<PeerRpcServiceId, PeerRpcEndPointCreator>>,
+    peer_rpc_endpoints: Arc<DashMap<(uuid::Uuid, PeerRpcServiceId), PeerRpcEndPoint>>,
+
+    client_resp_receivers: Arc<DashMap<PeerRpcClientCtxKey, PacketSender>>,
+}
+
+impl std::fmt::Debug for PeerRpcManager {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("PeerRpcManager")
+            .field("node_id", &self.tspt.my_peer_id())
+            .finish()
+    }
+}
+
+#[derive(Debug)]
+struct TaRpcPacketInfo {
+    from_peer: uuid::Uuid,
+    to_peer: uuid::Uuid,
+    service_id: PeerRpcServiceId,
+    is_req: bool,
+    content: Vec<u8>,
+}
+
+impl PeerRpcManager {
+    pub fn new(tspt: impl PeerRpcManagerTransport) -> Self {
+        Self {
+            service_map: Arc::new(DashMap::new()),
+            tasks: JoinSet::new(),
+            tspt: Arc::new(Box::new(tspt)),
+
+            service_registry: Arc::new(DashMap::new()),
+            peer_rpc_endpoints: Arc::new(DashMap::new()),
+
+            client_resp_receivers: Arc::new(DashMap::new()),
+        }
+    }
+
+    pub fn run_service<S, Req>(self: &Self, service_id: PeerRpcServiceId, s: S) -> ()
+    where
+        S: tarpc::server::Serve<Req> + Clone + Send + Sync + 'static,
+        Req: Send + 'static + serde::Serialize + for<'a> serde::Deserialize<'a>,
+        S::Resp:
+            Send + std::fmt::Debug + 'static + serde::Serialize + for<'a> serde::Deserialize<'a>,
+        S::Fut: Send + 'static,
+    {
+        let tspt = self.tspt.clone();
+        let creator = Box::new(move |peer_id: uuid::Uuid| {
+            let mut tasks = JoinSet::new();
+            let (packet_sender, mut packet_receiver) = mpsc::unbounded_channel::<Packet>();
+            let (mut client_transport, server_transport) = tarpc::transport::channel::unbounded();
+            let server = tarpc::server::BaseChannel::with_defaults(server_transport);
+
+            let my_peer_id_clone = tspt.my_peer_id();
+            let peer_id_clone = peer_id.clone();
+
+            let o = server.execute(s.clone());
+            tasks.spawn(o);
+
+            let tspt = tspt.clone();
+            tasks.spawn(async move {
+                let mut cur_req_uuid = None;
+                loop {
+                    tokio::select! {
+                        Some(resp) = client_transport.next() => {
+                            tracing::trace!(resp = ?resp, "recv packet from client");
+                            if resp.is_err() {
+                                tracing::warn!(err = ?resp.err(),
+                                    "[PEER RPC MGR] client_transport in server side got channel error, ignore it.");
+                                continue;
+                            }
+                            let resp = resp.unwrap();
+
+                            if cur_req_uuid.is_none() {
+                                tracing::error!("[PEER RPC MGR] cur_req_uuid is none, ignore this resp");
+                                continue;
+                            }
+
+                            let serialized_resp = bincode::serialize(&resp);
+                            if serialized_resp.is_err() {
+                                tracing::error!(error = ?serialized_resp.err(), "serialize resp failed");
+                                continue;
+                            }
+
+                            let msg = Packet::new_tarpc_packet(
+                                tspt.my_peer_id(),
+                                cur_req_uuid.take().unwrap(),
+                                service_id,
+                                false,
+                                serialized_resp.unwrap(),
+                            );
+
+                            if let Err(e) = tspt.send(msg.into(), &peer_id).await {
+                                tracing::error!(error = ?e, peer_id = ?peer_id, service_id = ?service_id, "send resp to peer failed");
+                            }
+                        }
+                        Some(packet) = packet_receiver.recv() => {
+                            let info = Self::parse_rpc_packet(&packet);
+                            if let Err(e) = info {
+                                tracing::error!(error = ?e, packet = ?packet, "parse rpc packet failed");
+                                continue;
+                            }
+                            let info = info.unwrap();
+
+                            assert_eq!(info.service_id, service_id);
+                            cur_req_uuid = Some(packet.from_peer.clone().into());
+
+                            tracing::trace!("recv packet from peer, packet: {:?}", packet);
+
+                            let decoded_ret = bincode::deserialize(&info.content.as_slice());
+                            if let Err(e) = decoded_ret {
+                                tracing::error!(error = ?e, "decode rpc packet failed");
+                                continue;
+                            }
+                            let decoded: tarpc::ClientMessage<Req> = decoded_ret.unwrap();
+
+                            if let Err(e) = client_transport.send(decoded).await {
+                                tracing::error!(error = ?e, "send to req to client transport failed");
+                            }
+                        }
+                        else => {
+                            tracing::warn!("[PEER RPC MGR] service runner destroy, peer_id: {}, service_id: {}", peer_id, service_id);
+                        }
+                    }
+                }
+            }.instrument(tracing::info_span!("service_runner", my_id = ?my_peer_id_clone, peer_id = ?peer_id_clone, service_id = ?service_id)));
+
+            tracing::info!(
+                "[PEER RPC MGR] create new service endpoint for peer {}, service {}",
+                peer_id,
+                service_id
+            );
+
+            return PeerRpcEndPoint {
+                peer_id,
+                packet_sender,
+                tasks,
+            };
+            // let resp = client_transport.next().await;
+        });
+
+        if let Some(_) = self.service_registry.insert(service_id, creator) {
+            panic!(
+                "[PEER RPC MGR] service {} is already registered",
+                service_id
+            );
+        }
+
+        log::info!(
+            "[PEER RPC MGR] register service {} succeed, my_node_id {}",
+            service_id,
+            self.tspt.my_peer_id()
+        )
+    }
+
+    fn parse_rpc_packet(packet: &Packet) -> Result<TaRpcPacketInfo, Error> {
+        match &packet.body {
+            PacketBody::Ctrl(CtrlPacketBody::TaRpc(id, is_req, body)) => Ok(TaRpcPacketInfo {
+                from_peer: packet.from_peer.clone().into(),
+                to_peer: packet.to_peer.clone().unwrap().into(),
+                service_id: *id,
+                is_req: *is_req,
+                content: body.clone(),
+            }),
+            _ => Err(Error::ShellCommandError("invalid packet".to_owned())),
+        }
+    }
+
+    pub fn run(&self) {
+        let tspt = self.tspt.clone();
+        let service_registry = self.service_registry.clone();
+        let peer_rpc_endpoints = self.peer_rpc_endpoints.clone();
+        let client_resp_receivers = self.client_resp_receivers.clone();
+        tokio::spawn(async move {
+            loop {
+                let o = tspt.recv().await.unwrap();
+                let packet = Packet::decode(&o);
+                let packet: Packet = packet.deserialize(&mut rkyv::Infallible).unwrap();
+                let info = Self::parse_rpc_packet(&packet).unwrap();
+
+                if info.is_req {
+                    if !service_registry.contains_key(&info.service_id) {
+                        log::warn!(
+                            "service {} not found, my_node_id: {}",
+                            info.service_id,
+                            tspt.my_peer_id()
+                        );
+                        continue;
+                    }
+
+                    let endpoint = peer_rpc_endpoints
+                        .entry((info.to_peer, info.service_id))
+                        .or_insert_with(|| {
+                            service_registry.get(&info.service_id).unwrap()(info.from_peer)
+                        });
+
+                    endpoint.packet_sender.send(packet).unwrap();
+                } else {
+                    if let Some(a) = client_resp_receivers
+                        .get(&PeerRpcClientCtxKey(info.from_peer, info.service_id))
+                    {
+                        log::trace!("recv resp: {:?}", packet);
+                        if let Err(e) = a.send(packet) {
+                            tracing::error!(error = ?e, "send resp to client failed");
+                        }
+                    } else {
+                        log::warn!("client resp receiver not found, info: {:?}", info);
+                    }
+                }
+            }
+        });
+    }
+
+    #[tracing::instrument(skip(f))]
+    pub async fn do_client_rpc_scoped<CM, Req, RpcRet, Fut>(
+        &self,
+        service_id: PeerRpcServiceId,
+        dst_peer_id: uuid::Uuid,
+        f: impl FnOnce(UnboundedChannel<CM, Req>) -> Fut,
+    ) -> RpcRet
+    where
+        CM: serde::Serialize + for<'a> serde::Deserialize<'a> + Send + Sync + 'static,
+        Req: serde::Serialize + for<'a> serde::Deserialize<'a> + Send + Sync + 'static,
+        Fut: std::future::Future<Output = RpcRet>,
+    {
+        let mut tasks = JoinSet::new();
+        let (packet_sender, mut packet_receiver) = mpsc::unbounded_channel::<Packet>();
+        let (client_transport, server_transport) =
+            tarpc::transport::channel::unbounded::<CM, Req>();
+
+        let (mut server_s, mut server_r) = server_transport.split();
+
+        let tspt = self.tspt.clone();
+        tasks.spawn(async move {
+            while let Some(a) = server_r.next().await {
+                if a.is_err() {
+                    tracing::error!(error = ?a.err(), "channel error");
+                    continue;
+                }
+
+                let a = bincode::serialize(&a.unwrap());
+                if a.is_err() {
+                    tracing::error!(error = ?a.err(), "bincode serialize failed");
+                    continue;
+                }
+
+                let a = Packet::new_tarpc_packet(
+                    tspt.my_peer_id(),
+                    dst_peer_id,
+                    service_id,
+                    true,
+                    a.unwrap(),
+                );
+
+                if let Err(e) = tspt.send(a.into(), &dst_peer_id).await {
+                    tracing::error!(error = ?e, dst_peer_id = ?dst_peer_id, "send to peer failed");
+                }
+            }
+
+            tracing::warn!("[PEER RPC MGR] server trasport read aborted");
+        });
+
+        tasks.spawn(async move {
+            while let Some(packet) = packet_receiver.recv().await {
+                tracing::trace!("tunnel recv: {:?}", packet);
+
+                let info = PeerRpcManager::parse_rpc_packet(&packet);
+                if let Err(e) = info {
+                    tracing::error!(error = ?e, "parse rpc packet failed");
+                    continue;
+                }
+
+                let decoded = bincode::deserialize(&info.unwrap().content.as_slice());
+                if let Err(e) = decoded {
+                    tracing::error!(error = ?e, "decode rpc packet failed");
+                    continue;
+                }
+
+                if let Err(e) = server_s.send(decoded.unwrap()).await {
+                    tracing::error!(error = ?e, "send to rpc server channel failed");
+                }
+            }
+
+            tracing::warn!("[PEER RPC MGR] server packet read aborted");
+        });
+
+        let _insert_ret = self
+            .client_resp_receivers
+            .insert(PeerRpcClientCtxKey(dst_peer_id, service_id), packet_sender);
+
+        f(client_transport).await
+    }
+
+    pub fn my_peer_id(&self) -> uuid::Uuid {
+        self.tspt.my_peer_id()
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use futures::{SinkExt, StreamExt};
+    use tokio_util::bytes::Bytes;
+
+    use crate::{
+        common::error::Error,
+        peers::{
+            peer_rpc::PeerRpcManager,
+            tests::{connect_peer_manager, create_mock_peer_manager, wait_route_appear},
+        },
+        tunnels::{self, ring_tunnel::create_ring_tunnel_pair},
+    };
+
+    use super::PeerRpcManagerTransport;
+
+    #[tarpc::service]
+    pub trait TestRpcService {
+        async fn hello(s: String) -> String;
+    }
+
+    #[derive(Clone)]
+    struct MockService {
+        prefix: String,
+    }
+
+    #[tarpc::server]
+    impl TestRpcService for MockService {
+        async fn hello(self, _: tarpc::context::Context, s: String) -> String {
+            format!("{} {}", self.prefix, s)
+        }
+    }
+
+    #[tokio::test]
+    async fn peer_rpc_basic_test() {
+        struct MockTransport {
+            tunnel: Box<dyn tunnels::Tunnel>,
+            my_peer_id: uuid::Uuid,
+        }
+
+        #[async_trait::async_trait]
+        impl PeerRpcManagerTransport for MockTransport {
+            fn my_peer_id(&self) -> uuid::Uuid {
+                self.my_peer_id
+            }
+            async fn send(&self, msg: Bytes, _dst_peer_id: &uuid::Uuid) -> Result<(), Error> {
+                println!("rpc mgr send: {:?}", msg);
+                self.tunnel.pin_sink().send(msg).await.unwrap();
+                Ok(())
+            }
+            async fn recv(&self) -> Result<Bytes, Error> {
+                let ret = self.tunnel.pin_stream().next().await.unwrap();
+                println!("rpc mgr recv: {:?}", ret);
+                return ret.map(|v| v.freeze()).map_err(|_| Error::Unknown);
+            }
+        }
+
+        let (ct, st) = create_ring_tunnel_pair();
+
+        let server_rpc_mgr = PeerRpcManager::new(MockTransport {
+            tunnel: st,
+            my_peer_id: uuid::Uuid::new_v4(),
+        });
+        server_rpc_mgr.run();
+        let s = MockService {
+            prefix: "hello".to_owned(),
+        };
+        server_rpc_mgr.run_service(1, s.serve());
+
+        let client_rpc_mgr = PeerRpcManager::new(MockTransport {
+            tunnel: ct,
+            my_peer_id: uuid::Uuid::new_v4(),
+        });
+        client_rpc_mgr.run();
+
+        let ret = client_rpc_mgr
+            .do_client_rpc_scoped(1, server_rpc_mgr.my_peer_id(), |c| async {
+                let c = TestRpcServiceClient::new(tarpc::client::Config::default(), c).spawn();
+                let ret = c.hello(tarpc::context::current(), "abc".to_owned()).await;
+                ret
+            })
+            .await;
+
+        println!("ret: {:?}", ret);
+        assert_eq!(ret.unwrap(), "hello abc");
+    }
+
+    #[tokio::test]
+    async fn test_rpc_with_peer_manager() {
+        let peer_mgr_a = create_mock_peer_manager().await;
+        let peer_mgr_b = create_mock_peer_manager().await;
+        connect_peer_manager(peer_mgr_a.clone(), peer_mgr_b.clone()).await;
+        wait_route_appear(peer_mgr_a.clone(), peer_mgr_b.my_node_id())
+            .await
+            .unwrap();
+
+        assert_eq!(peer_mgr_a.get_peer_map().list_peers().await.len(), 1);
+        assert_eq!(
+            peer_mgr_a.get_peer_map().list_peers().await[0],
+            peer_mgr_b.my_node_id()
+        );
+
+        let s = MockService {
+            prefix: "hello".to_owned(),
+        };
+        peer_mgr_b.get_peer_rpc_mgr().run_service(1, s.serve());
+
+        let ip_list = peer_mgr_a
+            .get_peer_rpc_mgr()
+            .do_client_rpc_scoped(1, peer_mgr_b.my_node_id(), |c| async {
+                let c = TestRpcServiceClient::new(tarpc::client::Config::default(), c).spawn();
+                let ret = c.hello(tarpc::context::current(), "abc".to_owned()).await;
+                ret
+            })
+            .await;
+
+        println!("ip_list: {:?}", ip_list);
+        assert_eq!(ip_list.as_ref().unwrap(), "hello abc");
+    }
+
+    #[tokio::test]
+    async fn test_multi_service_with_peer_manager() {
+        let peer_mgr_a = create_mock_peer_manager().await;
+        let peer_mgr_b = create_mock_peer_manager().await;
+        connect_peer_manager(peer_mgr_a.clone(), peer_mgr_b.clone()).await;
+        wait_route_appear(peer_mgr_a.clone(), peer_mgr_b.my_node_id())
+            .await
+            .unwrap();
+
+        assert_eq!(peer_mgr_a.get_peer_map().list_peers().await.len(), 1);
+        assert_eq!(
+            peer_mgr_a.get_peer_map().list_peers().await[0],
+            peer_mgr_b.my_node_id()
+        );
+
+        let s = MockService {
+            prefix: "hello_a".to_owned(),
+        };
+        peer_mgr_b.get_peer_rpc_mgr().run_service(1, s.serve());
+        let b = MockService {
+            prefix: "hello_b".to_owned(),
+        };
+        peer_mgr_b.get_peer_rpc_mgr().run_service(2, b.serve());
+
+        let ip_list = peer_mgr_a
+            .get_peer_rpc_mgr()
+            .do_client_rpc_scoped(1, peer_mgr_b.my_node_id(), |c| async {
+                let c = TestRpcServiceClient::new(tarpc::client::Config::default(), c).spawn();
+                let ret = c.hello(tarpc::context::current(), "abc".to_owned()).await;
+                ret
+            })
+            .await;
+
+        assert_eq!(ip_list.as_ref().unwrap(), "hello_a abc");
+
+        let ip_list = peer_mgr_a
+            .get_peer_rpc_mgr()
+            .do_client_rpc_scoped(2, peer_mgr_b.my_node_id(), |c| async {
+                let c = TestRpcServiceClient::new(tarpc::client::Config::default(), c).spawn();
+                let ret = c.hello(tarpc::context::current(), "abc".to_owned()).await;
+                ret
+            })
+            .await;
+
+        assert_eq!(ip_list.as_ref().unwrap(), "hello_b abc");
+    }
+}

easytier-core/src/peers/rip_route.rs

@@ -0,0 +1,480 @@
+use std::{net::Ipv4Addr, sync::Arc, time::Duration};
+
+use async_trait::async_trait;
+use dashmap::DashMap;
+use easytier_rpc::{NatType, StunInfo};
+use rkyv::{Archive, Deserialize, Serialize};
+use tokio::{sync::Mutex, task::JoinSet};
+use tokio_util::bytes::Bytes;
+use tracing::Instrument;
+use uuid::Uuid;
+
+use crate::{
+    common::{
+        error::Error,
+        global_ctx::ArcGlobalCtx,
+        rkyv_util::{decode_from_bytes, encode_to_bytes},
+        stun::StunInfoCollectorTrait,
+    },
+    peers::{
+        packet::{self, UUID},
+        route_trait::{Route, RouteInterfaceBox},
+        PeerId,
+    },
+};
+
+#[derive(Archive, Deserialize, Serialize, Clone, Debug)]
+#[archive(compare(PartialEq), check_bytes)]
+// Derives can be passed through to the generated type:
+#[archive_attr(derive(Debug))]
+pub struct SyncPeerInfo {
+    // means next hop in route table.
+    pub peer_id: UUID,
+    pub cost: u32,
+    pub ipv4_addr: Option<Ipv4Addr>,
+    pub proxy_cidrs: Vec<String>,
+    pub hostname: Option<String>,
+    pub udp_stun_info: i8,
+}
+
+impl SyncPeerInfo {
+    pub fn new_self(from_peer: UUID, global_ctx: &ArcGlobalCtx) -> Self {
+        SyncPeerInfo {
+            peer_id: from_peer,
+            cost: 0,
+            ipv4_addr: global_ctx.get_ipv4(),
+            proxy_cidrs: global_ctx
+                .get_proxy_cidrs()
+                .iter()
+                .map(|x| x.to_string())
+                .collect(),
+            hostname: global_ctx.get_hostname(),
+            udp_stun_info: global_ctx
+                .get_stun_info_collector()
+                .get_stun_info()
+                .udp_nat_type as i8,
+        }
+    }
+
+    pub fn clone_for_route_table(&self, next_hop: &UUID, cost: u32, from: &Self) -> Self {
+        SyncPeerInfo {
+            peer_id: next_hop.clone(),
+            cost,
+            ipv4_addr: from.ipv4_addr.clone(),
+            proxy_cidrs: from.proxy_cidrs.clone(),
+            hostname: from.hostname.clone(),
+            udp_stun_info: from.udp_stun_info,
+        }
+    }
+}
+
+#[derive(Archive, Deserialize, Serialize, Clone, Debug)]
+#[archive(compare(PartialEq), check_bytes)]
+// Derives can be passed through to the generated type:
+#[archive_attr(derive(Debug))]
+pub struct SyncPeer {
+    pub myself: SyncPeerInfo,
+    pub neighbors: Vec<SyncPeerInfo>,
+}
+
+impl SyncPeer {
+    pub fn new(
+        from_peer: UUID,
+        _to_peer: UUID,
+        neighbors: Vec<SyncPeerInfo>,
+        global_ctx: ArcGlobalCtx,
+    ) -> Self {
+        SyncPeer {
+            myself: SyncPeerInfo::new_self(from_peer, &global_ctx),
+            neighbors,
+        }
+    }
+}
+
+struct SyncPeerFromRemote {
+    packet: SyncPeer,
+    last_update: std::time::Instant,
+}
+
+type SyncPeerFromRemoteMap = Arc<DashMap<uuid::Uuid, SyncPeerFromRemote>>;
+
+#[derive(Clone, Debug)]
+struct RouteTable {
+    route_info: DashMap<uuid::Uuid, SyncPeerInfo>,
+    ipv4_peer_id_map: DashMap<Ipv4Addr, uuid::Uuid>,
+    cidr_peer_id_map: DashMap<cidr::IpCidr, uuid::Uuid>,
+}
+
+impl RouteTable {
+    fn new() -> Self {
+        RouteTable {
+            route_info: DashMap::new(),
+            ipv4_peer_id_map: DashMap::new(),
+            cidr_peer_id_map: DashMap::new(),
+        }
+    }
+
+    fn copy_from(&self, other: &Self) {
+        self.route_info.clear();
+        for item in other.route_info.iter() {
+            let (k, v) = item.pair();
+            self.route_info.insert(*k, v.clone());
+        }
+
+        self.ipv4_peer_id_map.clear();
+        for item in other.ipv4_peer_id_map.iter() {
+            let (k, v) = item.pair();
+            self.ipv4_peer_id_map.insert(*k, *v);
+        }
+
+        self.cidr_peer_id_map.clear();
+        for item in other.cidr_peer_id_map.iter() {
+            let (k, v) = item.pair();
+            self.cidr_peer_id_map.insert(*k, *v);
+        }
+    }
+}
+
+pub struct BasicRoute {
+    my_peer_id: packet::UUID,
+    global_ctx: ArcGlobalCtx,
+    interface: Arc<Mutex<Option<RouteInterfaceBox>>>,
+
+    route_table: Arc<RouteTable>,
+
+    sync_peer_from_remote: SyncPeerFromRemoteMap,
+
+    tasks: Mutex<JoinSet<()>>,
+
+    need_sync_notifier: Arc<tokio::sync::Notify>,
+}
+
+impl BasicRoute {
+    pub fn new(my_peer_id: Uuid, global_ctx: ArcGlobalCtx) -> Self {
+        BasicRoute {
+            my_peer_id: my_peer_id.into(),
+            global_ctx,
+            interface: Arc::new(Mutex::new(None)),
+
+            route_table: Arc::new(RouteTable::new()),
+
+            sync_peer_from_remote: Arc::new(DashMap::new()),
+            tasks: Mutex::new(JoinSet::new()),
+
+            need_sync_notifier: Arc::new(tokio::sync::Notify::new()),
+        }
+    }
+
+    fn update_route_table(
+        my_id: packet::UUID,
+        sync_peer_reqs: SyncPeerFromRemoteMap,
+        route_table: Arc<RouteTable>,
+    ) {
+        tracing::trace!(my_id = ?my_id, route_table = ?route_table, "update route table");
+
+        let new_route_table = Arc::new(RouteTable::new());
+        for item in sync_peer_reqs.iter() {
+            Self::update_route_table_with_req(
+                my_id.clone(),
+                &item.value().packet,
+                new_route_table.clone(),
+            );
+        }
+
+        route_table.copy_from(&new_route_table);
+    }
+
+    fn update_route_table_with_req(
+        my_id: packet::UUID,
+        packet: &SyncPeer,
+        route_table: Arc<RouteTable>,
+    ) {
+        let peer_id = packet.myself.peer_id.clone();
+        let update = |cost: u32, peer_info: &SyncPeerInfo| {
+            let node_id: uuid::Uuid = peer_info.peer_id.clone().into();
+            let ret = route_table
+                .route_info
+                .entry(node_id.clone().into())
+                .and_modify(|info| {
+                    if info.cost > cost {
+                        *info = info.clone_for_route_table(&peer_id, cost, &peer_info);
+                    }
+                })
+                .or_insert(
+                    peer_info
+                        .clone()
+                        .clone_for_route_table(&peer_id, cost, &peer_info),
+                )
+                .value()
+                .clone();
+
+            if ret.cost > 32 {
+                log::error!(
+                    "cost too large: {}, may lost connection, remove it",
+                    ret.cost
+                );
+                route_table.route_info.remove(&node_id);
+            }
+
+            log::trace!(
+                "update route info, to: {:?}, gateway: {:?}, cost: {}, peer: {:?}",
+                node_id,
+                peer_id,
+                cost,
+                &peer_info
+            );
+
+            if let Some(ipv4) = peer_info.ipv4_addr {
+                route_table
+                    .ipv4_peer_id_map
+                    .insert(ipv4.clone(), node_id.clone().into());
+            }
+
+            for cidr in peer_info.proxy_cidrs.iter() {
+                let cidr: cidr::IpCidr = cidr.parse().unwrap();
+                route_table
+                    .cidr_peer_id_map
+                    .insert(cidr, node_id.clone().into());
+            }
+        };
+
+        for neighbor in packet.neighbors.iter() {
+            if neighbor.peer_id == my_id {
+                continue;
+            }
+            update(neighbor.cost + 1, &neighbor);
+            log::trace!("route info: {:?}", neighbor);
+        }
+
+        // add the sender peer to route info
+        update(1, &packet.myself);
+
+        log::trace!("my_id: {:?}, current route table: {:?}", my_id, route_table);
+    }
+
+    async fn send_sync_peer_request(
+        interface: &RouteInterfaceBox,
+        my_peer_id: packet::UUID,
+        global_ctx: ArcGlobalCtx,
+        peer_id: PeerId,
+        route_table: Arc<RouteTable>,
+    ) -> Result<(), Error> {
+        let mut route_info_copy: Vec<SyncPeerInfo> = Vec::new();
+        // copy the route info
+        for item in route_table.route_info.iter() {
+            let (k, v) = item.pair();
+            route_info_copy.push(v.clone().clone_for_route_table(&(*k).into(), v.cost, &v));
+        }
+        let msg = SyncPeer::new(my_peer_id, peer_id.into(), route_info_copy, global_ctx);
+        // TODO: this may exceed the MTU of the tunnel
+        interface
+            .send_route_packet(encode_to_bytes::<_, 4096>(&msg), 1, &peer_id)
+            .await
+    }
+
+    async fn sync_peer_periodically(&self) {
+        let route_table = self.route_table.clone();
+        let global_ctx = self.global_ctx.clone();
+        let my_peer_id = self.my_peer_id.clone();
+        let interface = self.interface.clone();
+        let notifier = self.need_sync_notifier.clone();
+        self.tasks.lock().await.spawn(
+            async move {
+                loop {
+                    let lockd_interface = interface.lock().await;
+                    let interface = lockd_interface.as_ref().unwrap();
+                    let peers = interface.list_peers().await;
+                    for peer in peers.iter() {
+                        let ret = Self::send_sync_peer_request(
+                            interface,
+                            my_peer_id.clone(),
+                            global_ctx.clone(),
+                            *peer,
+                            route_table.clone(),
+                        )
+                        .await;
+
+                        match &ret {
+                            Ok(_) => {
+                                log::trace!("send sync peer request to peer: {}", peer);
+                            }
+                            Err(Error::PeerNoConnectionError(_)) => {
+                                log::trace!("peer {} no connection", peer);
+                            }
+                            Err(e) => {
+                                log::error!(
+                                    "send sync peer request to peer: {} error: {:?}",
+                                    peer,
+                                    e
+                                );
+                            }
+                        };
+                    }
+                    tokio::select! {
+                        _ = notifier.notified() => {
+                            log::trace!("sync peer request triggered by notifier");
+                        }
+                        _ = tokio::time::sleep(Duration::from_secs(1)) => {
+                            log::trace!("sync peer request triggered by timeout");
+                        }
+                    }
+                }
+            }
+            .instrument(
+                tracing::info_span!("sync_peer_periodically", my_id = ?self.my_peer_id, global_ctx = ?self.global_ctx),
+            ),
+        );
+    }
+
+    async fn check_expired_sync_peer_from_remote(&self) {
+        let route_table = self.route_table.clone();
+        let my_peer_id = self.my_peer_id.clone();
+        let sync_peer_from_remote = self.sync_peer_from_remote.clone();
+        let notifier = self.need_sync_notifier.clone();
+        self.tasks.lock().await.spawn(async move {
+            loop {
+                let mut need_update_route = false;
+                let now = std::time::Instant::now();
+                let mut need_remove = Vec::new();
+                for item in sync_peer_from_remote.iter() {
+                    let (k, v) = item.pair();
+                    if now.duration_since(v.last_update).as_secs() > 5 {
+                        need_update_route = true;
+                        need_remove.insert(0, k.clone());
+                    }
+                }
+
+                for k in need_remove.iter() {
+                    log::warn!("remove expired sync peer: {:?}", k);
+                    sync_peer_from_remote.remove(k);
+                }
+
+                if need_update_route {
+                    Self::update_route_table(
+                        my_peer_id.clone(),
+                        sync_peer_from_remote.clone(),
+                        route_table.clone(),
+                    );
+                    notifier.notify_one();
+                }
+
+                tokio::time::sleep(Duration::from_secs(1)).await;
+            }
+        });
+    }
+
+    fn get_peer_id_for_proxy(&self, ipv4: &Ipv4Addr) -> Option<PeerId> {
+        let ipv4 = std::net::IpAddr::V4(*ipv4);
+        for item in self.route_table.cidr_peer_id_map.iter() {
+            let (k, v) = item.pair();
+            if k.contains(&ipv4) {
+                return Some(*v);
+            }
+        }
+        None
+    }
+}
+
+#[async_trait]
+impl Route for BasicRoute {
+    async fn open(&self, interface: RouteInterfaceBox) -> Result<u8, ()> {
+        *self.interface.lock().await = Some(interface);
+        self.sync_peer_periodically().await;
+        self.check_expired_sync_peer_from_remote().await;
+        Ok(1)
+    }
+
+    async fn close(&self) {}
+
+    #[tracing::instrument(skip(self, packet), fields(my_id = ?self.my_peer_id, ctx = ?self.global_ctx))]
+    async fn handle_route_packet(&self, src_peer_id: uuid::Uuid, packet: Bytes) {
+        let packet = decode_from_bytes::<SyncPeer>(&packet).unwrap();
+        let p: SyncPeer = packet.deserialize(&mut rkyv::Infallible).unwrap();
+        let mut updated = true;
+        assert_eq!(packet.myself.peer_id.to_uuid(), src_peer_id);
+        self.sync_peer_from_remote
+            .entry(packet.myself.peer_id.to_uuid())
+            .and_modify(|v| {
+                if v.packet == *packet {
+                    updated = false;
+                } else {
+                    v.packet = p.clone();
+                }
+                v.last_update = std::time::Instant::now();
+            })
+            .or_insert(SyncPeerFromRemote {
+                packet: p.clone(),
+                last_update: std::time::Instant::now(),
+            });
+
+        if updated {
+            Self::update_route_table(
+                self.my_peer_id.clone(),
+                self.sync_peer_from_remote.clone(),
+                self.route_table.clone(),
+            );
+            self.need_sync_notifier.notify_one();
+        }
+    }
+
+    async fn get_peer_id_by_ipv4(&self, ipv4_addr: &Ipv4Addr) -> Option<PeerId> {
+        if let Some(peer_id) = self.route_table.ipv4_peer_id_map.get(ipv4_addr) {
+            return Some(*peer_id);
+        }
+
+        if let Some(peer_id) = self.get_peer_id_for_proxy(ipv4_addr) {
+            return Some(peer_id);
+        }
+
+        log::info!("no peer id for ipv4: {}", ipv4_addr);
+        return None;
+    }
+
+    async fn get_next_hop(&self, dst_peer_id: &PeerId) -> Option<PeerId> {
+        match self.route_table.route_info.get(dst_peer_id) {
+            Some(info) => {
+                return Some(info.peer_id.clone().into());
+            }
+            None => {
+                log::error!("no route info for dst_peer_id: {}", dst_peer_id);
+                return None;
+            }
+        }
+    }
+
+    async fn list_routes(&self) -> Vec<easytier_rpc::Route> {
+        let mut routes = Vec::new();
+
+        let parse_route_info = |real_peer_id: &Uuid, route_info: &SyncPeerInfo| {
+            let mut route = easytier_rpc::Route::default();
+            route.ipv4_addr = if let Some(ipv4_addr) = route_info.ipv4_addr {
+                ipv4_addr.to_string()
+            } else {
+                "".to_string()
+            };
+            route.peer_id = real_peer_id.to_string();
+            route.next_hop_peer_id = Uuid::from(route_info.peer_id.clone()).to_string();
+            route.cost = route_info.cost as i32;
+            route.proxy_cidrs = route_info.proxy_cidrs.clone();
+            route.hostname = if let Some(hostname) = &route_info.hostname {
+                hostname.clone()
+            } else {
+                "".to_string()
+            };
+
+            let mut stun_info = StunInfo::default();
+            if let Ok(udp_nat_type) = NatType::try_from(route_info.udp_stun_info as i32) {
+                stun_info.set_udp_nat_type(udp_nat_type);
+            }
+            route.stun_info = Some(stun_info);
+
+            route
+        };
+
+        self.route_table.route_info.iter().for_each(|item| {
+            routes.push(parse_route_info(item.key(), item.value()));
+        });
+
+        routes
+    }
+}

easytier-core/src/peers/route_trait.rs

@@ -0,0 +1,36 @@
+use std::{net::Ipv4Addr, sync::Arc};
+
+use async_trait::async_trait;
+use tokio_util::bytes::Bytes;
+
+use crate::common::error::Error;
+
+use super::PeerId;
+
+#[async_trait]
+pub trait RouteInterface {
+    async fn list_peers(&self) -> Vec<PeerId>;
+    async fn send_route_packet(
+        &self,
+        msg: Bytes,
+        route_id: u8,
+        dst_peer_id: &PeerId,
+    ) -> Result<(), Error>;
+}
+
+pub type RouteInterfaceBox = Box<dyn RouteInterface + Send + Sync>;
+
+#[async_trait]
+pub trait Route {
+    async fn open(&self, interface: RouteInterfaceBox) -> Result<u8, ()>;
+    async fn close(&self);
+
+    async fn get_peer_id_by_ipv4(&self, ipv4: &Ipv4Addr) -> Option<PeerId>;
+    async fn get_next_hop(&self, peer_id: &PeerId) -> Option<PeerId>;
+
+    async fn handle_route_packet(&self, src_peer_id: PeerId, packet: Bytes);
+
+    async fn list_routes(&self) -> Vec<easytier_rpc::Route>;
+}
+
+pub type ArcRoute = Arc<Box<dyn Route + Send + Sync>>;

easytier-core/src/peers/rpc_service.rs

@@ -0,0 +1,55 @@
+use std::sync::Arc;
+
+use easytier_rpc::peer_manage_rpc_server::PeerManageRpc;
+use easytier_rpc::{ListPeerRequest, ListPeerResponse, ListRouteRequest, ListRouteResponse};
+use tonic::{Request, Response, Status};
+
+use super::peer_manager::PeerManager;
+
+pub struct PeerManagerRpcService {
+    peer_manager: Arc<PeerManager>,
+}
+
+impl PeerManagerRpcService {
+    pub fn new(peer_manager: Arc<PeerManager>) -> Self {
+        PeerManagerRpcService { peer_manager }
+    }
+}
+
+#[tonic::async_trait]
+impl PeerManageRpc for PeerManagerRpcService {
+    async fn list_peer(
+        &self,
+        _request: Request<ListPeerRequest>, // Accept request of type HelloRequest
+    ) -> Result<Response<ListPeerResponse>, Status> {
+        let mut reply = ListPeerResponse::default();
+
+        let peers = self.peer_manager.get_peer_map().list_peers().await;
+        for peer in peers {
+            let mut peer_info = easytier_rpc::PeerInfo::default();
+            peer_info.peer_id = peer.to_string();
+
+            if let Some(conns) = self
+                .peer_manager
+                .get_peer_map()
+                .list_peer_conns(&peer)
+                .await
+            {
+                peer_info.conns = conns;
+            }
+
+            reply.peer_infos.push(peer_info);
+        }
+
+        Ok(Response::new(reply))
+    }
+
+    async fn list_route(
+        &self,
+        _request: Request<ListRouteRequest>, // Accept request of type HelloRequest
+    ) -> Result<Response<ListRouteResponse>, Status> {
+        let mut reply = ListRouteResponse::default();
+        reply.routes = self.peer_manager.list_routes().await;
+        Ok(Response::new(reply))
+    }
+}

easytier-core/src/peers/tests.rs

@@ -0,0 +1,60 @@
+use std::sync::Arc;
+
+use crate::{
+    common::{error::Error, global_ctx::tests::get_mock_global_ctx},
+    peers::rip_route::BasicRoute,
+    tunnels::ring_tunnel::create_ring_tunnel_pair,
+};
+
+use super::peer_manager::PeerManager;
+
+pub async fn create_mock_peer_manager() -> Arc<PeerManager> {
+    let (s, _r) = tokio::sync::mpsc::channel(1000);
+    let peer_mgr = Arc::new(PeerManager::new(get_mock_global_ctx(), s));
+    peer_mgr
+        .set_route(BasicRoute::new(
+            peer_mgr.my_node_id(),
+            peer_mgr.get_global_ctx(),
+        ))
+        .await;
+    peer_mgr.run().await.unwrap();
+    peer_mgr
+}
+
+pub async fn connect_peer_manager(client: Arc<PeerManager>, server: Arc<PeerManager>) {
+    let (a_ring, b_ring) = create_ring_tunnel_pair();
+    let a_mgr_copy = client.clone();
+    tokio::spawn(async move {
+        a_mgr_copy.add_client_tunnel(a_ring).await.unwrap();
+    });
+    let b_mgr_copy = server.clone();
+    tokio::spawn(async move {
+        b_mgr_copy.add_tunnel_as_server(b_ring).await.unwrap();
+    });
+}
+
+pub async fn wait_route_appear_with_cost(
+    peer_mgr: Arc<PeerManager>,
+    node_id: uuid::Uuid,
+    cost: Option<i32>,
+) -> Result<(), Error> {
+    let now = std::time::Instant::now();
+    while now.elapsed().as_secs() < 5 {
+        let route = peer_mgr.list_routes().await;
+        if route.iter().any(|r| {
+            r.peer_id.clone().parse::<uuid::Uuid>().unwrap() == node_id
+                && (cost.is_none() || r.cost == cost.unwrap())
+        }) {
+            return Ok(());
+        }
+        tokio::time::sleep(std::time::Duration::from_millis(50)).await;
+    }
+    return Err(Error::NotFound);
+}
+
+pub async fn wait_route_appear(
+    peer_mgr: Arc<PeerManager>,
+    node_id: uuid::Uuid,
+) -> Result<(), Error> {
+    wait_route_appear_with_cost(peer_mgr, node_id, None).await
+}

easytier-core/src/rpc/cli.rs

@@ -0,0 +1 @@
+tonic::include_proto!("cli"); // The string specified here must match the proto package name

easytier-core/src/rpc/lib.rs

@@ -0,0 +1,4 @@
+pub mod cli;
+pub use cli::*;
+
+pub mod peer;

easytier-core/src/rpc/peer.rs

@@ -0,0 +1,20 @@
+use serde::{Deserialize, Serialize};
+
+#[derive(Clone, PartialEq, Debug, Serialize, Deserialize)]
+pub struct GetIpListResponse {
+    pub public_ipv4: String,
+    pub interface_ipv4s: Vec<String>,
+    pub public_ipv6: String,
+    pub interface_ipv6s: Vec<String>,
+}
+
+impl GetIpListResponse {
+    pub fn new() -> Self {
+        GetIpListResponse {
+            public_ipv4: "".to_string(),
+            interface_ipv4s: vec![],
+            public_ipv6: "".to_string(),
+            interface_ipv6s: vec![],
+        }
+    }
+}

easytier-core/src/tests/mod.rs

@@ -0,0 +1,174 @@
+mod three_node;
+
+pub fn get_guest_veth_name(net_ns: &str) -> &str {
+    Box::leak(format!("veth_{}_g", net_ns).into_boxed_str())
+}
+
+pub fn get_host_veth_name(net_ns: &str) -> &str {
+    Box::leak(format!("veth_{}_h", net_ns).into_boxed_str())
+}
+
+pub fn del_netns(name: &str) {
+    // del veth host
+    let _ = std::process::Command::new("ip")
+        .args(&["link", "del", get_host_veth_name(name)])
+        .output();
+
+    let _ = std::process::Command::new("ip")
+        .args(&["netns", "del", name])
+        .output();
+}
+
+pub fn create_netns(name: &str, ipv4: &str) {
+    // create netns
+    let _ = std::process::Command::new("ip")
+        .args(&["netns", "add", name])
+        .output()
+        .unwrap();
+
+    // set lo up
+    let _ = std::process::Command::new("ip")
+        .args(&["netns", "exec", name, "ip", "link", "set", "lo", "up"])
+        .output()
+        .unwrap();
+
+    let _ = std::process::Command::new("ip")
+        .args(&[
+            "link",
+            "add",
+            get_host_veth_name(name),
+            "type",
+            "veth",
+            "peer",
+            "name",
+            get_guest_veth_name(name),
+        ])
+        .output()
+        .unwrap();
+
+    let _ = std::process::Command::new("ip")
+        .args(&["link", "set", get_guest_veth_name(name), "netns", name])
+        .output()
+        .unwrap();
+
+    let _ = std::process::Command::new("ip")
+        .args(&[
+            "netns",
+            "exec",
+            name,
+            "ip",
+            "link",
+            "set",
+            get_guest_veth_name(name),
+            "up",
+        ])
+        .output()
+        .unwrap();
+
+    let _ = std::process::Command::new("ip")
+        .args(&["link", "set", get_host_veth_name(name), "up"])
+        .output()
+        .unwrap();
+
+    let _ = std::process::Command::new("ip")
+        .args(&[
+            "netns",
+            "exec",
+            name,
+            "ip",
+            "addr",
+            "add",
+            ipv4,
+            "dev",
+            get_guest_veth_name(name),
+        ])
+        .output()
+        .unwrap();
+}
+
+pub fn prepare_bridge(name: &str) {
+    // del bridge with brctl
+    let _ = std::process::Command::new("brctl")
+        .args(&["delbr", name])
+        .output();
+
+    // create new br
+    let _ = std::process::Command::new("brctl")
+        .args(&["addbr", name])
+        .output();
+}
+
+pub fn add_ns_to_bridge(br_name: &str, ns_name: &str) {
+    // use brctl to add ns to bridge
+    let _ = std::process::Command::new("brctl")
+        .args(&["addif", br_name, get_host_veth_name(ns_name)])
+        .output()
+        .unwrap();
+
+    // set bridge up
+    let _ = std::process::Command::new("ip")
+        .args(&["link", "set", br_name, "up"])
+        .output()
+        .unwrap();
+}
+
+pub fn enable_log() {
+    let filter = tracing_subscriber::EnvFilter::builder()
+        .with_default_directive(tracing::level_filters::LevelFilter::INFO.into())
+        .from_env()
+        .unwrap();
+    tracing_subscriber::fmt::fmt()
+        .pretty()
+        .with_env_filter(filter)
+        .init();
+}
+
+fn check_route(ipv4: &str, dst_peer_id: uuid::Uuid, routes: Vec<easytier_rpc::Route>) {
+    let mut found = false;
+    for r in routes.iter() {
+        if r.ipv4_addr == ipv4.to_string() {
+            found = true;
+            assert_eq!(r.peer_id, dst_peer_id.to_string(), "{:?}", routes);
+        }
+    }
+    assert!(found);
+}
+
+async fn wait_proxy_route_appear(
+    mgr: &std::sync::Arc<crate::peers::peer_manager::PeerManager>,
+    ipv4: &str,
+    dst_peer_id: uuid::Uuid,
+    proxy_cidr: &str,
+) {
+    let now = std::time::Instant::now();
+    loop {
+        for r in mgr.list_routes().await.iter() {
+            let r = r;
+            if r.proxy_cidrs.contains(&proxy_cidr.to_owned()) {
+                assert_eq!(r.peer_id, dst_peer_id.to_string());
+                assert_eq!(r.ipv4_addr, ipv4);
+                return;
+            }
+        }
+        if now.elapsed().as_secs() > 5 {
+            panic!("wait proxy route appear timeout");
+        }
+        tokio::time::sleep(tokio::time::Duration::from_millis(100)).await;
+    }
+}
+
+fn set_link_status(net_ns: &str, up: bool) {
+    let _ = std::process::Command::new("ip")
+        .args(&[
+            "netns",
+            "exec",
+            net_ns,
+            "ip",
+            "link",
+            "set",
+            get_guest_veth_name(net_ns),
+            if up { "up" } else { "down" },
+        ])
+        .output()
+        .unwrap();
+}

easytier-core/src/tests/three_node.rs

@@ -0,0 +1,227 @@
+use super::*;
+
+use crate::{
+    common::netns::{NetNS, ROOT_NETNS_NAME},
+    instance::instance::{Instance, InstanceConfigWriter},
+    tunnels::{
+        common::tests::_tunnel_pingpong_netns,
+        ring_tunnel::RingTunnelConnector,
+        tcp_tunnel::{TcpTunnelConnector, TcpTunnelListener},
+        udp_tunnel::UdpTunnelConnector,
+    },
+};
+
+pub fn prepare_linux_namespaces() {
+    del_netns("net_a");
+    del_netns("net_b");
+    del_netns("net_c");
+    del_netns("net_d");
+
+    create_netns("net_a", "10.1.1.1/24");
+    create_netns("net_b", "10.1.1.2/24");
+    create_netns("net_c", "10.1.2.3/24");
+    create_netns("net_d", "10.1.2.4/24");
+
+    prepare_bridge("br_a");
+    prepare_bridge("br_b");
+
+    add_ns_to_bridge("br_a", "net_a");
+    add_ns_to_bridge("br_a", "net_b");
+    add_ns_to_bridge("br_b", "net_c");
+    add_ns_to_bridge("br_b", "net_d");
+}
+
+pub async fn prepare_inst_configs() {
+    InstanceConfigWriter::new("inst1")
+        .set_ns(Some("net_a".into()))
+        .set_addr("10.144.144.1".to_owned());
+
+    InstanceConfigWriter::new("inst2")
+        .set_ns(Some("net_b".into()))
+        .set_addr("10.144.144.2".to_owned());
+
+    InstanceConfigWriter::new("inst3")
+        .set_ns(Some("net_c".into()))
+        .set_addr("10.144.144.3".to_owned());
+}
+
+pub async fn init_three_node(proto: &str) -> Vec<Instance> {
+    log::set_max_level(log::LevelFilter::Info);
+    prepare_linux_namespaces();
+
+    prepare_inst_configs().await;
+
+    let mut inst1 = Instance::new("inst1");
+    let mut inst2 = Instance::new("inst2");
+    let mut inst3 = Instance::new("inst3");
+
+    inst1.run().await.unwrap();
+    inst2.run().await.unwrap();
+    inst3.run().await.unwrap();
+
+    if proto == "tcp" {
+        inst2
+            .get_conn_manager()
+            .add_connector(TcpTunnelConnector::new(
+                "tcp://10.1.1.1:11010".parse().unwrap(),
+            ));
+    } else {
+        inst2
+            .get_conn_manager()
+            .add_connector(UdpTunnelConnector::new(
+                "udp://10.1.1.1:11010".parse().unwrap(),
+            ));
+    }
+
+    inst2
+        .get_conn_manager()
+        .add_connector(RingTunnelConnector::new(
+            format!("ring://{}", inst3.id()).parse().unwrap(),
+        ));
+
+    // wait inst2 have two route.
+    let now = std::time::Instant::now();
+    loop {
+        if inst2.get_peer_manager().list_routes().await.len() == 2 {
+            break;
+        }
+        if now.elapsed().as_secs() > 5 {
+            panic!("wait inst2 have two route timeout");
+        }
+        tokio::time::sleep(tokio::time::Duration::from_millis(100)).await;
+    }
+
+    vec![inst1, inst2, inst3]
+}
+
+#[tokio::test]
+#[serial_test::serial]
+pub async fn basic_three_node_test_tcp() {
+    let insts = init_three_node("tcp").await;
+
+    check_route(
+        "10.144.144.2",
+        insts[1].id(),
+        insts[0].get_peer_manager().list_routes().await,
+    );
+
+    check_route(
+        "10.144.144.3",
+        insts[2].id(),
+        insts[0].get_peer_manager().list_routes().await,
+    );
+}
+
+#[tokio::test]
+#[serial_test::serial]
+pub async fn basic_three_node_test_udp() {
+    let insts = init_three_node("udp").await;
+
+    check_route(
+        "10.144.144.2",
+        insts[1].id(),
+        insts[0].get_peer_manager().list_routes().await,
+    );
+
+    check_route(
+        "10.144.144.3",
+        insts[2].id(),
+        insts[0].get_peer_manager().list_routes().await,
+    );
+}
+
+#[tokio::test]
+#[serial_test::serial]
+pub async fn tcp_proxy_three_node_test() {
+    let insts = init_three_node("tcp").await;
+
+    insts[2]
+        .get_global_ctx()
+        .add_proxy_cidr("10.1.2.0/24".parse().unwrap())
+        .unwrap();
+    assert_eq!(insts[2].get_global_ctx().get_proxy_cidrs().len(), 1);
+
+    wait_proxy_route_appear(
+        &insts[0].get_peer_manager(),
+        "10.144.144.3",
+        insts[2].id(),
+        "10.1.2.0/24",
+    )
+    .await;
+
+    // wait updater
+    tokio::time::sleep(tokio::time::Duration::from_secs(6)).await;
+
+    let tcp_listener = TcpTunnelListener::new("tcp://10.1.2.4:22223".parse().unwrap());
+    let tcp_connector = TcpTunnelConnector::new("tcp://10.1.2.4:22223".parse().unwrap());
+
+    _tunnel_pingpong_netns(
+        tcp_listener,
+        tcp_connector,
+        NetNS::new(Some("net_d".into())),
+        NetNS::new(Some("net_a".into())),
+    )
+    .await;
+}
+
+#[tokio::test]
+#[serial_test::serial]
+pub async fn icmp_proxy_three_node_test() {
+    let insts = init_three_node("tcp").await;
+
+    insts[2]
+        .get_global_ctx()
+        .add_proxy_cidr("10.1.2.0/24".parse().unwrap())
+        .unwrap();
+    assert_eq!(insts[2].get_global_ctx().get_proxy_cidrs().len(), 1);
+
+    wait_proxy_route_appear(
+        &insts[0].get_peer_manager(),
+        "10.144.144.3",
+        insts[2].id(),
+        "10.1.2.0/24",
+    )
+    .await;
+
+    // wait updater
+    tokio::time::sleep(tokio::time::Duration::from_secs(6)).await;
+
+    // send ping with shell in net_a to net_d
+    let _g = NetNS::new(Some(ROOT_NETNS_NAME.to_owned())).guard();
+    let code = tokio::process::Command::new("ip")
+        .args(&[
+            "netns", "exec", "net_a", "ping", "-c", "1", "-W", "1", "10.1.2.4",
+        ])
+        .status()
+        .await
+        .unwrap();
+    assert_eq!(code.code().unwrap(), 0);
+}
+
+#[tokio::test]
+#[serial_test::serial]
+pub async fn proxy_three_node_disconnect_test() {
+    InstanceConfigWriter::new("inst4")
+        .set_ns(Some("net_d".into()))
+        .set_addr("10.144.144.4".to_owned());
+
+    let mut inst4 = Instance::new("inst4");
+    inst4
+        .get_conn_manager()
+        .add_connector(TcpTunnelConnector::new(
+            "tcp://10.1.2.3:11010".parse().unwrap(),
+        ));
+    inst4.run().await.unwrap();
+
+    tokio::spawn(async {
+        loop {
+            tokio::time::sleep(tokio::time::Duration::from_secs(6)).await;
+            set_link_status("net_d", false);
+            tokio::time::sleep(tokio::time::Duration::from_secs(6)).await;
+            set_link_status("net_d", true);
+        }
+    });
+
+    // TODO: add some traffic here, also should check route & peer list
+    tokio::time::sleep(tokio::time::Duration::from_secs(35)).await;
+}

easytier-core/src/tunnels/codec.rs

@@ -0,0 +1,54 @@
+use std::result::Result;
+use tokio::io;
+use tokio_util::{
+    bytes::{BufMut, Bytes, BytesMut},
+    codec::{Decoder, Encoder},
+};
+
+#[derive(Copy, Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Default)]
+pub struct BytesCodec {
+    capacity: usize,
+}
+
+impl BytesCodec {
+    /// Creates a new `BytesCodec` for shipping around raw bytes.
+    pub fn new(capacity: usize) -> BytesCodec {
+        BytesCodec { capacity }
+    }
+}
+
+impl Decoder for BytesCodec {
+    type Item = BytesMut;
+    type Error = io::Error;
+
+    fn decode(&mut self, buf: &mut BytesMut) -> Result<Option<BytesMut>, io::Error> {
+        if !buf.is_empty() {
+            let len = buf.len();
+            let ret = Some(buf.split_to(len));
+            buf.reserve(self.capacity);
+            Ok(ret)
+        } else {
+            Ok(None)
+        }
+    }
+}
+
+impl Encoder<Bytes> for BytesCodec {
+    type Error = io::Error;
+
+    fn encode(&mut self, data: Bytes, buf: &mut BytesMut) -> Result<(), io::Error> {
+        buf.reserve(data.len());
+        buf.put(data);
+        Ok(())
+    }
+}
+
+impl Encoder<BytesMut> for BytesCodec {
+    type Error = io::Error;
+
+    fn encode(&mut self, data: BytesMut, buf: &mut BytesMut) -> Result<(), io::Error> {
+        buf.reserve(data.len());
+        buf.put(data);
+        Ok(())
+    }
+}

easytier-core/src/tunnels/common.rs

@@ -0,0 +1,399 @@
+use std::{
+    collections::VecDeque,
+    net::IpAddr,
+    sync::Arc,
+    task::{ready, Context, Poll},
+};
+
+use async_stream::stream;
+use futures::{Future, FutureExt, Sink, SinkExt, Stream, StreamExt};
+use tokio::{sync::Mutex, time::error::Elapsed};
+
+use std::pin::Pin;
+
+use crate::tunnels::{SinkError, TunnelError};
+
+use super::{DatagramSink, DatagramStream, SinkItem, StreamT, Tunnel, TunnelInfo};
+
+pub struct FramedTunnel<R, W> {
+    read: Arc<Mutex<R>>,
+    write: Arc<Mutex<W>>,
+
+    info: Option<TunnelInfo>,
+}
+
+impl<R, RE, W, WE> FramedTunnel<R, W>
+where
+    R: Stream<Item = Result<StreamT, RE>> + Send + Sync + Unpin + 'static,
+    W: Sink<SinkItem, Error = WE> + Send + Sync + Unpin + 'static,
+    RE: std::error::Error + std::fmt::Debug + Send + Sync + 'static,
+    WE: std::error::Error + std::fmt::Debug + Send + Sync + 'static + From<Elapsed>,
+{
+    pub fn new(read: R, write: W, info: Option<TunnelInfo>) -> Self {
+        FramedTunnel {
+            read: Arc::new(Mutex::new(read)),
+            write: Arc::new(Mutex::new(write)),
+            info,
+        }
+    }
+
+    pub fn new_tunnel_with_info(read: R, write: W, info: TunnelInfo) -> Box<dyn Tunnel> {
+        Box::new(FramedTunnel::new(read, write, Some(info)))
+    }
+
+    pub fn recv_stream(&self) -> impl DatagramStream {
+        let read = self.read.clone();
+        let info = self.info.clone();
+        stream! {
+            loop {
+                let read_ret = read.lock().await.next().await;
+                if read_ret.is_none() {
+                    tracing::info!(?info, "read_ret is none");
+                    yield Err(TunnelError::CommonError("recv stream closed".to_string()));
+                } else {
+                    let read_ret = read_ret.unwrap();
+                    if read_ret.is_err() {
+                        let err = read_ret.err().unwrap();
+                        tracing::info!(?info, "recv stream read error");
+                        yield Err(TunnelError::CommonError(err.to_string()));
+                    } else {
+                        yield Ok(read_ret.unwrap());
+                    }
+                }
+            }
+        }
+    }
+
+    pub fn send_sink(&self) -> impl DatagramSink {
+        struct SendSink<W, WE> {
+            write: Arc<Mutex<W>>,
+            max_buffer_size: usize,
+            sending_buffers: Option<VecDeque<SinkItem>>,
+            send_task:
+                Option<Pin<Box<dyn Future<Output = Result<(), WE>> + Send + Sync + 'static>>>,
+            close_task:
+                Option<Pin<Box<dyn Future<Output = Result<(), WE>> + Send + Sync + 'static>>>,
+        }
+
+        impl<W, WE> SendSink<W, WE>
+        where
+            W: Sink<SinkItem, Error = WE> + Send + Sync + Unpin + 'static,
+            WE: std::error::Error + std::fmt::Debug + Send + Sync + From<Elapsed>,
+        {
+            fn try_send_buffser(
+                &mut self,
+                cx: &mut Context<'_>,
+            ) -> Poll<std::result::Result<(), WE>> {
+                if self.send_task.is_none() {
+                    let mut buffers = self.sending_buffers.take().unwrap();
+                    let tun = self.write.clone();
+                    let send_task = async move {
+                        if buffers.is_empty() {
+                            return Ok(());
+                        }
+
+                        let mut locked_tun = tun.lock_owned().await;
+                        while let Some(buf) = buffers.front() {
+                            log::trace!(
+                                "try_send buffer, len: {:?}, buf: {:?}",
+                                buffers.len(),
+                                &buf
+                            );
+                            let timeout_task = tokio::time::timeout(
+                                std::time::Duration::from_secs(1),
+                                locked_tun.send(buf.clone()),
+                            );
+                            let send_res = timeout_task.await;
+                            let Ok(send_res) = send_res else {
+                                // panic!("send timeout");
+                                let err = send_res.err().unwrap();
+                                return Err(err.into());
+                            };
+                            let Ok(_) = send_res else {
+                                let err = send_res.err().unwrap();
+                                println!("send error: {:?}", err);
+                                return Err(err);
+                            };
+                            buffers.pop_front();
+                        }
+                        return Ok(());
+                    };
+                    self.send_task = Some(Box::pin(send_task));
+                }
+
+                let ret = ready!(self.send_task.as_mut().unwrap().poll_unpin(cx));
+                self.send_task = None;
+                self.sending_buffers = Some(VecDeque::new());
+                return Poll::Ready(ret);
+            }
+        }
+
+        impl<W, WE> Sink<SinkItem> for SendSink<W, WE>
+        where
+            W: Sink<SinkItem, Error = WE> + Send + Sync + Unpin + 'static,
+            WE: std::error::Error + std::fmt::Debug + Send + Sync + From<Elapsed>,
+        {
+            type Error = SinkError;
+
+            fn poll_ready(
+                self: Pin<&mut Self>,
+                cx: &mut std::task::Context<'_>,
+            ) -> Poll<Result<(), Self::Error>> {
+                let self_mut = self.get_mut();
+                let sending_buf = self_mut.sending_buffers.as_ref();
+                // if sending_buffers is None, must already be doing flush
+                if sending_buf.is_none() || sending_buf.unwrap().len() > self_mut.max_buffer_size {
+                    return self_mut.poll_flush_unpin(cx);
+                } else {
+                    return Poll::Ready(Ok(()));
+                }
+            }
+
+            fn start_send(self: Pin<&mut Self>, item: SinkItem) -> Result<(), Self::Error> {
+                assert!(self.send_task.is_none());
+                let self_mut = self.get_mut();
+                self_mut.sending_buffers.as_mut().unwrap().push_back(item);
+                Ok(())
+            }
+
+            fn poll_flush(
+                self: Pin<&mut Self>,
+                cx: &mut std::task::Context<'_>,
+            ) -> Poll<Result<(), Self::Error>> {
+                let self_mut = self.get_mut();
+                let ret = self_mut.try_send_buffser(cx);
+                match ret {
+                    Poll::Ready(Ok(())) => Poll::Ready(Ok(())),
+                    Poll::Ready(Err(e)) => Poll::Ready(Err(SinkError::CommonError(e.to_string()))),
+                    Poll::Pending => {
+                        return Poll::Pending;
+                    }
+                }
+            }
+
+            fn poll_close(
+                self: Pin<&mut Self>,
+                cx: &mut std::task::Context<'_>,
+            ) -> Poll<Result<(), Self::Error>> {
+                let self_mut = self.get_mut();
+                if self_mut.close_task.is_none() {
+                    let tun = self_mut.write.clone();
+                    let close_task = async move {
+                        let mut locked_tun = tun.lock_owned().await;
+                        return locked_tun.close().await;
+                    };
+                    self_mut.close_task = Some(Box::pin(close_task));
+                }
+
+                let ret = ready!(self_mut.close_task.as_mut().unwrap().poll_unpin(cx));
+                self_mut.close_task = None;
+
+                if ret.is_err() {
+                    return Poll::Ready(Err(SinkError::CommonError(
+                        ret.err().unwrap().to_string(),
+                    )));
+                } else {
+                    return Poll::Ready(Ok(()));
+                }
+            }
+        }
+
+        SendSink {
+            write: self.write.clone(),
+            max_buffer_size: 1000,
+            sending_buffers: Some(VecDeque::new()),
+            send_task: None,
+            close_task: None,
+        }
+    }
+}
+
+impl<R, RE, W, WE> Tunnel for FramedTunnel<R, W>
+where
+    R: Stream<Item = Result<StreamT, RE>> + Send + Sync + Unpin + 'static,
+    W: Sink<SinkItem, Error = WE> + Send + Sync + Unpin + 'static,
+    RE: std::error::Error + std::fmt::Debug + Send + Sync + 'static,
+    WE: std::error::Error + std::fmt::Debug + Send + Sync + 'static + From<Elapsed>,
+{
+    fn stream(&self) -> Box<dyn DatagramStream> {
+        Box::new(self.recv_stream())
+    }
+
+    fn sink(&self) -> Box<dyn DatagramSink> {
+        Box::new(self.send_sink())
+    }
+
+    fn info(&self) -> Option<TunnelInfo> {
+        if self.info.is_none() {
+            None
+        } else {
+            Some(self.info.clone().unwrap())
+        }
+    }
+}
+
+pub struct TunnelWithCustomInfo {
+    tunnel: Box<dyn Tunnel>,
+    info: TunnelInfo,
+}
+
+impl TunnelWithCustomInfo {
+    pub fn new(tunnel: Box<dyn Tunnel>, info: TunnelInfo) -> Self {
+        TunnelWithCustomInfo { tunnel, info }
+    }
+}
+
+impl Tunnel for TunnelWithCustomInfo {
+    fn stream(&self) -> Box<dyn DatagramStream> {
+        self.tunnel.stream()
+    }
+
+    fn sink(&self) -> Box<dyn DatagramSink> {
+        self.tunnel.sink()
+    }
+
+    fn info(&self) -> Option<TunnelInfo> {
+        Some(self.info.clone())
+    }
+}
+
+pub(crate) fn get_interface_name_by_ip(local_ip: &IpAddr) -> Option<String> {
+    let ifaces = pnet::datalink::interfaces();
+    for iface in ifaces {
+        for ip in iface.ips {
+            if ip.ip() == *local_ip {
+                return Some(iface.name);
+            }
+        }
+    }
+    None
+}
+
+pub mod tests {
+    use std::time::Instant;
+
+    use futures::SinkExt;
+    use tokio_stream::StreamExt;
+    use tokio_util::bytes::{BufMut, Bytes, BytesMut};
+
+    use crate::{
+        common::netns::NetNS,
+        tunnels::{close_tunnel, TunnelConnector, TunnelListener},
+    };
+
+    pub async fn _tunnel_echo_server(tunnel: Box<dyn super::Tunnel>, once: bool) {
+        let mut recv = Box::into_pin(tunnel.stream());
+        let mut send = Box::into_pin(tunnel.sink());
+
+        while let Some(ret) = recv.next().await {
+            if ret.is_err() {
+                log::trace!("recv error: {:?}", ret.err().unwrap());
+                break;
+            }
+            let res = ret.unwrap();
+            log::trace!("recv a msg, try echo back: {:?}", res);
+            send.send(Bytes::from(res)).await.unwrap();
+            if once {
+                break;
+            }
+        }
+        log::warn!("echo server exit...");
+    }
+
+    pub(crate) async fn _tunnel_pingpong<L, C>(listener: L, connector: C)
+    where
+        L: TunnelListener + Send + Sync + 'static,
+        C: TunnelConnector + Send + Sync + 'static,
+    {
+        _tunnel_pingpong_netns(listener, connector, NetNS::new(None), NetNS::new(None)).await
+    }
+
+    pub(crate) async fn _tunnel_pingpong_netns<L, C>(
+        mut listener: L,
+        mut connector: C,
+        l_netns: NetNS,
+        c_netns: NetNS,
+    ) where
+        L: TunnelListener + Send + Sync + 'static,
+        C: TunnelConnector + Send + Sync + 'static,
+    {
+        l_netns
+            .run_async(|| async {
+                listener.listen().await.unwrap();
+            })
+            .await;
+
+        let lis = tokio::spawn(async move {
+            let ret = listener.accept().await.unwrap();
+            assert_eq!(
+                ret.info().unwrap().local_addr,
+                listener.local_url().to_string()
+            );
+            _tunnel_echo_server(ret, false).await
+        });
+
+        let tunnel = c_netns.run_async(|| connector.connect()).await.unwrap();
+
+        assert_eq!(
+            tunnel.info().unwrap().remote_addr,
+            connector.remote_url().to_string()
+        );
+
+        let mut send = tunnel.pin_sink();
+        let mut recv = tunnel.pin_stream();
+        let send_data = Bytes::from("abc");
+        send.send(send_data).await.unwrap();
+        let ret = tokio::time::timeout(tokio::time::Duration::from_secs(1), recv.next())
+            .await
+            .unwrap()
+            .unwrap()
+            .unwrap();
+        println!("echo back: {:?}", ret);
+        assert_eq!(ret, Bytes::from("abc"));
+
+        close_tunnel(&tunnel).await.unwrap();
+
+        if connector.remote_url().scheme() == "udp" {
+            lis.abort();
+        } else {
+            // lis should finish in 1 second
+            let ret = tokio::time::timeout(tokio::time::Duration::from_secs(1), lis).await;
+            assert!(ret.is_ok());
+        }
+    }
+
+    pub(crate) async fn _tunnel_bench<L, C>(mut listener: L, mut connector: C)
+    where
+        L: TunnelListener + Send + Sync + 'static,
+        C: TunnelConnector + Send + Sync + 'static,
+    {
+        listener.listen().await.unwrap();
+
+        let lis = tokio::spawn(async move {
+            let ret = listener.accept().await.unwrap();
+            _tunnel_echo_server(ret, false).await
+        });
+
+        let tunnel = connector.connect().await.unwrap();
+
+        let mut send = tunnel.pin_sink();
+        let mut recv = tunnel.pin_stream();
+
+        // prepare a 4k buffer with random data
+        let mut send_buf = BytesMut::new();
+        for _ in 0..64 {
+            send_buf.put_i128(rand::random::<i128>());
+        }
+
+        let now = Instant::now();
+        let mut count = 0;
+        while now.elapsed().as_secs() < 3 {
+            send.send(send_buf.clone().freeze()).await.unwrap();
+            let _ = recv.next().await.unwrap().unwrap();
+            count += 1;
+        }
+        println!("bps: {}", (count / 1024) * 4 / now.elapsed().as_secs());
+
+        lis.abort();
+    }
+}

easytier-core/src/tunnels/mod.rs

@@ -0,0 +1,159 @@
+pub mod codec;
+pub mod common;
+pub mod ring_tunnel;
+pub mod stats;
+pub mod tcp_tunnel;
+pub mod tunnel_filter;
+pub mod udp_tunnel;
+
+use std::{fmt::Debug, net::SocketAddr, pin::Pin, sync::Arc};
+
+use async_trait::async_trait;
+use easytier_rpc::TunnelInfo;
+use futures::{Sink, SinkExt, Stream};
+
+use thiserror::Error;
+use tokio_util::bytes::{Bytes, BytesMut};
+
+#[derive(Error, Debug)]
+pub enum TunnelError {
+    #[error("Error: {0}")]
+    CommonError(String),
+    #[error("io error")]
+    IOError(#[from] std::io::Error),
+    #[error("wait resp error")]
+    WaitRespError(String),
+    #[error("Connect Error: {0}")]
+    ConnectError(String),
+    #[error("Invalid Protocol: {0}")]
+    InvalidProtocol(String),
+    #[error("Invalid Addr: {0}")]
+    InvalidAddr(String),
+    #[error("Tun Error: {0}")]
+    TunError(String),
+    #[error("timeout")]
+    Timeout(#[from] tokio::time::error::Elapsed),
+}
+
+pub type StreamT = BytesMut;
+pub type StreamItem = Result<StreamT, TunnelError>;
+pub type SinkItem = Bytes;
+pub type SinkError = TunnelError;
+
+pub trait DatagramStream: Stream<Item = StreamItem> + Send + Sync {}
+impl<T> DatagramStream for T where T: Stream<Item = StreamItem> + Send + Sync {}
+pub trait DatagramSink: Sink<SinkItem, Error = SinkError> + Send + Sync {}
+impl<T> DatagramSink for T where T: Sink<SinkItem, Error = SinkError> + Send + Sync {}
+
+#[auto_impl::auto_impl(Box, Arc)]
+pub trait Tunnel: Send + Sync {
+    fn stream(&self) -> Box<dyn DatagramStream>;
+    fn sink(&self) -> Box<dyn DatagramSink>;
+
+    fn pin_stream(&self) -> Pin<Box<dyn DatagramStream>> {
+        Box::into_pin(self.stream())
+    }
+
+    fn pin_sink(&self) -> Pin<Box<dyn DatagramSink>> {
+        Box::into_pin(self.sink())
+    }
+
+    fn info(&self) -> Option<TunnelInfo>;
+}
+
+pub async fn close_tunnel(t: &Box<dyn Tunnel>) -> Result<(), TunnelError> {
+    t.pin_sink().close().await
+}
+
+#[auto_impl::auto_impl(Arc)]
+pub trait TunnelConnCounter: 'static + Send + Sync + Debug {
+    fn get(&self) -> u32;
+}
+
+#[async_trait]
+#[auto_impl::auto_impl(Box)]
+pub trait TunnelListener: Send + Sync {
+    async fn listen(&mut self) -> Result<(), TunnelError>;
+    async fn accept(&mut self) -> Result<Box<dyn Tunnel>, TunnelError>;
+    fn local_url(&self) -> url::Url;
+    fn get_conn_counter(&self) -> Arc<Box<dyn TunnelConnCounter>> {
+        #[derive(Debug)]
+        struct FakeTunnelConnCounter {}
+        impl TunnelConnCounter for FakeTunnelConnCounter {
+            fn get(&self) -> u32 {
+                0
+            }
+        }
+        Arc::new(Box::new(FakeTunnelConnCounter {}))
+    }
+}
+
+#[async_trait]
+#[auto_impl::auto_impl(Box)]
+pub trait TunnelConnector {
+    async fn connect(&mut self) -> Result<Box<dyn Tunnel>, TunnelError>;
+    fn remote_url(&self) -> url::Url;
+    fn set_bind_addrs(&mut self, _addrs: Vec<SocketAddr>) {}
+}
+
+pub fn build_url_from_socket_addr(addr: &String, scheme: &str) -> url::Url {
+    url::Url::parse(format!("{}://{}", scheme, addr).as_str()).unwrap()
+}
+
+impl std::fmt::Debug for dyn Tunnel {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("Tunnel")
+            .field("info", &self.info())
+            .finish()
+    }
+}
+
+impl std::fmt::Debug for dyn TunnelConnector + Sync + Send {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("TunnelConnector")
+            .field("remote_url", &self.remote_url())
+            .finish()
+    }
+}
+
+impl std::fmt::Debug for dyn TunnelListener {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("TunnelListener")
+            .field("local_url", &self.local_url())
+            .finish()
+    }
+}
+
+pub(crate) trait FromUrl {
+    fn from_url(url: url::Url) -> Result<Self, TunnelError>
+    where
+        Self: Sized;
+}
+
+pub(crate) fn check_scheme_and_get_socket_addr<T>(
+    url: &url::Url,
+    scheme: &str,
+) -> Result<T, TunnelError>
+where
+    T: FromUrl,
+{
+    if url.scheme() != scheme {
+        return Err(TunnelError::InvalidProtocol(url.scheme().to_string()));
+    }
+
+    Ok(T::from_url(url.clone())?)
+}
+
+impl FromUrl for SocketAddr {
+    fn from_url(url: url::Url) -> Result<Self, TunnelError> {
+        Ok(url.socket_addrs(|| None)?.pop().unwrap())
+    }
+}
+
+impl FromUrl for uuid::Uuid {
+    fn from_url(url: url::Url) -> Result<Self, TunnelError> {
+        let o = url.host_str().unwrap();
+        let o = uuid::Uuid::parse_str(o).map_err(|e| TunnelError::InvalidAddr(e.to_string()))?;
+        Ok(o)
+    }
+}

easytier-core/src/tunnels/ring_tunnel.rs

@@ -0,0 +1,391 @@
+use std::{
+    collections::HashMap,
+    sync::{atomic::AtomicBool, Arc},
+    task::Poll,
+};
+
+use async_stream::stream;
+use crossbeam_queue::ArrayQueue;
+
+use async_trait::async_trait;
+use futures::Sink;
+use once_cell::sync::Lazy;
+use tokio::sync::{Mutex, Notify};
+
+use futures::FutureExt;
+use tokio_util::bytes::BytesMut;
+use uuid::Uuid;
+
+use crate::tunnels::{SinkError, SinkItem};
+
+use super::{
+    build_url_from_socket_addr, check_scheme_and_get_socket_addr, DatagramSink, DatagramStream,
+    Tunnel, TunnelConnector, TunnelError, TunnelInfo, TunnelListener,
+};
+
+static RING_TUNNEL_CAP: usize = 1000;
+
+pub struct RingTunnel {
+    id: Uuid,
+    ring: Arc<ArrayQueue<SinkItem>>,
+    consume_notify: Arc<Notify>,
+    produce_notify: Arc<Notify>,
+    closed: Arc<AtomicBool>,
+}
+
+impl RingTunnel {
+    pub fn new(cap: usize) -> Self {
+        RingTunnel {
+            id: Uuid::new_v4(),
+            ring: Arc::new(ArrayQueue::new(cap)),
+            consume_notify: Arc::new(Notify::new()),
+            produce_notify: Arc::new(Notify::new()),
+            closed: Arc::new(AtomicBool::new(false)),
+        }
+    }
+
+    pub fn new_with_id(id: Uuid, cap: usize) -> Self {
+        let mut ret = Self::new(cap);
+        ret.id = id;
+        ret
+    }
+
+    fn recv_stream(&self) -> impl DatagramStream {
+        let ring = self.ring.clone();
+        let produce_notify = self.produce_notify.clone();
+        let consume_notify = self.consume_notify.clone();
+        let closed = self.closed.clone();
+        let id = self.id;
+        stream! {
+            loop {
+                if closed.load(std::sync::atomic::Ordering::Relaxed) {
+                    log::warn!("ring recv tunnel {:?} closed", id);
+                    yield Err(TunnelError::CommonError("Closed".to_owned()));
+                }
+                match ring.pop() {
+                    Some(v) => {
+                        let mut out = BytesMut::new();
+                        out.extend_from_slice(&v);
+                        consume_notify.notify_one();
+                        log::trace!("id: {}, recv buffer, len: {:?}, buf: {:?}", id, v.len(), &v);
+                        yield Ok(out);
+                    },
+                    None => {
+                        log::trace!("waiting recv buffer, id: {}", id);
+                        produce_notify.notified().await;
+                    }
+                }
+            }
+        }
+    }
+
+    fn send_sink(&self) -> impl DatagramSink {
+        let ring = self.ring.clone();
+        let produce_notify = self.produce_notify.clone();
+        let consume_notify = self.consume_notify.clone();
+        let closed = self.closed.clone();
+        let id = self.id;
+
+        // type T = RingTunnel;
+
+        use tokio::task::JoinHandle;
+
+        struct T {
+            ring: RingTunnel,
+            wait_consume_task: Option<JoinHandle<()>>,
+        }
+
+        impl T {
+            fn wait_ring_consume(
+                self: std::pin::Pin<&mut Self>,
+                cx: &mut std::task::Context<'_>,
+                expected_size: usize,
+            ) -> std::task::Poll<()> {
+                let self_mut = self.get_mut();
+                if self_mut.ring.ring.len() <= expected_size {
+                    return Poll::Ready(());
+                }
+                if self_mut.wait_consume_task.is_none() {
+                    let id = self_mut.ring.id;
+                    let consume_notify = self_mut.ring.consume_notify.clone();
+                    let ring = self_mut.ring.ring.clone();
+                    let task = async move {
+                        log::trace!(
+                            "waiting ring consume done, expected_size: {}, id: {}",
+                            expected_size,
+                            id
+                        );
+                        while ring.len() > expected_size {
+                            consume_notify.notified().await;
+                        }
+                        log::trace!(
+                            "ring consume done, expected_size: {}, id: {}",
+                            expected_size,
+                            id
+                        );
+                    };
+                    self_mut.wait_consume_task = Some(tokio::spawn(task));
+                }
+                let task = self_mut.wait_consume_task.as_mut().unwrap();
+                match task.poll_unpin(cx) {
+                    Poll::Ready(_) => {
+                        self_mut.wait_consume_task = None;
+                        Poll::Ready(())
+                    }
+                    Poll::Pending => Poll::Pending,
+                }
+            }
+        }
+
+        impl Sink<SinkItem> for T {
+            type Error = SinkError;
+
+            fn poll_ready(
+                self: std::pin::Pin<&mut Self>,
+                cx: &mut std::task::Context<'_>,
+            ) -> std::task::Poll<Result<(), Self::Error>> {
+                let expected_size = self.ring.ring.capacity() - 1;
+                match self.wait_ring_consume(cx, expected_size) {
+                    Poll::Ready(_) => Poll::Ready(Ok(())),
+                    Poll::Pending => Poll::Pending,
+                }
+            }
+
+            fn start_send(
+                self: std::pin::Pin<&mut Self>,
+                item: SinkItem,
+            ) -> Result<(), Self::Error> {
+                log::trace!("id: {}, send buffer, buf: {:?}", self.ring.id, &item);
+                self.ring.ring.push(item).unwrap();
+                self.ring.produce_notify.notify_one();
+                Ok(())
+            }
+
+            fn poll_flush(
+                self: std::pin::Pin<&mut Self>,
+                _cx: &mut std::task::Context<'_>,
+            ) -> std::task::Poll<Result<(), Self::Error>> {
+                Poll::Ready(Ok(()))
+            }
+
+            fn poll_close(
+                self: std::pin::Pin<&mut Self>,
+                _cx: &mut std::task::Context<'_>,
+            ) -> std::task::Poll<Result<(), Self::Error>> {
+                self.ring
+                    .closed
+                    .store(true, std::sync::atomic::Ordering::Relaxed);
+                log::warn!("ring tunnel send {:?} closed", self.ring.id);
+                self.ring.produce_notify.notify_one();
+                Poll::Ready(Ok(()))
+            }
+        }
+
+        T {
+            ring: RingTunnel {
+                id,
+                ring,
+                consume_notify,
+                produce_notify,
+                closed,
+            },
+            wait_consume_task: None,
+        }
+    }
+}
+
+struct Connection {
+    client: RingTunnel,
+    server: RingTunnel,
+    connect_notify: Arc<Notify>,
+}
+
+impl Tunnel for RingTunnel {
+    fn stream(&self) -> Box<dyn DatagramStream> {
+        Box::new(self.recv_stream())
+    }
+
+    fn sink(&self) -> Box<dyn DatagramSink> {
+        Box::new(self.send_sink())
+    }
+
+    fn info(&self) -> Option<TunnelInfo> {
+        None
+        // Some(TunnelInfo {
+        //     tunnel_type: "ring".to_owned(),
+        //     local_addr: format!("ring://{}", self.id),
+        //     remote_addr: format!("ring://{}", self.id),
+        // })
+    }
+}
+
+static CONNECTION_MAP: Lazy<Arc<Mutex<HashMap<uuid::Uuid, Arc<Connection>>>>> =
+    Lazy::new(|| Arc::new(Mutex::new(HashMap::new())));
+
+#[derive(Debug)]
+pub struct RingTunnelListener {
+    listerner_addr: url::Url,
+}
+
+impl RingTunnelListener {
+    pub fn new(key: url::Url) -> Self {
+        RingTunnelListener {
+            listerner_addr: key,
+        }
+    }
+}
+struct ConnectionForServer {
+    conn: Arc<Connection>,
+}
+
+impl Tunnel for ConnectionForServer {
+    fn stream(&self) -> Box<dyn DatagramStream> {
+        Box::new(self.conn.server.recv_stream())
+    }
+
+    fn sink(&self) -> Box<dyn DatagramSink> {
+        Box::new(self.conn.client.send_sink())
+    }
+
+    fn info(&self) -> Option<TunnelInfo> {
+        Some(TunnelInfo {
+            tunnel_type: "ring".to_owned(),
+            local_addr: build_url_from_socket_addr(&self.conn.server.id.into(), "ring").into(),
+            remote_addr: build_url_from_socket_addr(&self.conn.client.id.into(), "ring").into(),
+        })
+    }
+}
+
+struct ConnectionForClient {
+    conn: Arc<Connection>,
+}
+
+impl Tunnel for ConnectionForClient {
+    fn stream(&self) -> Box<dyn DatagramStream> {
+        Box::new(self.conn.client.recv_stream())
+    }
+
+    fn sink(&self) -> Box<dyn DatagramSink> {
+        Box::new(self.conn.server.send_sink())
+    }
+
+    fn info(&self) -> Option<TunnelInfo> {
+        Some(TunnelInfo {
+            tunnel_type: "ring".to_owned(),
+            local_addr: build_url_from_socket_addr(&self.conn.client.id.into(), "ring").into(),
+            remote_addr: build_url_from_socket_addr(&self.conn.server.id.into(), "ring").into(),
+        })
+    }
+}
+
+impl RingTunnelListener {
+    async fn add_connection(listener_addr: uuid::Uuid) {
+        CONNECTION_MAP.lock().await.insert(
+            listener_addr.clone(),
+            Arc::new(Connection {
+                client: RingTunnel::new(RING_TUNNEL_CAP),
+                server: RingTunnel::new_with_id(listener_addr.clone(), RING_TUNNEL_CAP),
+                connect_notify: Arc::new(Notify::new()),
+            }),
+        );
+    }
+
+    fn get_addr(&self) -> Result<uuid::Uuid, TunnelError> {
+        check_scheme_and_get_socket_addr::<Uuid>(&self.listerner_addr, "ring")
+    }
+}
+
+#[async_trait]
+impl TunnelListener for RingTunnelListener {
+    async fn listen(&mut self) -> Result<(), TunnelError> {
+        log::info!("listen new conn of key: {}", self.listerner_addr);
+        Self::add_connection(self.get_addr()?).await;
+        Ok(())
+    }
+
+    async fn accept(&mut self) -> Result<Box<dyn Tunnel>, TunnelError> {
+        log::info!("waiting accept new conn of key: {}", self.listerner_addr);
+        let val = CONNECTION_MAP
+            .lock()
+            .await
+            .get(&self.get_addr()?)
+            .unwrap()
+            .clone();
+        val.connect_notify.notified().await;
+        CONNECTION_MAP.lock().await.remove(&self.get_addr()?);
+        Self::add_connection(self.get_addr()?).await;
+        log::info!("accept new conn of key: {}", self.listerner_addr);
+        Ok(Box::new(ConnectionForServer { conn: val }))
+    }
+
+    fn local_url(&self) -> url::Url {
+        self.listerner_addr.clone()
+    }
+}
+
+pub struct RingTunnelConnector {
+    remote_addr: url::Url,
+}
+
+impl RingTunnelConnector {
+    pub fn new(remote_addr: url::Url) -> Self {
+        RingTunnelConnector { remote_addr }
+    }
+}
+
+#[async_trait]
+impl TunnelConnector for RingTunnelConnector {
+    async fn connect(&mut self) -> Result<Box<dyn Tunnel>, super::TunnelError> {
+        let val = CONNECTION_MAP
+            .lock()
+            .await
+            .get(&check_scheme_and_get_socket_addr::<Uuid>(
+                &self.remote_addr,
+                "ring",
+            )?)
+            .unwrap()
+            .clone();
+        val.connect_notify.notify_one();
+        log::info!("connecting");
+        Ok(Box::new(ConnectionForClient { conn: val }))
+    }
+
+    fn remote_url(&self) -> url::Url {
+        self.remote_addr.clone()
+    }
+}
+
+pub fn create_ring_tunnel_pair() -> (Box<dyn Tunnel>, Box<dyn Tunnel>) {
+    let conn = Arc::new(Connection {
+        client: RingTunnel::new(RING_TUNNEL_CAP),
+        server: RingTunnel::new(RING_TUNNEL_CAP),
+        connect_notify: Arc::new(Notify::new()),
+    });
+    (
+        Box::new(ConnectionForServer { conn: conn.clone() }),
+        Box::new(ConnectionForClient { conn: conn }),
+    )
+}
+
+#[cfg(test)]
+mod tests {
+    use crate::tunnels::common::tests::{_tunnel_bench, _tunnel_pingpong};
+
+    use super::*;
+
+    #[tokio::test]
+    async fn ring_pingpong() {
+        let id: url::Url = format!("ring://{}", Uuid::new_v4()).parse().unwrap();
+        let listener = RingTunnelListener::new(id.clone());
+        let connector = RingTunnelConnector::new(id.clone());
+        _tunnel_pingpong(listener, connector).await
+    }
+
+    #[tokio::test]
+    async fn ring_bench() {
+        let id: url::Url = format!("ring://{}", Uuid::new_v4()).parse().unwrap();
+        let listener = RingTunnelListener::new(id.clone());
+        let connector = RingTunnelConnector::new(id);
+        _tunnel_bench(listener, connector).await
+    }
+}

easytier-core/src/tunnels/stats.rs

@@ -0,0 +1,101 @@
+use std::sync::atomic::{AtomicU32, AtomicU64};
+
+pub struct WindowLatency {
+    latency_us_window: Vec<AtomicU64>,
+    latency_us_window_index: AtomicU32,
+    latency_us_window_size: AtomicU32,
+}
+
+impl WindowLatency {
+    pub fn new(window_size: u32) -> Self {
+        Self {
+            latency_us_window: (0..window_size).map(|_| AtomicU64::new(0)).collect(),
+            latency_us_window_index: AtomicU32::new(0),
+            latency_us_window_size: AtomicU32::new(window_size),
+        }
+    }
+
+    pub fn record_latency(&self, latency_us: u64) {
+        let index = self
+            .latency_us_window_index
+            .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
+        let index = index
+            % self
+                .latency_us_window_size
+                .load(std::sync::atomic::Ordering::Relaxed);
+        self.latency_us_window[index as usize]
+            .store(latency_us, std::sync::atomic::Ordering::Relaxed);
+    }
+
+    pub fn get_latency_us(&self) -> u64 {
+        let window_size = self
+            .latency_us_window_size
+            .load(std::sync::atomic::Ordering::Relaxed);
+        let mut sum = 0;
+        let mut count = 0;
+        for i in 0..window_size {
+            let latency_us =
+                self.latency_us_window[i as usize].load(std::sync::atomic::Ordering::Relaxed);
+            if latency_us > 0 {
+                sum += latency_us;
+                count += 1;
+            }
+        }
+
+        if count == 0 {
+            0
+        } else {
+            sum / count
+        }
+    }
+}
+
+pub struct Throughput {
+    tx_bytes: AtomicU64,
+    rx_bytes: AtomicU64,
+
+    tx_packets: AtomicU64,
+    rx_packets: AtomicU64,
+}
+
+impl Throughput {
+    pub fn new() -> Self {
+        Self {
+            tx_bytes: AtomicU64::new(0),
+            rx_bytes: AtomicU64::new(0),
+
+            tx_packets: AtomicU64::new(0),
+            rx_packets: AtomicU64::new(0),
+        }
+    }
+
+    pub fn tx_bytes(&self) -> u64 {
+        self.tx_bytes.load(std::sync::atomic::Ordering::Relaxed)
+    }
+
+    pub fn rx_bytes(&self) -> u64 {
+        self.rx_bytes.load(std::sync::atomic::Ordering::Relaxed)
+    }
+
+    pub fn tx_packets(&self) -> u64 {
+        self.tx_packets.load(std::sync::atomic::Ordering::Relaxed)
+    }
+
+    pub fn rx_packets(&self) -> u64 {
+        self.rx_packets.load(std::sync::atomic::Ordering::Relaxed)
+    }
+
+    pub fn record_tx_bytes(&self, bytes: u64) {
+        self.tx_bytes
+            .fetch_add(bytes, std::sync::atomic::Ordering::Relaxed);
+        self.tx_packets
+            .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
+    }
+
+    pub fn record_rx_bytes(&self, bytes: u64) {
+        self.rx_bytes
+            .fetch_add(bytes, std::sync::atomic::Ordering::Relaxed);
+        self.rx_packets
+            .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
+    }
+}

easytier-core/src/tunnels/tcp_tunnel.rs

@@ -0,0 +1,284 @@
+use std::net::SocketAddr;
+
+use async_trait::async_trait;
+use futures::{stream::FuturesUnordered, StreamExt};
+use tokio::net::{TcpListener, TcpSocket, TcpStream};
+use tokio_util::codec::{FramedRead, FramedWrite, LengthDelimitedCodec};
+
+use super::{
+    check_scheme_and_get_socket_addr, common::FramedTunnel, Tunnel, TunnelInfo, TunnelListener,
+};
+
+#[derive(Debug)]
+pub struct TcpTunnelListener {
+    addr: url::Url,
+    listener: Option<TcpListener>,
+}
+
+impl TcpTunnelListener {
+    pub fn new(addr: url::Url) -> Self {
+        TcpTunnelListener {
+            addr,
+            listener: None,
+        }
+    }
+}
+
+#[async_trait]
+impl TunnelListener for TcpTunnelListener {
+    async fn listen(&mut self) -> Result<(), super::TunnelError> {
+        let addr = check_scheme_and_get_socket_addr::<SocketAddr>(&self.addr, "tcp")?;
+
+        let socket = if addr.is_ipv4() {
+            TcpSocket::new_v4()?
+        } else {
+            TcpSocket::new_v6()?
+        };
+
+        socket.set_reuseaddr(true)?;
+        #[cfg(all(unix, not(target_os = "solaris"), not(target_os = "illumos")))]
+        socket.set_reuseport(true)?;
+        socket.bind(addr)?;
+
+        self.listener = Some(socket.listen(1024)?);
+        Ok(())
+    }
+
+    async fn accept(&mut self) -> Result<Box<dyn Tunnel>, super::TunnelError> {
+        let listener = self.listener.as_ref().unwrap();
+        let (stream, _) = listener.accept().await?;
+        stream.set_nodelay(true).unwrap();
+        let info = TunnelInfo {
+            tunnel_type: "tcp".to_owned(),
+            local_addr: self.local_url().into(),
+            remote_addr: super::build_url_from_socket_addr(&stream.peer_addr()?.to_string(), "tcp")
+                .into(),
+        };
+
+        let (r, w) = tokio::io::split(stream);
+        Ok(FramedTunnel::new_tunnel_with_info(
+            FramedRead::new(r, LengthDelimitedCodec::new()),
+            FramedWrite::new(w, LengthDelimitedCodec::new()),
+            info,
+        ))
+    }
+
+    fn local_url(&self) -> url::Url {
+        self.addr.clone()
+    }
+}
+
+fn get_tunnel_with_tcp_stream(
+    stream: TcpStream,
+    remote_url: url::Url,
+) -> Result<Box<dyn Tunnel>, super::TunnelError> {
+    stream.set_nodelay(true).unwrap();
+
+    let info = TunnelInfo {
+        tunnel_type: "tcp".to_owned(),
+        local_addr: super::build_url_from_socket_addr(&stream.local_addr()?.to_string(), "tcp")
+            .into(),
+        remote_addr: remote_url.into(),
+    };
+
+    let (r, w) = tokio::io::split(stream);
+    Ok(Box::new(FramedTunnel::new_tunnel_with_info(
+        FramedRead::new(r, LengthDelimitedCodec::new()),
+        FramedWrite::new(w, LengthDelimitedCodec::new()),
+        info,
+    )))
+}
+
+#[derive(Debug)]
+pub struct TcpTunnelConnector {
+    addr: url::Url,
+
+    bind_addrs: Vec<SocketAddr>,
+}
+
+impl TcpTunnelConnector {
+    pub fn new(addr: url::Url) -> Self {
+        TcpTunnelConnector {
+            addr,
+            bind_addrs: vec![],
+        }
+    }
+
+    async fn connect_with_default_bind(&mut self) -> Result<Box<dyn Tunnel>, super::TunnelError> {
+        tracing::info!(addr = ?self.addr, "connect tcp start");
+        let addr = check_scheme_and_get_socket_addr::<SocketAddr>(&self.addr, "tcp")?;
+        let stream = TcpStream::connect(addr).await?;
+        tracing::info!(addr = ?self.addr, "connect tcp succ");
+        return get_tunnel_with_tcp_stream(stream, self.addr.clone().into());
+    }
+
+    async fn connect_with_custom_bind(
+        &mut self,
+        is_ipv4: bool,
+    ) -> Result<Box<dyn Tunnel>, super::TunnelError> {
+        let mut futures = FuturesUnordered::new();
+        let dst_addr = check_scheme_and_get_socket_addr::<SocketAddr>(&self.addr, "tcp")?;
+
+        for bind_addr in self.bind_addrs.iter() {
+            let socket = if is_ipv4 {
+                TcpSocket::new_v4()?
+            } else {
+                TcpSocket::new_v6()?
+            };
+            socket.set_reuseaddr(true)?;
+
+            #[cfg(all(unix, not(target_os = "solaris"), not(target_os = "illumos")))]
+            socket.set_reuseport(true)?;
+
+            socket.bind(*bind_addr)?;
+            // linux does not use interface of bind_addr to send packet, so we need to bind device
+            // mac can handle this with bind correctly
+            #[cfg(any(target_os = "android", target_os = "fuchsia", target_os = "linux"))]
+            if let Some(dev_name) = super::common::get_interface_name_by_ip(&bind_addr.ip()) {
+                tracing::trace!(dev_name = ?dev_name, "bind device");
+                socket.bind_device(Some(dev_name.as_bytes()))?;
+            }
+            futures.push(socket.connect(dst_addr.clone()));
+        }
+
+        let Some(ret) = futures.next().await else {
+            return Err(super::TunnelError::CommonError(
+                "join connect futures failed".to_owned(),
+            ));
+        };
+
+        return get_tunnel_with_tcp_stream(ret?, self.addr.clone().into());
+    }
+}
+
+#[async_trait]
+impl super::TunnelConnector for TcpTunnelConnector {
+    async fn connect(&mut self) -> Result<Box<dyn Tunnel>, super::TunnelError> {
+        if self.bind_addrs.is_empty() {
+            self.connect_with_default_bind().await
+        } else if self.bind_addrs[0].is_ipv4() {
+            self.connect_with_custom_bind(true).await
+        } else {
+            self.connect_with_custom_bind(false).await
+        }
+    }
+
+    fn remote_url(&self) -> url::Url {
+        self.addr.clone()
+    }
+    fn set_bind_addrs(&mut self, addrs: Vec<SocketAddr>) {
+        self.bind_addrs = addrs;
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use futures::SinkExt;
+
+    use crate::tunnels::{
+        common::tests::{_tunnel_bench, _tunnel_pingpong},
+        TunnelConnector,
+    };
+
+    use super::*;
+
+    #[tokio::test]
+    async fn tcp_pingpong() {
+        let listener = TcpTunnelListener::new("tcp://0.0.0.0:11011".parse().unwrap());
+        let connector = TcpTunnelConnector::new("tcp://127.0.0.1:11011".parse().unwrap());
+        _tunnel_pingpong(listener, connector).await
+    }
+
+    #[tokio::test]
+    async fn tcp_bench() {
+        let listener = TcpTunnelListener::new("tcp://0.0.0.0:11012".parse().unwrap());
+        let connector = TcpTunnelConnector::new("tcp://127.0.0.1:11012".parse().unwrap());
+        _tunnel_bench(listener, connector).await
+    }
+
+    #[tokio::test]
+    async fn tcp_bench_with_bind() {
+        let listener = TcpTunnelListener::new("tcp://127.0.0.1:11013".parse().unwrap());
+        let mut connector = TcpTunnelConnector::new("tcp://127.0.0.1:11013".parse().unwrap());
+        connector.set_bind_addrs(vec!["127.0.0.1:0".parse().unwrap()]);
+        _tunnel_pingpong(listener, connector).await
+    }
+
+    #[tokio::test]
+    #[should_panic]
+    async fn tcp_bench_with_bind_fail() {
+        let listener = TcpTunnelListener::new("tcp://127.0.0.1:11014".parse().unwrap());
+        let mut connector = TcpTunnelConnector::new("tcp://127.0.0.1:11014".parse().unwrap());
+        connector.set_bind_addrs(vec!["10.0.0.1:0".parse().unwrap()]);
+        _tunnel_pingpong(listener, connector).await
+    }
+
+    // test slow send lock in framed tunnel
+    #[tokio::test]
+    async fn tcp_multiple_sender_and_slow_receiver() {
+        // console_subscriber::init();
+        let mut listener = TcpTunnelListener::new("tcp://127.0.0.1:11014".parse().unwrap());
+        let mut connector = TcpTunnelConnector::new("tcp://127.0.0.1:11014".parse().unwrap());
+
+        listener.listen().await.unwrap();
+        let t1 = tokio::spawn(async move {
+            let t = listener.accept().await.unwrap();
+            let mut stream = t.pin_stream();
+
+            let now = tokio::time::Instant::now();
+
+            while let Some(Ok(_)) = stream.next().await {
+                tokio::time::sleep(tokio::time::Duration::from_millis(100)).await;
+                if now.elapsed().as_secs() > 5 {
+                    break;
+                }
+            }
+
+            tracing::info!("t1 exit");
+        });
+
+        let tunnel = connector.connect().await.unwrap();
+        let mut sink1 = tunnel.pin_sink();
+        let t2 = tokio::spawn(async move {
+            for i in 0..1000000 {
+                let a = sink1.send(b"hello".to_vec().into()).await;
+                if a.is_err() {
+                    tracing::info!(?a, "t2 exit with err");
+                    break;
+                }
+
+                if i % 5000 == 0 {
+                    tracing::info!(i, "send2 1000");
+                }
+            }
+
+            tracing::info!("t2 exit");
+        });
+
+        let mut sink2 = tunnel.pin_sink();
+        let t3 = tokio::spawn(async move {
+            for i in 0..1000000 {
+                let a = sink2.send(b"hello".to_vec().into()).await;
+                if a.is_err() {
+                    tracing::info!(?a, "t3 exit with err");
+                    break;
+                }
+
+                if i % 5000 == 0 {
+                    tracing::info!(i, "send2 1000");
+                }
+            }
+
+            tracing::info!("t3 exit");
+        });
+
+        let t4 = tokio::spawn(async move {
+            tokio::time::sleep(tokio::time::Duration::from_secs(5)).await;
+            tracing::info!("closing");
+            let close_ret = tunnel.pin_sink().close().await;
+            tracing::info!("closed {:?}", close_ret);
+        });
+
+        let _ = tokio::join!(t1, t2, t3, t4);
+    }
+}

easytier-core/src/tunnels/tunnel_filter.rs

@@ -0,0 +1,228 @@
+use std::{
+    sync::Arc,
+    task::{Context, Poll},
+};
+
+use easytier_rpc::TunnelInfo;
+use futures::{Sink, SinkExt, Stream, StreamExt};
+
+use self::stats::Throughput;
+
+use super::*;
+use crate::tunnels::{DatagramSink, DatagramStream, SinkError, SinkItem, StreamItem, Tunnel};
+
+pub trait TunnelFilter {
+    fn before_send(&self, data: SinkItem) -> Result<SinkItem, SinkError>;
+    fn after_received(&self, data: StreamItem) -> Result<BytesMut, TunnelError>;
+}
+
+pub struct TunnelWithFilter<T, F> {
+    inner: T,
+    filter: Arc<F>,
+}
+
+impl<T, F> Tunnel for TunnelWithFilter<T, F>
+where
+    T: Tunnel + Send + Sync + 'static,
+    F: TunnelFilter + Send + Sync + 'static,
+{
+    fn sink(&self) -> Box<dyn DatagramSink> {
+        struct SinkWrapper<F> {
+            sink: Pin<Box<dyn DatagramSink>>,
+            filter: Arc<F>,
+        }
+        impl<F> Sink<SinkItem> for SinkWrapper<F>
+        where
+            F: TunnelFilter + Send + Sync + 'static,
+        {
+            type Error = SinkError;
+
+            fn poll_ready(
+                self: Pin<&mut Self>,
+                cx: &mut Context<'_>,
+            ) -> Poll<Result<(), Self::Error>> {
+                self.get_mut().sink.poll_ready_unpin(cx)
+            }
+
+            fn start_send(self: Pin<&mut Self>, item: SinkItem) -> Result<(), Self::Error> {
+                let item = self.filter.before_send(item)?;
+                self.get_mut().sink.start_send_unpin(item)
+            }
+
+            fn poll_flush(
+                self: Pin<&mut Self>,
+                cx: &mut Context<'_>,
+            ) -> Poll<Result<(), Self::Error>> {
+                self.get_mut().sink.poll_flush_unpin(cx)
+            }
+
+            fn poll_close(
+                self: Pin<&mut Self>,
+                cx: &mut Context<'_>,
+            ) -> Poll<Result<(), Self::Error>> {
+                self.get_mut().sink.poll_close_unpin(cx)
+            }
+        }
+
+        Box::new(SinkWrapper {
+            sink: self.inner.pin_sink(),
+            filter: self.filter.clone(),
+        })
+    }
+
+    fn stream(&self) -> Box<dyn DatagramStream> {
+        struct StreamWrapper<F> {
+            stream: Pin<Box<dyn DatagramStream>>,
+            filter: Arc<F>,
+        }
+        impl<F> Stream for StreamWrapper<F>
+        where
+            F: TunnelFilter + Send + Sync + 'static,
+        {
+            type Item = StreamItem;
+
+            fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
+                let self_mut = self.get_mut();
+                match self_mut.stream.poll_next_unpin(cx) {
+                    Poll::Ready(Some(ret)) => {
+                        Poll::Ready(Some(self_mut.filter.after_received(ret)))
+                    }
+                    Poll::Ready(None) => Poll::Ready(None),
+                    Poll::Pending => Poll::Pending,
+                }
+            }
+        }
+
+        Box::new(StreamWrapper {
+            stream: self.inner.pin_stream(),
+            filter: self.filter.clone(),
+        })
+    }
+
+    fn info(&self) -> Option<TunnelInfo> {
+        self.inner.info()
+    }
+}
+
+impl<T, F> TunnelWithFilter<T, F>
+where
+    T: Tunnel + Send + Sync + 'static,
+    F: TunnelFilter + Send + Sync + 'static,
+{
+    pub fn new(inner: T, filter: Arc<F>) -> Self {
+        Self { inner, filter }
+    }
+}
+
+pub struct PacketRecorderTunnelFilter {
+    pub received: Arc<std::sync::Mutex<Vec<Bytes>>>,
+    pub sent: Arc<std::sync::Mutex<Vec<Bytes>>>,
+}
+
+impl TunnelFilter for PacketRecorderTunnelFilter {
+    fn before_send(&self, data: SinkItem) -> Result<SinkItem, SinkError> {
+        self.received.lock().unwrap().push(data.clone());
+        Ok(data)
+    }
+
+    fn after_received(&self, data: StreamItem) -> Result<BytesMut, TunnelError> {
+        match data {
+            Ok(v) => {
+                self.sent.lock().unwrap().push(v.clone().into());
+                Ok(v)
+            }
+            Err(e) => Err(e),
+        }
+    }
+}
+
+impl PacketRecorderTunnelFilter {
+    pub fn new() -> Self {
+        Self {
+            received: Arc::new(std::sync::Mutex::new(Vec::new())),
+            sent: Arc::new(std::sync::Mutex::new(Vec::new())),
+        }
+    }
+}
+
+pub struct StatsRecorderTunnelFilter {
+    throughput: Arc<Throughput>,
+}
+
+impl TunnelFilter for StatsRecorderTunnelFilter {
+    fn before_send(&self, data: SinkItem) -> Result<SinkItem, SinkError> {
+        self.throughput.record_tx_bytes(data.len() as u64);
+        Ok(data)
+    }
+
+    fn after_received(&self, data: StreamItem) -> Result<BytesMut, TunnelError> {
+        match data {
+            Ok(v) => {
+                self.throughput.record_rx_bytes(v.len() as u64);
+                Ok(v)
+            }
+            Err(e) => Err(e),
+        }
+    }
+}
+
+impl StatsRecorderTunnelFilter {
+    pub fn new() -> Self {
+        Self {
+            throughput: Arc::new(Throughput::new()),
+        }
+    }
+
+    pub fn get_throughput(&self) -> Arc<Throughput> {
+        self.throughput.clone()
+    }
+}
+
+#[macro_export]
+macro_rules! define_tunnel_filter_chain {
+    ($type_name:ident $(, $field_name:ident = $filter_type:ty)+) => (
+        pub struct $type_name {
+            $($field_name: std::sync::Arc<$filter_type>,)+
+        }
+
+        impl $type_name {
+            pub fn new() -> Self {
+                Self {
+                    $($field_name: std::sync::Arc::new(<$filter_type>::new()),)+
+                }
+            }
+
+            pub fn wrap_tunnel(&self, tunnel: impl Tunnel + 'static) -> impl Tunnel {
+                $(
+                    let tunnel = crate::tunnels::tunnel_filter::TunnelWithFilter::new(tunnel, self.$field_name.clone());
+                )+
+                tunnel
+            }
+        }
+    )
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::tunnels::ring_tunnel::RingTunnel;
+    #[tokio::test]
+    async fn test_nested_filter() {
+        define_tunnel_filter_chain!(
+            Filter,
+            a = PacketRecorderTunnelFilter,
+            b = PacketRecorderTunnelFilter,
+            c = PacketRecorderTunnelFilter
+        );
+
+        let filter = Filter::new();
+        let tunnel = filter.wrap_tunnel(RingTunnel::new(1));
+
+        let mut s = tunnel.pin_sink();
+        s.send(Bytes::from("hello")).await.unwrap();
+
+        assert_eq!(1, filter.a.received.lock().unwrap().len());
+        assert_eq!(1, filter.b.received.lock().unwrap().len());
+        assert_eq!(1, filter.c.received.lock().unwrap().len());
+    }
+}

easytier-core/src/tunnels/udp_tunnel.rs

@@ -0,0 +1,574 @@
+use std::{fmt::Debug, pin::Pin, sync::Arc};
+
+use async_trait::async_trait;
+use dashmap::DashMap;
+use easytier_rpc::TunnelInfo;
+use futures::{stream::FuturesUnordered, SinkExt, StreamExt};
+use rkyv::{Archive, Deserialize, Serialize};
+use std::net::SocketAddr;
+use tokio::{net::UdpSocket, sync::Mutex, task::JoinSet};
+use tokio_util::{
+    bytes::{Buf, Bytes, BytesMut},
+    udp::UdpFramed,
+};
+use tracing::Instrument;
+
+use crate::{
+    common::rkyv_util::{self, encode_to_bytes},
+    tunnels::{build_url_from_socket_addr, close_tunnel, TunnelConnCounter, TunnelConnector},
+};
+
+use super::{
+    codec::BytesCodec,
+    common::{FramedTunnel, TunnelWithCustomInfo},
+    ring_tunnel::create_ring_tunnel_pair,
+    DatagramSink, DatagramStream, Tunnel, TunnelListener,
+};
+
+pub const UDP_DATA_MTU: usize = 2500;
+
+#[derive(Archive, Deserialize, Serialize, Debug)]
+#[archive(compare(PartialEq), check_bytes)]
+// Derives can be passed through to the generated type:
+#[archive_attr(derive(Debug))]
+pub enum UdpPacketPayload {
+    Syn,
+    Sack,
+    HolePunch(Vec<u8>),
+    Data(Vec<u8>),
+}
+
+#[derive(Archive, Deserialize, Serialize, Debug)]
+#[archive(compare(PartialEq), check_bytes)]
+#[archive_attr(derive(Debug))]
+pub struct UdpPacket {
+    pub conn_id: u32,
+    pub payload: UdpPacketPayload,
+}
+
+impl UdpPacket {
+    pub fn new_data_packet(conn_id: u32, data: Vec<u8>) -> Self {
+        Self {
+            conn_id,
+            payload: UdpPacketPayload::Data(data),
+        }
+    }
+
+    pub fn new_hole_punch_packet(data: Vec<u8>) -> Self {
+        Self {
+            conn_id: 0,
+            payload: UdpPacketPayload::HolePunch(data),
+        }
+    }
+
+    pub fn new_syn_packet(conn_id: u32) -> Self {
+        Self {
+            conn_id,
+            payload: UdpPacketPayload::Syn,
+        }
+    }
+
+    pub fn new_sack_packet(conn_id: u32) -> Self {
+        Self {
+            conn_id,
+            payload: UdpPacketPayload::Sack,
+        }
+    }
+}
+
+fn try_get_data_payload(mut buf: BytesMut, conn_id: u32) -> Option<BytesMut> {
+    let Ok(udp_packet) = rkyv_util::decode_from_bytes_checked::<UdpPacket>(&buf) else {
+        tracing::warn!(?buf, "udp decode error");
+        return None;
+    };
+
+    if udp_packet.conn_id != conn_id.clone() {
+        tracing::warn!(?udp_packet, ?conn_id, "udp conn id not match");
+        return None;
+    }
+
+    let ArchivedUdpPacketPayload::Data(payload) = &udp_packet.payload else {
+        tracing::warn!(?udp_packet, "udp payload not data");
+        return None;
+    };
+
+    let ptr_range = payload.as_ptr_range();
+    let offset = ptr_range.start as usize - buf.as_ptr() as usize;
+    let len = ptr_range.end as usize - ptr_range.start as usize;
+    buf.advance(offset);
+    buf.truncate(len);
+    tracing::trace!(?offset, ?len, ?buf, "udp payload data");
+
+    Some(buf)
+}
+
+fn get_tunnel_from_socket(
+    socket: Arc<UdpSocket>,
+    addr: SocketAddr,
+    conn_id: u32,
+) -> Box<dyn super::Tunnel> {
+    let udp = UdpFramed::new(socket.clone(), BytesCodec::new(UDP_DATA_MTU));
+    let (sink, stream) = udp.split();
+
+    let recv_addr = addr;
+    let stream = stream.filter_map(move |v| async move {
+        tracing::trace!(?v, "udp stream recv something");
+        if v.is_err() {
+            tracing::warn!(?v, "udp stream error");
+            return Some(Err(super::TunnelError::CommonError(
+                "udp stream error".to_owned(),
+            )));
+        }
+
+        let (buf, addr) = v.unwrap();
+        assert_eq!(addr, recv_addr.clone());
+        Some(Ok(try_get_data_payload(buf, conn_id.clone())?))
+    });
+    let stream = Box::pin(stream);
+
+    let sender_addr = addr;
+    let sink = Box::pin(sink.with(move |v: Bytes| async move {
+        if false {
+            return Err(super::TunnelError::CommonError("udp sink error".to_owned()));
+        }
+
+        // TODO: two copy here, how to avoid?
+        let udp_packet = UdpPacket::new_data_packet(conn_id, v.to_vec());
+        tracing::trace!(?udp_packet, ?v, "udp send packet");
+        let v = encode_to_bytes::<_, UDP_DATA_MTU>(&udp_packet);
+
+        Ok((v, sender_addr))
+    }));
+
+    FramedTunnel::new_tunnel_with_info(
+        stream,
+        sink,
+        // TODO: this remote addr is not a url
+        super::TunnelInfo {
+            tunnel_type: "udp".to_owned(),
+            local_addr: super::build_url_from_socket_addr(
+                &socket.local_addr().unwrap().to_string(),
+                "udp",
+            )
+            .into(),
+            remote_addr: super::build_url_from_socket_addr(&addr.to_string(), "udp").into(),
+        },
+    )
+}
+
+struct StreamSinkPair(
+    Pin<Box<dyn DatagramStream>>,
+    Pin<Box<dyn DatagramSink>>,
+    u32,
+);
+type ArcStreamSinkPair = Arc<Mutex<StreamSinkPair>>;
+
+pub struct UdpTunnelListener {
+    addr: url::Url,
+    socket: Option<Arc<UdpSocket>>,
+
+    sock_map: Arc<DashMap<SocketAddr, ArcStreamSinkPair>>,
+    forward_tasks: Arc<Mutex<JoinSet<()>>>,
+
+    conn_recv: tokio::sync::mpsc::Receiver<Box<dyn Tunnel>>,
+    conn_send: Option<tokio::sync::mpsc::Sender<Box<dyn Tunnel>>>,
+}
+
+impl UdpTunnelListener {
+    pub fn new(addr: url::Url) -> Self {
+        let (conn_send, conn_recv) = tokio::sync::mpsc::channel(100);
+        Self {
+            addr,
+            socket: None,
+            sock_map: Arc::new(DashMap::new()),
+            forward_tasks: Arc::new(Mutex::new(JoinSet::new())),
+            conn_recv,
+            conn_send: Some(conn_send),
+        }
+    }
+
+    async fn try_forward_packet(
+        sock_map: &DashMap<SocketAddr, ArcStreamSinkPair>,
+        buf: BytesMut,
+        addr: SocketAddr,
+    ) -> Result<(), super::TunnelError> {
+        let entry = sock_map.get_mut(&addr);
+        if entry.is_none() {
+            log::warn!("udp forward packet: {:?}, {:?}, no entry", addr, buf);
+            return Ok(());
+        }
+
+        log::trace!("udp forward packet: {:?}, {:?}", addr, buf);
+        let entry = entry.unwrap();
+        let pair = entry.value().clone();
+        drop(entry);
+
+        let Some(buf) = try_get_data_payload(buf, pair.lock().await.2) else {
+            return Ok(());
+        };
+        pair.lock().await.1.send(buf.freeze()).await?;
+        Ok(())
+    }
+
+    async fn handle_connect(
+        socket: Arc<UdpSocket>,
+        addr: SocketAddr,
+        forward_tasks: Arc<Mutex<JoinSet<()>>>,
+        sock_map: Arc<DashMap<SocketAddr, ArcStreamSinkPair>>,
+        local_url: url::Url,
+        conn_id: u32,
+    ) -> Result<Box<dyn Tunnel>, super::TunnelError> {
+        tracing::info!(?conn_id, ?addr, "udp connection accept handling",);
+
+        let udp_packet = UdpPacket::new_sack_packet(conn_id);
+        let sack_buf = encode_to_bytes::<_, UDP_DATA_MTU>(&udp_packet);
+        socket.send_to(&sack_buf, addr).await?;
+
+        let (ctunnel, stunnel) = create_ring_tunnel_pair();
+        let udp_tunnel = get_tunnel_from_socket(socket.clone(), addr, conn_id);
+        let ss_pair = StreamSinkPair(ctunnel.pin_stream(), ctunnel.pin_sink(), conn_id);
+        let addr_copy = addr.clone();
+        sock_map.insert(addr, Arc::new(Mutex::new(ss_pair)));
+        let ctunnel_stream = ctunnel.pin_stream();
+        forward_tasks.lock().await.spawn(async move {
+            let ret = ctunnel_stream
+                .map(|v| {
+                    tracing::trace!(?v, "udp stream recv something in forward task");
+                    if v.is_err() {
+                        return Err(super::TunnelError::CommonError(
+                            "udp stream error".to_owned(),
+                        ));
+                    }
+                    Ok(v.unwrap().freeze())
+                })
+                .forward(udp_tunnel.pin_sink())
+                .await;
+            if let None = sock_map.remove(&addr_copy) {
+                log::warn!("udp forward packet: {:?}, no entry", addr_copy);
+            }
+            close_tunnel(&ctunnel).await.unwrap();
+            log::warn!("udp connection forward done: {:?}, {:?}", addr_copy, ret);
+        });
+
+        Ok(Box::new(TunnelWithCustomInfo::new(
+            stunnel,
+            TunnelInfo {
+                tunnel_type: "udp".to_owned(),
+                local_addr: local_url.into(),
+                remote_addr: build_url_from_socket_addr(&addr.to_string(), "udp").into(),
+            },
+        )))
+    }
+
+    pub fn get_socket(&self) -> Option<Arc<UdpSocket>> {
+        self.socket.clone()
+    }
+}
+
+#[async_trait]
+impl TunnelListener for UdpTunnelListener {
+    async fn listen(&mut self) -> Result<(), super::TunnelError> {
+        let addr = super::check_scheme_and_get_socket_addr::<SocketAddr>(&self.addr, "udp")?;
+        self.socket = Some(Arc::new(UdpSocket::bind(addr).await?));
+
+        let socket = self.socket.as_ref().unwrap().clone();
+        let forward_tasks = self.forward_tasks.clone();
+        let sock_map = self.sock_map.clone();
+        let conn_send = self.conn_send.take().unwrap();
+        let local_url = self.local_url().clone();
+        self.forward_tasks.lock().await.spawn(
+            async move {
+                loop {
+                    let mut buf = BytesMut::new();
+                    buf.resize(2500, 0);
+                    let (_size, addr) = socket.recv_from(&mut buf).await.unwrap();
+                    let _ = buf.split_off(_size);
+                    log::trace!(
+                        "udp recv packet: {:?}, buf: {:?}, size: {}",
+                        addr,
+                        buf,
+                        _size
+                    );
+
+                    let Ok(udp_packet) = rkyv_util::decode_from_bytes_checked::<UdpPacket>(&buf)
+                    else {
+                        tracing::warn!(?buf, "udp decode error in forward task");
+                        continue;
+                    };
+
+                    if matches!(udp_packet.payload, ArchivedUdpPacketPayload::Syn) {
+                        let conn = Self::handle_connect(
+                            socket.clone(),
+                            addr,
+                            forward_tasks.clone(),
+                            sock_map.clone(),
+                            local_url.clone(),
+                            udp_packet.conn_id.into(),
+                        )
+                        .await
+                        .unwrap();
+                        if let Err(e) = conn_send.send(conn).await {
+                            tracing::warn!(?e, "udp send conn to accept channel error");
+                        }
+                    } else {
+                        Self::try_forward_packet(sock_map.as_ref(), buf, addr)
+                            .await
+                            .unwrap();
+                    }
+                }
+            }
+            .instrument(tracing::info_span!("udp forward task", ?self.socket)),
+        );
+
+        // let forward_tasks_clone = self.forward_tasks.clone();
+        // tokio::spawn(async move {
+        //     loop {
+        //         let mut locked_forward_tasks = forward_tasks_clone.lock().await;
+        //         tokio::select! {
+        //             ret = locked_forward_tasks.join_next() => {
+        //                 tracing::warn!(?ret, "udp forward task exit");
+        //             }
+        //             else => {
+        //                 drop(locked_forward_tasks);
+        //                 tokio::time::sleep(tokio::time::Duration::from_secs(1)).await;
+        //                 continue;
+        //             }
+        //         }
+        //     }
+        // });
+
+        Ok(())
+    }
+
+    async fn accept(&mut self) -> Result<Box<dyn super::Tunnel>, super::TunnelError> {
+        log::info!("start udp accept: {:?}", self.addr);
+        while let Some(conn) = self.conn_recv.recv().await {
+            return Ok(conn);
+        }
+        return Err(super::TunnelError::CommonError(
+            "udp accept error".to_owned(),
+        ));
+    }
+
+    fn local_url(&self) -> url::Url {
+        self.addr.clone()
+    }
+
+    fn get_conn_counter(&self) -> Arc<Box<dyn TunnelConnCounter>> {
+        struct UdpTunnelConnCounter {
+            sock_map: Arc<DashMap<SocketAddr, ArcStreamSinkPair>>,
+        }
+
+        impl Debug for UdpTunnelConnCounter {
+            fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+                f.debug_struct("UdpTunnelConnCounter")
+                    .field("sock_map_len", &self.sock_map.len())
+                    .finish()
+            }
+        }
+
+        impl TunnelConnCounter for UdpTunnelConnCounter {
+            fn get(&self) -> u32 {
+                self.sock_map.len() as u32
+            }
+        }
+
+        Arc::new(Box::new(UdpTunnelConnCounter {
+            sock_map: self.sock_map.clone(),
+        }))
+    }
+}
+
+pub struct UdpTunnelConnector {
+    addr: url::Url,
+    bind_addrs: Vec<SocketAddr>,
+}
+
+impl UdpTunnelConnector {
+    pub fn new(addr: url::Url) -> Self {
+        Self {
+            addr,
+            bind_addrs: vec![],
+        }
+    }
+
+    async fn wait_sack(
+        socket: &UdpSocket,
+        addr: SocketAddr,
+        conn_id: u32,
+    ) -> Result<(), super::TunnelError> {
+        let mut buf = BytesMut::new();
+        buf.resize(128, 0);
+
+        let (usize, recv_addr) = tokio::time::timeout(
+            tokio::time::Duration::from_secs(3),
+            socket.recv_from(&mut buf),
+        )
+        .await??;
+
+        if recv_addr != addr {
+            return Err(super::TunnelError::ConnectError(format!(
+                "udp connect error, unexpected sack addr: {:?}, {:?}",
+                recv_addr, addr
+            )));
+        }
+
+        let _ = buf.split_off(usize);
+
+        let Ok(udp_packet) = rkyv_util::decode_from_bytes_checked::<UdpPacket>(&buf) else {
+            tracing::warn!(?buf, "udp decode error in wait sack");
+            return Err(super::TunnelError::ConnectError(format!(
+                "udp connect error, decode error. buf: {:?}",
+                buf
+            )));
+        };
+
+        if conn_id != udp_packet.conn_id {
+            return Err(super::TunnelError::ConnectError(format!(
+                "udp connect error, conn id not match. conn_id: {:?}, {:?}",
+                conn_id, udp_packet.conn_id
+            )));
+        }
+
+        if !matches!(udp_packet.payload, ArchivedUdpPacketPayload::Sack) {
+            return Err(super::TunnelError::ConnectError(format!(
+                "udp connect error, unexpected payload. payload: {:?}",
+                udp_packet.payload
+            )));
+        }
+
+        Ok(())
+    }
+
+    async fn wait_sack_loop(
+        socket: &UdpSocket,
+        addr: SocketAddr,
+        conn_id: u32,
+    ) -> Result<(), super::TunnelError> {
+        while let Err(err) = Self::wait_sack(socket, addr, conn_id).await {
+            tracing::warn!(?err, "udp wait sack error");
+        }
+        Ok(())
+    }
+
+    pub async fn try_connect_with_socket(
+        &self,
+        socket: UdpSocket,
+    ) -> Result<Box<dyn super::Tunnel>, super::TunnelError> {
+        let addr = super::check_scheme_and_get_socket_addr::<SocketAddr>(&self.addr, "udp")?;
+        log::warn!("udp connect: {:?}", self.addr);
+
+        // send syn
+        let conn_id = rand::random();
+        let udp_packet = UdpPacket::new_syn_packet(conn_id);
+        let b = encode_to_bytes::<_, UDP_DATA_MTU>(&udp_packet);
+        let ret = socket.send_to(&b, &addr).await?;
+        tracing::warn!(?udp_packet, ?ret, "udp send syn");
+
+        // wait sack
+        tokio::time::timeout(
+            tokio::time::Duration::from_secs(3),
+            Self::wait_sack_loop(&socket, addr, conn_id),
+        )
+        .await??;
+
+        // sack done
+        let local_addr = socket.local_addr().unwrap().to_string();
+        Ok(Box::new(TunnelWithCustomInfo::new(
+            get_tunnel_from_socket(Arc::new(socket), addr, conn_id),
+            TunnelInfo {
+                tunnel_type: "udp".to_owned(),
+                local_addr: super::build_url_from_socket_addr(&local_addr, "udp").into(),
+                remote_addr: self.remote_url().into(),
+            },
+        )))
+    }
+
+    async fn connect_with_default_bind(&mut self) -> Result<Box<dyn Tunnel>, super::TunnelError> {
+        let socket = UdpSocket::bind("0.0.0.0:0").await?;
+        return self.try_connect_with_socket(socket).await;
+    }
+
+    async fn connect_with_custom_bind(&mut self) -> Result<Box<dyn Tunnel>, super::TunnelError> {
+        let mut futures = FuturesUnordered::new();
+
+        for bind_addr in self.bind_addrs.iter() {
+            let socket = UdpSocket::bind(*bind_addr).await?;
+
+            // linux does not use interface of bind_addr to send packet, so we need to bind device
+            // mac can handle this with bind correctly
+            #[cfg(any(target_os = "android", target_os = "fuchsia", target_os = "linux"))]
+            if let Some(dev_name) = super::common::get_interface_name_by_ip(&bind_addr.ip()) {
+                tracing::trace!(dev_name = ?dev_name, "bind device");
+                socket.bind_device(Some(dev_name.as_bytes()))?;
+            }
+
+            futures.push(self.try_connect_with_socket(socket));
+        }
+
+        let Some(ret) = futures.next().await else {
+            return Err(super::TunnelError::CommonError(
+                "join connect futures failed".to_owned(),
+            ));
+        };
+
+        return ret;
+    }
+}
+
+#[async_trait]
+impl super::TunnelConnector for UdpTunnelConnector {
+    async fn connect(&mut self) -> Result<Box<dyn super::Tunnel>, super::TunnelError> {
+        if self.bind_addrs.is_empty() {
+            self.connect_with_default_bind().await
+        } else {
+            self.connect_with_custom_bind().await
+        }
+    }
+
+    fn remote_url(&self) -> url::Url {
+        self.addr.clone()
+    }
+
+    fn set_bind_addrs(&mut self, addrs: Vec<SocketAddr>) {
+        self.bind_addrs = addrs;
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use crate::tunnels::common::tests::{_tunnel_bench, _tunnel_pingpong};
+
+    use super::*;
+
+    #[tokio::test]
+    async fn udp_pingpong() {
+        let listener = UdpTunnelListener::new("udp://0.0.0.0:5556".parse().unwrap());
+        let connector = UdpTunnelConnector::new("udp://127.0.0.1:5556".parse().unwrap());
+        _tunnel_pingpong(listener, connector).await
+    }
+
+    #[tokio::test]
+    async fn udp_bench() {
+        let listener = UdpTunnelListener::new("udp://0.0.0.0:5555".parse().unwrap());
+        let connector = UdpTunnelConnector::new("udp://127.0.0.1:5555".parse().unwrap());
+        _tunnel_bench(listener, connector).await
+    }
+
+    #[tokio::test]
+    async fn udp_bench_with_bind() {
+        let listener = UdpTunnelListener::new("udp://127.0.0.1:5554".parse().unwrap());
+        let mut connector = UdpTunnelConnector::new("udp://127.0.0.1:5554".parse().unwrap());
+        connector.set_bind_addrs(vec!["127.0.0.1:0".parse().unwrap()]);
+        _tunnel_pingpong(listener, connector).await
+    }
+
+    #[tokio::test]
+    #[should_panic]
+    async fn udp_bench_with_bind_fail() {
+        let listener = UdpTunnelListener::new("udp://127.0.0.1:5553".parse().unwrap());
+        let mut connector = UdpTunnelConnector::new("udp://127.0.0.1:5553".parse().unwrap());
+        connector.set_bind_addrs(vec!["10.0.0.1:0".parse().unwrap()]);
+        _tunnel_pingpong(listener, connector).await
+    }
+}

scripts/deploy.sh

@@ -0,0 +1,29 @@
+CWD=$(dirname `cargo locate-project | jq '.root' -r`)
+cargo build --release --all-targets
+
+TARGET_HOSTS=("root@public.kkrainbow.top" "root@47.242.85.82" "root@192.168.60.162")
+CARGO_BUILD_OUTPUT_DIR="$CWD/target/release"
+
+copy_bin_to_remote() {
+    local TARGET_HOST=$1
+    scp ${CARGO_BUILD_OUTPUT_DIR}/easytier-core $TARGET_HOST:/tmp/easytier-core &
+    scp ${CARGO_BUILD_OUTPUT_DIR}/easytier-cli $TARGET_HOST:/tmp/easytier-cli &
+}
+
+
+for TARGET_HOST in ${TARGET_HOSTS[@]}; do
+    ssh $TARGET_HOST "killall easytier-core"
+    copy_bin_to_remote $TARGET_HOST
+done
+
+wait
+
+run_with_args() {
+    local TARGET_HOST=$1
+    local ARGS=$2
+    ssh $TARGET_HOST "nohup /tmp/easytier-core $ARGS > /tmp/easytier-core.log 2>&1 &"
+}
+
+run_with_args "root@192.168.60.162" "--ipv4 10.144.144.10 --peers tcp://public.kkrainbow.top:11010"
+run_with_args "root@public.kkrainbow.top" "--ipv4 10.144.144.20"
+run_with_args "root@47.242.85.82" "--ipv4 10.144.144.30 --peers tcp://public.kkrainbow.top:11010"

scripts/local_init.sh

@@ -0,0 +1,10 @@
+ip netns add net_a
+ip netns add net_b
+ip netns add net_c
+ip link add veth0 type veth peer name veth1
+ip link set veth0 netns net_a
+ip link set veth1 netns net_b
+ip netns exec net_a ip addr add 10.144.145.1/24 dev veth0
+ip netns exec net_b ip addr add 10.144.145.2/24 dev veth1
+ip netns exec net_a ip link set veth0 up
+ip netns exec net_b ip link set veth1 up