diff --git a/.env b/.env index a9465f8..9eefcbb 100644 --- a/.env +++ b/.env @@ -9,3 +9,5 @@ FOOTER_BLURB="Opéré par @max" LOCAL_DOMAINS="xolus.net" LOCAL_BLURB="

Relais ActivityPub francophone

" # OPENTELEMETRY_URL=http://localhost:4317 +PROMETHEUS_ADDR=127.0.0.1 +PROMETHEUS_PORT=9000 diff --git a/Cargo.lock b/Cargo.lock index 5bd9064..fa0b763 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4,9 +4,9 @@ version = 3 [[package]] name = "activitystreams" -version = "0.7.0-alpha.20" +version = "0.7.0-alpha.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "459a89e7d449cf49e57044d59dcf637d2927362b7a75737c90bd3679ba476a78" +checksum = "1e673a517c3cd11c3b1870be31f2c95580e30f9ca79e89082ba94325097ed9c4" dependencies = [ "activitystreams-kinds", "iri-string", @@ -18,9 +18,9 @@ dependencies = [ [[package]] name = "activitystreams-ext" -version = "0.1.0-alpha.2" +version = "0.1.0-alpha.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb8e19a0810cc25df3535061a08b7d8f8a734d309ea4411c57a9767e4a2ffa0e" +checksum = "cd4cc5f95aabf2bc2e0672400f7494094ffae30b597317cb825b4ac5c8a405b3" dependencies = [ "activitystreams", "serde", @@ -29,9 +29,9 @@ dependencies = [ [[package]] name = "activitystreams-kinds" -version = "0.2.1" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d014a4fb8828870b7b46bee6257b9a89d06188ae8d435381ba94f14c8c697d8" +checksum = "e97dfe76efd8c0b113cc3580a6b5f4acba47662e3cfbbfcce081c9ac89798990" dependencies = [ "iri-string", "serde", @@ -65,7 +65,7 @@ dependencies = [ "actix-service", "actix-tls", "actix-utils", - "ahash", + "ahash 0.7.6", "base64", "bitflags", "brotli", @@ -197,7 +197,7 @@ dependencies = [ "actix-service", "actix-tls", "actix-utils", - "ahash", + "ahash 0.7.6", "bytes", "bytestring", "cfg-if", @@ -253,6 +253,17 @@ dependencies = [ "version_check", ] +[[package]] +name = "ahash" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf6ccdb167abbf410dcb915cabd428929d7f6a04980b54a11f26a39f1c7f7107" +dependencies = [ + "cfg-if", + "once_cell", + "version_check", +] + [[package]] name = "aho-corasick" version = "0.7.20" @@ -279,9 +290,9 @@ dependencies = [ [[package]] name = "ammonia" -version = "3.2.1" +version = "3.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b477377562f3086b7778d241786e9406b883ccfaa03557c0fe0924b9349f13a" +checksum = "64e6d1c7838db705c9b756557ee27c384ce695a1c51a6fe528784cb1c6840170" dependencies = [ "html5ever", "maplit", @@ -292,13 +303,13 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.66" +version = "1.0.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "216261ddc8289130e551ddcd5ce8a064710c0d064a4d2895c67151c92b5443f6" +checksum = "2cb2f989d18dd141ab8ae82f64d1a8cdd37e0840f73a406896cf5e99502fab61" [[package]] name = "ap-relay" -version = "0.3.66" +version = "0.3.79" dependencies = [ "activitystreams", "activitystreams-ext", @@ -320,6 +331,7 @@ dependencies = [ "http-signature-normalization-actix", "lru", "metrics", + "metrics-exporter-prometheus", "metrics-util", "mime", "minify-html", @@ -340,6 +352,7 @@ dependencies = [ "sled", "teloxide", "thiserror", + "time", "tokio", "toml", "tracing", @@ -368,9 +381,9 @@ dependencies = [ [[package]] name = "arc-swap" -version = "1.5.1" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "983cd8b9d4b02a6dc6ffa557262eb5858a27a0038ffffe21a0f133eaa819a164" +checksum = "bddcadddf5e9015d310179a59bb28c4d4b9920ad0f11e8e14dbadf654890c9a6" [[package]] name = "arrayvec" @@ -378,15 +391,6 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "23b62fc65de8e4e7f52534fb52b0f3ed04746ae267519eef2a83941e8085068b" -[[package]] -name = "async-mutex" -version = "1.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "479db852db25d9dbf6204e6cb6253698f175c15726470f78af0d918e99d6156e" -dependencies = [ - "event-listener", -] - [[package]] name = "async-stream" version = "0.3.3" @@ -410,26 +414,15 @@ dependencies = [ [[package]] name = "async-trait" -version = "0.1.58" +version = "0.1.60" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e805d94e6b5001b651426cf4cd446b1ab5f319d27bab5c644f61de0a804360c" +checksum = "677d1d8ab452a3936018a687b20e6f7cf5363d713b732b8884001317b0e48aa3" dependencies = [ "proc-macro2", "quote", "syn", ] -[[package]] -name = "atty" -version = "0.2.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" -dependencies = [ - "hermit-abi", - "libc", - "winapi", -] - [[package]] name = "autocfg" version = "1.1.0" @@ -448,7 +441,7 @@ dependencies = [ "actix-service", "actix-tls", "actix-utils", - "ahash", + "ahash 0.7.6", "base64", "bytes", "cfg-if", @@ -472,9 +465,9 @@ dependencies = [ [[package]] name = "axum" -version = "0.5.17" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "acee9fd5073ab6b045a275b3e709c163dd36c90685219cb21804a147b58dba43" +checksum = "08b108ad2665fa3f6e6a517c3d80ec3e77d224c47d605167aefaa5d7ef97fa48" dependencies = [ "async-trait", "axum-core", @@ -490,9 +483,9 @@ dependencies = [ "mime", "percent-encoding", "pin-project-lite", + "rustversion", "serde", "sync_wrapper", - "tokio", "tower", "tower-http", "tower-layer", @@ -501,9 +494,9 @@ dependencies = [ [[package]] name = "axum-core" -version = "0.2.9" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37e5939e02c56fecd5c017c37df4238c0a839fa76b7f97acdd7efb804fd181cc" +checksum = "79b8558f5a0581152dc94dcd289132a1d377494bdeafcd41869b3258e3e2ad92" dependencies = [ "async-trait", "bytes", @@ -511,6 +504,7 @@ dependencies = [ "http", "http-body", "mime", + "rustversion", "tower-layer", "tower-service", ] @@ -527,13 +521,12 @@ dependencies = [ [[package]] name = "background-jobs-actix" -version = "0.14.2" +version = "0.14.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99f8bfe0a984c8d0bc7e67b376cc05e0b9015fdd3ee878900046120ef781c47e" +checksum = "b6c7b21d56df90647f9bef717379663f8d92374f3707d95890a3e6da2f8a7359" dependencies = [ "actix-rt", "anyhow", - "async-mutex", "async-trait", "background-jobs-core", "metrics", @@ -548,9 +541,9 @@ dependencies = [ [[package]] name = "background-jobs-core" -version = "0.14.1" +version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1274e49ae8eff1fc6b4943660e59ce2f2e13e65a23a707924a50a40c7b94fc4d" +checksum = "21825e3893be7b103cd52cf3ee65db388b35816b5ec1229032b342a334eef533" dependencies = [ "actix-rt", "anyhow", @@ -683,9 +676,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.0.77" +version = "1.0.78" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9f73505338f7d905b19d18738976aae232eb46b8efc15554ffc56deb5d9ebe4" +checksum = "a20104e2335ce8a659d6dd92a51a767a0c062599c73b343fd152cb401e828c3d" [[package]] name = "cfg-if" @@ -715,14 +708,14 @@ dependencies = [ [[package]] name = "clap" -version = "4.0.26" +version = "4.0.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2148adefda54e14492fb9bddcc600b4344c5d1a3123bd666dcb939c6f0e0e57e" +checksum = "a7db700bc935f9e43e88d00b0850dae18a63773cfbec6d8e070fccf7fef89a39" dependencies = [ - "atty", "bitflags", "clap_derive", "clap_lex", + "is-terminal", "once_cell", "strsim", "termcolor", @@ -752,14 +745,14 @@ dependencies = [ [[package]] name = "config" -version = "0.13.2" +version = "0.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11f1667b8320afa80d69d8bbe40830df2c8a06003d86f73d8e003b2c48df416d" +checksum = "d379af7f68bfc21714c6c7dea883544201741d2ce8274bb12fa54f89507f52a7" dependencies = [ "async-trait", "json5", "lazy_static", - "nom 7.1.1", + "nom 7.1.2", "pathdiff", "ron", "rust-ini", @@ -930,17 +923,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "907076dfda823b0b36d2a1bb5f90c96660a5bbcd7729e10727f07858f22c4edc" dependencies = [ "cfg-if", - "hashbrown", + "hashbrown 0.12.3", "lock_api", "once_cell", - "parking_lot_core 0.9.4", + "parking_lot_core 0.9.5", ] [[package]] name = "der" -version = "0.6.0" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13dd2ae565c0a381dde7fade45fce95984c568bdcb4700a4fdbe3175e0380b2f" +checksum = "f1a467a65c5e759bce6e65eaf91cc29f466cdc57cb65777bd646872a8a1fd4de" dependencies = [ "const-oid", "pem-rfc7468", @@ -1023,6 +1016,27 @@ dependencies = [ "scopeguard", ] +[[package]] +name = "errno" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f639046355ee4f37944e44f60642c6f3a7efa3cf6b78c78a0d989a8ce6c396a1" +dependencies = [ + "errno-dragonfly", + "libc", + "winapi", +] + +[[package]] +name = "errno-dragonfly" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf" +dependencies = [ + "cc", + "libc", +] + [[package]] name = "event-listener" version = "2.5.3" @@ -1046,9 +1060,9 @@ checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" [[package]] name = "flate2" -version = "1.0.24" +version = "1.0.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f82b0f4c27ad9f8bfd1f3208d882da2b09c301bc1c828fd3a00d0216d2fbbff6" +checksum = "a8a2db397cb1c8772f31494cb8917e48cd1e64f0fa7efac59fbd741a0a8ce841" dependencies = [ "crc32fast", "miniz_oxide", @@ -1239,7 +1253,16 @@ version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" dependencies = [ - "ahash", + "ahash 0.7.6", +] + +[[package]] +name = "hashbrown" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ff8ae62cd3a9102e5637afc8452c55acf3844001bd5374e0b0bd7b6616c038" +dependencies = [ + "ahash 0.8.2", ] [[package]] @@ -1251,7 +1274,7 @@ dependencies = [ "base64", "byteorder", "flate2", - "nom 7.1.1", + "nom 7.1.2", "num-traits", ] @@ -1263,9 +1286,9 @@ checksum = "2540771e65fc8cb83cd6e8a237f70c319bd5c29f78ed1084ba5d50eeac86f7f9" [[package]] name = "hermit-abi" -version = "0.1.19" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" +checksum = "ee512640fe35acbfb4bb779db6f0d80704c2cacfa2e39b601ef3e3f47d1ae4c7" dependencies = [ "libc", ] @@ -1314,18 +1337,18 @@ checksum = "0bfe8eed0a9285ef776bb792479ea3834e8b94e13d615c2f66d03dd50a435a29" [[package]] name = "http-signature-normalization" -version = "0.6.0" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8f45adbef81d7ea3bd7e9bcc6734b7245dad05a14abdcc7ddc0988791d63515" +checksum = "b95e3149194de5f3f9d5225bcc6a8677979f8ff8ce39c85654730ad4824f101e" dependencies = [ "httpdate", ] [[package]] name = "http-signature-normalization-actix" -version = "0.6.2" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7483d0ee4d093fa4bfe5956cd405492c07808a5064a29cfe3960d474f21f39c2" +checksum = "1dc95d9ca3b4e2f93a97e5ccf9f26992c69a272e0abad8807180f0a9e9b59e31" dependencies = [ "actix-http", "actix-rt", @@ -1386,9 +1409,9 @@ dependencies = [ [[package]] name = "hyper-rustls" -version = "0.23.1" +version = "0.23.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59df7c4e19c950e6e0e868dcc0a300b09a9b88e9ec55bd879ca819087a77355d" +checksum = "1788965e61b367cd03a62950836d5cd41560c3577d90e40e0819373194d1661c" dependencies = [ "http", "hyper", @@ -1432,7 +1455,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1885e79c1fc4b10f0e172c475f458b7f7b93061064d98c3293e98c5ba0c8b399" dependencies = [ "autocfg", - "hashbrown", + "hashbrown 0.12.3", ] [[package]] @@ -1454,20 +1477,43 @@ dependencies = [ ] [[package]] -name = "ipnet" -version = "2.5.1" +name = "io-lifetimes" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f88c5561171189e69df9d98bcf18fd5f9558300f7ea7b801eb8a0fd748bd8745" +checksum = "46112a93252b123d31a119a8d1a1ac19deac4fac6e0e8b0df58f0d4e5870e63c" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "ipnet" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11b0d96e660696543b251e58030cf9787df56da39dab19ad60eae7353040917e" [[package]] name = "iri-string" -version = "0.5.6" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf071934ee7ee97e52fa1868a9540a7885eab75926bd70794030304a9797cea1" +checksum = "21859b667d66a4c1dacd9df0863b3efb65785474255face87f5bca39dd8407c0" dependencies = [ + "memchr", "serde", ] +[[package]] +name = "is-terminal" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28dfb6c8100ccc63462345b67d1bbc3679177c75ee4bf59bf29c8b1d110b8189" +dependencies = [ + "hermit-abi", + "io-lifetimes", + "rustix", + "windows-sys", +] + [[package]] name = "itertools" version = "0.9.0" @@ -1488,9 +1534,9 @@ dependencies = [ [[package]] name = "itoa" -version = "1.0.4" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4217ad341ebadf8d8e724e264f13e593e0648f5b3e94b3896a5df283be015ecc" +checksum = "fad582f4b9e86b6caa621cabeb0963332d92eea04729ab12892c2533951e6440" [[package]] name = "js-sys" @@ -1542,9 +1588,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.137" +version = "0.2.139" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc7fcc620a3bff7cdd7a365be3376c97191aeaccc2a603e600951e452615bf89" +checksum = "201de327520df007757c1f0adce6e827fe8562fbc28bfd9c15571c66ca1f5f79" [[package]] name = "libm" @@ -1558,6 +1604,12 @@ version = "0.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" +[[package]] +name = "linux-raw-sys" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f051f77a7c8e6957c0696eac88f26b0117e54f52d3fc682ab19397a8812846a4" + [[package]] name = "local-channel" version = "0.1.3" @@ -1597,11 +1649,11 @@ dependencies = [ [[package]] name = "lru" -version = "0.8.1" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6e8aaa3f231bb4bd57b84b2d5dc3ae7f350265df8aa96492e0bc394a1571909" +checksum = "71e7d46de488603ffdd5f30afbc64fbba2378214a2c3a2fb83abf3d33126df17" dependencies = [ - "hashbrown", + "hashbrown 0.13.1", ] [[package]] @@ -1650,9 +1702,9 @@ dependencies = [ [[package]] name = "matchit" -version = "0.5.0" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73cbba799671b762df5a175adf59ce145165747bb891505c43d09aefbbf38beb" +checksum = "b87248edafb776e59e6ee64a79086f65890d3510f2c656c000bf2a7e8a0aea40" [[package]] name = "md5" @@ -1681,11 +1733,29 @@ version = "0.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b9b8653cec6897f73b519a43fba5ee3d50f62fe9af80b428accdcc093b4a849" dependencies = [ - "ahash", + "ahash 0.7.6", "metrics-macros", "portable-atomic", ] +[[package]] +name = "metrics-exporter-prometheus" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8603921e1f54ef386189335f288441af761e0fc61bcb552168d9cedfe63ebc70" +dependencies = [ + "hyper", + "indexmap", + "ipnet", + "metrics", + "metrics-util", + "parking_lot 0.12.1", + "portable-atomic", + "quanta", + "thiserror", + "tokio", +] + [[package]] name = "metrics-macros" version = "0.6.0" @@ -1706,7 +1776,7 @@ dependencies = [ "aho-corasick", "crossbeam-epoch", "crossbeam-utils", - "hashbrown", + "hashbrown 0.12.3", "indexmap", "metrics", "num_cpus", @@ -1765,9 +1835,9 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "miniz_oxide" -version = "0.5.4" +version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96590ba8f175222643a85693f33d26e9c8a015f599c216509b1a6894af675d34" +checksum = "b275950c28b37e794e8c55d88aeb5e139d0ce23fdbbeda68f8d7174abdf9e8fa" dependencies = [ "adler", ] @@ -1826,9 +1896,9 @@ dependencies = [ [[package]] name = "nom" -version = "7.1.1" +version = "7.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8903e5a29a317527874d0402f867152a3d21c908bb0b933e416c65e301d4c36" +checksum = "e5507769c4919c998e69e49c839d9dc6e693ede4cc4290d6ad8b41d4f09c548c" dependencies = [ "memchr", "minimal-lexical", @@ -1842,7 +1912,7 @@ checksum = "37794436ca3029a3089e0b95d42da1f0b565ad271e4d3bb4bad0c7bb70b10605" dependencies = [ "bytecount", "memchr", - "nom 7.1.1", + "nom 7.1.2", ] [[package]] @@ -1928,9 +1998,9 @@ dependencies = [ [[package]] name = "num_cpus" -version = "1.14.0" +version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6058e64324c71e02bc2b150e4f3bc8286db6c83092132ffa3f6b1eab0f9def5" +checksum = "0fac9e2da13b5eb447a6ce3d392f23a29d8694bff781bf03a16cd9ac8697593b" dependencies = [ "hermit-abi", "libc", @@ -1938,9 +2008,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.16.0" +version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86f0b0d4bf799edbc74508c1e8bf170ff5f41238e5f8225603ca7caaae2b7860" +checksum = "6f61fba1741ea2b3d6a1e3178721804bb716a68a6aeba1149b5d52e3d464ea66" [[package]] name = "opentelemetry" @@ -2038,7 +2108,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ccd746e37177e1711c20dd619a1620f34f5c8b569c53590a72dedd5344d8924a" dependencies = [ "dlv-list", - "hashbrown", + "hashbrown 0.12.3", ] [[package]] @@ -2061,7 +2131,7 @@ checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99" dependencies = [ "instant", "lock_api", - "parking_lot_core 0.8.5", + "parking_lot_core 0.8.6", ] [[package]] @@ -2071,14 +2141,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" dependencies = [ "lock_api", - "parking_lot_core 0.9.4", + "parking_lot_core 0.9.5", ] [[package]] name = "parking_lot_core" -version = "0.8.5" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d76e8e1493bcac0d2766c42737f34458f1c8c50c0d23bcb24ea953affb273216" +checksum = "60a2cfe6f0ad2bfc16aefa463b497d5c7a5ecd44a23efa72aa342d90177356dc" dependencies = [ "cfg-if", "instant", @@ -2090,9 +2160,9 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.9.4" +version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4dc9e0dc2adc1c69d09143aff38d3d30c5c3f0df0dad82e6d25547af174ebec0" +checksum = "7ff9f3fef3968a3ec5945535ed654cb38ff72d7495a25619e2247fb15a2ed9ba" dependencies = [ "cfg-if", "libc", @@ -2114,9 +2184,9 @@ dependencies = [ [[package]] name = "paste" -version = "1.0.9" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1de2e551fb905ac83f73f7aedf2f0cb4a0da7e35efa24a202a936269f1f18e1" +checksum = "d01a5bd0424d00070b0098dd17ebca6f961a959dead1dbcbbbc1d1cd8d3deeba" [[package]] name = "pathdiff" @@ -2141,9 +2211,9 @@ checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e" [[package]] name = "pest" -version = "2.4.1" +version = "2.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a528564cc62c19a7acac4d81e01f39e53e25e17b934878f4c6d25cc2836e62f8" +checksum = "0f6e86fb9e7026527a0d46bc308b841d73170ef8f443e1807f6ef88526a816d4" dependencies = [ "thiserror", "ucd-trie", @@ -2151,9 +2221,9 @@ dependencies = [ [[package]] name = "pest_derive" -version = "2.4.1" +version = "2.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5fd9bc6500181952d34bd0b2b0163a54d794227b498be0b7afa7698d0a7b18f" +checksum = "96504449aa860c8dcde14f9fba5c58dc6658688ca1fe363589d6327b8662c603" dependencies = [ "pest", "pest_generator", @@ -2161,9 +2231,9 @@ dependencies = [ [[package]] name = "pest_generator" -version = "2.4.1" +version = "2.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2610d5ac5156217b4ff8e46ddcef7cdf44b273da2ac5bca2ecbfa86a330e7c4" +checksum = "798e0220d1111ae63d66cb66a5dcb3fc2d986d520b98e49e1852bfdb11d7c5e7" dependencies = [ "pest", "pest_meta", @@ -2174,9 +2244,9 @@ dependencies = [ [[package]] name = "pest_meta" -version = "2.4.1" +version = "2.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "824749bf7e21dd66b36fbe26b3f45c713879cccd4a009a917ab8e045ca8246fe" +checksum = "984298b75898e30a843e278a9f2452c31e349a073a0ce6fd950a12a74464e065" dependencies = [ "once_cell", "pest", @@ -2287,9 +2357,9 @@ dependencies = [ [[package]] name = "portable-atomic" -version = "0.3.15" +version = "0.3.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15eb2c6e362923af47e13c23ca5afb859e83d54452c55b0b9ac763b8f7c1ac16" +checksum = "26f6a7b87c2e435a3241addceeeff740ff8b7e76b74c13bf9acb17fa454ea00b" [[package]] name = "ppv-lite86" @@ -2305,9 +2375,9 @@ checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c" [[package]] name = "prettyplease" -version = "0.1.21" +version = "0.1.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c142c0e46b57171fe0c528bee8c5b7569e80f0c17e377cd0e30ea57dbc11bb51" +checksum = "2c8992a85d8e93a28bdf76137db888d3874e3b230dee5ed8bebac4c9f7617773" dependencies = [ "proc-macro2", "syn", @@ -2339,18 +2409,18 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.47" +version = "1.0.49" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ea3d908b0e36316caf9e9e2c4625cdde190a7e6f440d794667ed17a1855e725" +checksum = "57a8eca9f9c4ffde41714334dee777596264c7825420f521abc92b5b5deb63a5" dependencies = [ "unicode-ident", ] [[package]] name = "prost" -version = "0.11.2" +version = "0.11.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0841812012b2d4a6145fae9a6af1534873c32aa67fff26bd09f8fa42c83f95a" +checksum = "c01db6702aa05baa3f57dec92b8eeeeb4cb19e894e73996b32a4093289e54592" dependencies = [ "bytes", "prost-derive", @@ -2358,9 +2428,9 @@ dependencies = [ [[package]] name = "prost-build" -version = "0.11.2" +version = "0.11.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d8b442418ea0822409d9e7d047cbf1e7e9e1760b172bf9982cf29d517c93511" +checksum = "cb5320c680de74ba083512704acb90fe00f28f79207286a848e730c45dd73ed6" dependencies = [ "bytes", "heck", @@ -2380,9 +2450,9 @@ dependencies = [ [[package]] name = "prost-derive" -version = "0.11.2" +version = "0.11.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "164ae68b6587001ca506d3bf7f1000bfa248d0e1217b618108fba4ec1d0cc306" +checksum = "c8842bad1a5419bca14eac663ba798f6bc19c413c2fdceb5f3ba3b0932d96720" dependencies = [ "anyhow", "itertools 0.10.5", @@ -2393,9 +2463,9 @@ dependencies = [ [[package]] name = "prost-types" -version = "0.11.2" +version = "0.11.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "747761bc3dc48f9a34553bf65605cf6cb6288ba219f3450b4275dbd81539551a" +checksum = "017f79637768cde62820bc2d4fe0e45daaa027755c323ad077767c6c5f173091" dependencies = [ "bytes", "prost", @@ -2419,9 +2489,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.21" +version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbe448f377a7d6961e30f5955f9b8d106c3f5e449d493ee1b125c1d43c2b5179" +checksum = "8856d8364d252a14d474036ea1358d63c9e6965c8e5c1885c18f73d70bff9c7b" dependencies = [ "proc-macro2", ] @@ -2643,7 +2713,7 @@ dependencies = [ "arc-swap", "fastrand", "lazy_static", - "nom 7.1.1", + "nom 7.1.2", "nom_locate", "num-bigint", "num-integer", @@ -2663,7 +2733,7 @@ dependencies = [ "itertools 0.10.5", "md5", "mime", - "nom 7.1.1", + "nom 7.1.2", "rsass", ] @@ -2686,6 +2756,20 @@ dependencies = [ "semver", ] +[[package]] +name = "rustix" +version = "0.36.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4feacf7db682c6c329c4ede12649cd36ecab0f3be5b7d74e6a20304725db4549" +dependencies = [ + "bitflags", + "errno", + "io-lifetimes", + "libc", + "linux-raw-sys", + "windows-sys", +] + [[package]] name = "rustls" version = "0.20.7" @@ -2708,10 +2792,16 @@ dependencies = [ ] [[package]] -name = "ryu" +name = "rustversion" version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4501abdff3ae82a1c1b477a17252eb69cee9e66eb915c1abaa4f44d873df9f09" +checksum = "5583e89e108996506031660fe09baa5011b9dd0341b89029313006d1fb508d70" + +[[package]] +name = "ryu" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b4b9743ed687d4b4bcedf9ff5eaa7398495ae14e61cba0a295704edbc7decde" [[package]] name = "scopeguard" @@ -2731,24 +2821,24 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.14" +version = "1.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e25dfac463d778e353db5be2449d1cce89bd6fd23c9f1ea21310ce6e5a1b29c4" +checksum = "58bc9567378fc7690d6b2addae4e60ac2eeea07becb2c64b9f218b53865cba2a" [[package]] name = "serde" -version = "1.0.147" +version = "1.0.152" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d193d69bae983fc11a79df82342761dfbf28a99fc8d203dca4c3c1b590948965" +checksum = "bb7d1f0d3021d347a83e556fc4683dea2ea09d87bccdf88ff5c12545d89d5efb" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.147" +version = "1.0.152" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f1d362ca8fc9c3e3a7484440752472d68a6caa98f1ab81d99b5dfe517cec852" +checksum = "af487d118eecd09402d70a5d72551860e788df87b464af30e5ea6a38c75c541e" dependencies = [ "proc-macro2", "quote", @@ -2757,9 +2847,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.89" +version = "1.0.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "020ff22c755c2ed3f8cf162dbb41a7268d934702f3ed3631656ea597e08fc3db" +checksum = "877c235533714907a8c2464236f5c4b2a17262ef1bd71f38f35ea592c8da6883" dependencies = [ "itoa", "ryu", @@ -2955,9 +3045,9 @@ checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601" [[package]] name = "syn" -version = "1.0.103" +version = "1.0.107" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a864042229133ada95abf3b54fdc62ef5ccabe9515b64717bcb9a1919e59445d" +checksum = "1f4064b5b16e03ae50984a5a8ed5d4f8803e6bc1fd170a3cda91a1be4b18e3f5" dependencies = [ "proc-macro2", "quote", @@ -2990,9 +3080,9 @@ checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" [[package]] name = "teloxide" -version = "0.11.2" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19017dde82bddcbbdf8e40484f23985f4097b1baef4f7c0e006195ad1e6d4e3c" +checksum = "59c3b28292b33a57a8d71ce000c23fdaffeb0b4aec35fa9351d4be7ec6376a3f" dependencies = [ "aquamarine", "bytes", @@ -3093,18 +3183,18 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.37" +version = "1.0.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10deb33631e3c9018b9baf9dcbbc4f737320d2b576bac10f6aefa048fa407e3e" +checksum = "6a9cd18aa97d5c45c6603caea1da6628790b37f7a34b6ca89522331c5180fed0" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.37" +version = "1.0.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "982d17546b47146b28f7c22e3d08465f6b8903d0ea13c1660d9d84a6e7adcdbb" +checksum = "1fb327af4685e4d03fa8cbcf1716380da910eeb2bb8be417e7f9fd3fb164f36f" dependencies = [ "proc-macro2", "quote", @@ -3164,9 +3254,9 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" [[package]] name = "tokio" -version = "1.22.0" +version = "1.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d76ce4a75fb488c605c54bf610f221cea8b0dafb53333c1a67e8ee199dcd2ae3" +checksum = "eab6d665857cc6ca78d6e80303a02cea7a7851e85dfbd77cbdc09bd129f1ef46" dependencies = [ "autocfg", "bytes", @@ -3179,7 +3269,7 @@ dependencies = [ "socket2", "tokio-macros", "tracing", - "winapi", + "windows-sys", ] [[package]] @@ -3194,9 +3284,9 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "1.8.0" +version = "1.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9724f9a975fb987ef7a3cd9be0350edcbe130698af5b8f7a631e23d42d052484" +checksum = "d266c00fde287f55d3f1c3e96c500c362a2b8c695076ec180f27918820bc6df8" dependencies = [ "proc-macro2", "quote", @@ -3241,18 +3331,18 @@ dependencies = [ [[package]] name = "toml" -version = "0.5.9" +version = "0.5.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d82e1a7758622a465f8cee077614c73484dac5b836c02ff6a40d5d1010324d7" +checksum = "1333c76748e868a4d9d1017b5ab53171dfd095f70c712fdb4653a406547f598f" dependencies = [ "serde", ] [[package]] name = "tonic" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55b9af819e54b8f33d453655bef9b9acc171568fb49523078d0cc4e7484200ec" +checksum = "8f219fad3b929bef19b1f86fbc0358d35daed8f2cac972037ac0dc10bbb8d5fb" dependencies = [ "async-stream", "async-trait", @@ -3282,9 +3372,9 @@ dependencies = [ [[package]] name = "tonic-build" -version = "0.8.2" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48c6fd7c2581e36d63388a9e04c350c21beb7a8b059580b2e93993c526899ddc" +checksum = "5bf5e9b9c0f7e0a7c027dcfaba7b2c60816c7049171f679d99ee2ff65d0de8c4" dependencies = [ "prettyplease", "proc-macro2", @@ -3315,9 +3405,9 @@ dependencies = [ [[package]] name = "tower-http" -version = "0.3.4" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c530c8675c1dbf98facee631536fa116b5fb6382d7dd6dc1b118d970eafe3ba" +checksum = "f873044bf02dd1e8239e9c1293ea39dad76dc594ec16185d0a1bf31d8dc8d858" dependencies = [ "bitflags", "bytes", @@ -3359,9 +3449,9 @@ dependencies = [ [[package]] name = "tracing-actix-web" -version = "0.6.2" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d725b8fa6ef307b3f4856913523337de45c47cc79271bafd7acfb39559e3a2da" +checksum = "d16c2a0c52b267d46ea9a46012a28b3513ce166c28eaeaa875829ed2f8debd19" dependencies = [ "actix-web", "pin-project", @@ -3476,9 +3566,9 @@ checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642" [[package]] name = "typenum" -version = "1.15.0" +version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcf81ac59edc17cc8697ff311e8f5ef2d99fcbd9817b34cec66f90b6c3dfd987" +checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba" [[package]] name = "ucd-trie" @@ -3503,9 +3593,9 @@ checksum = "099b7128301d285f79ddd55b9a83d5e6b9e97c92e0ea0daebee7263e932de992" [[package]] name = "unicode-ident" -version = "1.0.5" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ceab39d59e4c9499d4e5a8ee0e2735b891bb7308ac83dfb4e80cad195c9f6f3" +checksum = "84a22b9f218b40614adcb3f4ff08b703773ad44fa9423e4e0d346d5db86e4ebc" [[package]] name = "unicode-normalization" @@ -3672,9 +3762,9 @@ dependencies = [ [[package]] name = "webpki-roots" -version = "0.22.5" +version = "0.22.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "368bfe657969fb01238bb756d351dcade285e0f6fcbd36dcb23359a5169975be" +checksum = "b6c71e40d7d2c34a5106301fb632274ca37242cd0c9d3e64dbece371a40a2d87" dependencies = [ "webpki", ] diff --git a/Cargo.toml b/Cargo.toml index fbc9e5f..e61fa5f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "ap-relay" description = "A simple activitypub relay" -version = "0.3.66" +version = "0.3.79" authors = ["asonix "] license = "AGPL-3.0" readme = "README.md" @@ -29,8 +29,8 @@ actix-web = { version = "4.0.1", default-features = false, features = [ "compress-gzip", ] } actix-webfinger = "0.4.0" -activitystreams = "0.7.0-alpha.19" -activitystreams-ext = "0.1.0-alpha.2" +activitystreams = "0.7.0-alpha.21" +activitystreams-ext = "0.1.0-alpha.3" ammonia = "3.1.0" awc = { version = "3.0.0", default-features = false, features = ["rustls"] } bcrypt = "0.13" @@ -41,8 +41,11 @@ console-subscriber = { version = "0.1", optional = true } dashmap = "5.1.0" dotenv = "0.15.0" futures-util = "0.3.17" -lru = "0.8.0" +lru = "0.9.0" metrics = "0.20.1" +metrics-exporter-prometheus = { version = "0.11.0", default-features = false, features = [ + "http-listener", +] } metrics-util = "0.14.0" mime = "0.3.16" minify-html = "0.10.0" @@ -66,6 +69,7 @@ teloxide = { version = "0.11.1", default-features = false, features = [ "rustls", ] } thiserror = "1.0" +time = { version = "0.3.17", features = ["serde"] } tracing = "0.1" tracing-awc = "0.1.6" tracing-error = "0.2" @@ -86,12 +90,12 @@ default-features = false features = ["background-jobs-actix", "error-logging"] [dependencies.http-signature-normalization-actix] -version = "0.6.0" +version = "0.8.0" default-features = false features = ["client", "server", "sha-2"] [dependencies.tracing-actix-web] -version = "0.6.1" +version = "0.7.0" [build-dependencies] anyhow = "1.0" diff --git a/README.md b/README.md index 64d873e..4e0f8e2 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ $ sudo docker run --rm -it \ -e ADDR=0.0.0.0 \ -e SLED_PATH=/mnt/sled/db-0.34 \ -p 8080:8080 \ - asonix/relay:0.3.52 + asonix/relay:0.3.78 ``` This will launch the relay with the database stored in "./sled/db-0.34" and listening on port 8080 #### Cargo @@ -103,6 +103,8 @@ TLS_CERT=/path/to/cert FOOTER_BLURB="Contact @asonix for inquiries" LOCAL_DOMAINS=masto.asonix.dog LOCAL_BLURB="

Welcome to my cool relay where I have cool relay things happening. I hope you enjoy your stay!

" +PROMETHEUS_ADDR=0.0.0.0 +PROMETHEUS_PORT=9000 ``` #### Descriptions @@ -128,6 +130,8 @@ Where to store the on-disk database of connected servers. This defaults to `./sl The log level to print. Available levels are `ERROR`, `WARN`, `INFO`, `DEBUG`, and `TRACE`. You can also specify module paths to enable some logs but not others, such as `RUST_LOG=warn,tracing_actix_web=info,relay=info`. This defaults to `warn` ##### `SOURCE_REPO` The URL to the source code for the relay. This defaults to `https://git.asonix.dog/asonix/relay`, but should be changed if you're running a fork hosted elsewhere. +##### `REPOSITORY_COMMIT_BASE` +The base path of the repository commit hash reference. For example, `/src/commit/` for Gitea, `/tree/` for GitLab. ##### `API_TOKEN` The Secret token used to access the admin APIs. This must be set for the commandline to function ##### `OPENTELEMETRY_URL` @@ -146,6 +150,10 @@ Optional - Add custom notes in the footer of the page Optional - domains of mastodon servers run by the same admin as the relay ##### `LOCAL_BLURB` Optional - description for the relay +##### `PROMETHEUS_ADDR` +Optional - Address to bind to for serving the prometheus scrape endpoint +##### `PROMETHEUS_PORT` +Optional - Port to bind to for serving the prometheus scrape endpoint ### Subscribing Mastodon admins can subscribe to this relay by adding the `/inbox` route to their relay settings. @@ -165,10 +173,13 @@ example, if the server is `https://relay.my.tld`, the correct URL would be - Follow Public, become a listener of the relay - Undo Follow {self-actor}, stop listening on the relay, an Undo Follow will be sent back - Undo Follow Public, stop listening on the relay -- Delete {anything}, the Delete {anything} is relayed verbatim to listening servers +- Delete {anything}, the Delete {anything} is relayed verbatim to listening servers. Note that this activity will likely be rejected by the listening servers unless it has been signed with a JSON-LD signature -- Update {anything}, the Update {anything} is relayed verbatim to listening servers +- Update {anything}, the Update {anything} is relayed verbatim to listening servers. + Note that this activity will likely be rejected by the listening servers unless it has been + signed with a JSON-LD signature +- Add {anything}, the Add {anything} is relayed verbatim to listening servers. Note that this activity will likely be rejected by the listening servers unless it has been signed with a JSON-LD signature diff --git a/docker/prod/Dockerfile b/docker/prod/Dockerfile deleted file mode 100644 index 280aa95..0000000 --- a/docker/prod/Dockerfile +++ /dev/null @@ -1,41 +0,0 @@ -ARG REPO_ARCH=amd64 - -# cross-build environment -FROM asonix/rust-builder:$REPO_ARCH-latest AS builder - -ARG TAG=main -ARG BINARY=relay -ARG PROJECT=relay -ARG GIT_REPOSITORY=https://git.asonix.dog/asonix/$PROJECT - -ENV \ - BINARY=${BINARY} - -ADD \ - --chown=build:build \ - $GIT_REPOSITORY/archive/$TAG.tar.gz \ - /opt/build/repo.tar.gz - -RUN \ - tar zxf repo.tar.gz - -WORKDIR /opt/build/$PROJECT - -RUN \ - build - -# production environment -FROM asonix/rust-runner:$REPO_ARCH-latest - -ARG BINARY=relay - -ENV \ - BINARY=${BINARY} - -COPY \ - --from=builder \ - /opt/build/binary \ - /usr/bin/${BINARY} - -ENTRYPOINT ["/sbin/tini", "--"] -CMD /usr/bin/${BINARY} diff --git a/docker/prod/build-image.sh b/docker/prod/build-image.sh deleted file mode 100755 index 2782f1e..0000000 --- a/docker/prod/build-image.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/usr/bin/env bash - -function require() { - if [ "$1" = "" ]; then - echo "input '$2' required" - print_help - exit 1 - fi -} - -function print_help() { - echo "deploy.sh" - echo "" - echo "Usage:" - echo " deploy.sh [repo] [tag] [arch]" - echo "" - echo "Args:" - echo " repo: The docker repository to publish the image" - echo " tag: The tag applied to the docker image" - echo " arch: The architecuture of the doker image" -} - -REPO=$1 -TAG=$2 -ARCH=$3 - -require "$REPO" repo -require "$TAG" tag -require "$ARCH" arch - -sudo docker build \ - --pull \ - --build-arg TAG=$TAG \ - --build-arg REPO_ARCH=$ARCH \ - -t $REPO:$ARCH-$TAG \ - -f Dockerfile \ - . diff --git a/docker/prod/deploy.sh b/docker/prod/deploy.sh deleted file mode 100755 index fcd7539..0000000 --- a/docker/prod/deploy.sh +++ /dev/null @@ -1,87 +0,0 @@ -#!/usr/bin/env bash - -function require() { - if [ "$1" = "" ]; then - echo "input '$2' required" - print_help - exit 1 - fi -} - -function print_help() { - echo "deploy.sh" - echo "" - echo "Usage:" - echo " deploy.sh [tag] [branch] [push]" - echo "" - echo "Args:" - echo " tag: The git tag to be applied to the repository and docker build" - echo " branch: The git branch to use for tagging and publishing" - echo " push: Whether or not to push the image" - echo "" - echo "Examples:" - echo " ./deploy.sh v0.3.0-alpha.13 main true" - echo " ./deploy.sh v0.3.0-alpha.13-shell-out asonix/shell-out false" -} - -function build_image() { - tag=$1 - arch=$2 - push=$3 - - ./build-image.sh asonix/relay $tag $arch - - sudo docker tag asonix/relay:$arch-$tag asonix/relay:$arch-latest - - if [ "$push" == "true" ]; then - sudo docker push asonix/relay:$arch-$tag - sudo docker push asonix/relay:$arch-latest - fi -} - -# Creating the new tag -new_tag="$1" -branch="$2" -push=$3 - -require "$new_tag" "tag" -require "$branch" "branch" -require "$push" "push" - -if ! sudo docker run --rm -it arm64v8/alpine:3.11 /bin/sh -c 'echo "docker is configured correctly"' -then - echo "docker is not configured to run on qemu-emulated architectures, fixing will require sudo" - sudo docker run --rm --privileged multiarch/qemu-user-static --reset -p yes -fi - -set -xe - -git checkout $branch - -# Changing the docker-compose prod -sed -i "s/asonix\/relay:.*/asonix\/relay:$new_tag/" docker-compose.yml -git add ../prod/docker-compose.yml -# The commit -git commit -m"Version $new_tag" -git tag $new_tag - -# Push -git push origin $new_tag -git push - -# Build for arm64v8, arm32v7 and amd64 -build_image $new_tag arm64v8 $push -build_image $new_tag arm32v7 $push -build_image $new_tag amd64 $push - -# Build for other archs -# TODO - -if [ "$push" == "true" ]; then - ./manifest.sh relay $new_tag - ./manifest.sh relay latest - - # pushd ../../ - # cargo publish - # popd -fi diff --git a/docker/prod/docker-compose.yml b/docker/prod/docker-compose.yml index 7201789..7fe2704 100644 --- a/docker/prod/docker-compose.yml +++ b/docker/prod/docker-compose.yml @@ -2,7 +2,7 @@ version: '3.3' services: relay: - image: asonix/relay:v0.3.8 + image: asonix/relay:v0.3.73 ports: - "8079:8079" restart: always @@ -14,6 +14,7 @@ services: - RESTRICTED_MODE=false - VALIDATE_SIGNATURES=true - HTTPS=true - - DATABASE_URL=postgres://pg_user:pg_pass@pg_host:pg_port/pg_database + - SLED_PATH=/mnt/sled/db-0.34 - PRETTY_LOG=false - PUBLISH_BLOCKS=true + - API_TOKEN=somepasswordishtoken diff --git a/docker/prod/manifest.sh b/docker/prod/manifest.sh deleted file mode 100755 index d426a97..0000000 --- a/docker/prod/manifest.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env bash - -function require() { - if [ "$1" = "" ]; then - echo "input '$2' required" - print_help - exit 1 - fi -} -function print_help() { - echo "deploy.sh" - echo "" - echo "Usage:" - echo " manifest.sh [repo] [tag]" - echo "" - echo "Args:" - echo " repo: The docker repository to update" - echo " tag: The git tag to be applied to the image manifest" -} - -REPO=$1 -TAG=$2 - -require "$REPO" "repo" -require "$TAG" "tag" - -set -xe - -sudo docker manifest create asonix/$REPO:$TAG \ - -a asonix/$REPO:arm64v8-$TAG \ - -a asonix/$REPO:arm32v7-$TAG \ - -a asonix/$REPO:amd64-$TAG - -sudo docker manifest annotate asonix/$REPO:$TAG \ - asonix/$REPO:arm64v8-$TAG --os linux --arch arm64 --variant v8 - -sudo docker manifest annotate asonix/$REPO:$TAG \ - asonix/$REPO:arm32v7-$TAG --os linux --arch arm --variant v7 - -sudo docker manifest annotate asonix/$REPO:$TAG \ - asonix/$REPO:amd64-$TAG --os linux --arch amd64 - -sudo docker manifest push asonix/$REPO:$TAG --purge diff --git a/src/admin.rs b/src/admin.rs index e7fc665..156a5d9 100644 --- a/src/admin.rs +++ b/src/admin.rs @@ -1,4 +1,6 @@ use activitystreams::iri_string::types::IriString; +use std::collections::{BTreeMap, BTreeSet}; +use time::OffsetDateTime; pub mod client; pub mod routes; @@ -22,3 +24,9 @@ pub(crate) struct BlockedDomains { pub(crate) struct ConnectedActors { pub(crate) connected_actors: Vec, } + +#[derive(serde::Deserialize, serde::Serialize)] +pub(crate) struct LastSeen { + pub(crate) last_seen: BTreeMap>, + pub(crate) never: Vec, +} diff --git a/src/admin/client.rs b/src/admin/client.rs index 3602487..fdb1687 100644 --- a/src/admin/client.rs +++ b/src/admin/client.rs @@ -1,5 +1,5 @@ use crate::{ - admin::{AllowedDomains, BlockedDomains, ConnectedActors, Domains}, + admin::{AllowedDomains, BlockedDomains, ConnectedActors, Domains, LastSeen}, collector::Snapshot, config::{AdminUrlKind, Config}, error::{Error, ErrorKind}, @@ -55,6 +55,10 @@ pub(crate) async fn stats(client: &Client, config: &Config) -> Result Result { + get_results(client, config, AdminUrlKind::LastSeen).await +} + async fn get_results( client: &Client, config: &Config, diff --git a/src/admin/routes.rs b/src/admin/routes.rs index 6578bfd..c13a6e3 100644 --- a/src/admin/routes.rs +++ b/src/admin/routes.rs @@ -1,5 +1,5 @@ use crate::{ - admin::{AllowedDomains, BlockedDomains, ConnectedActors, Domains}, + admin::{AllowedDomains, BlockedDomains, ConnectedActors, Domains, LastSeen}, collector::{MemoryCollector, Snapshot}, error::Error, extractors::Admin, @@ -8,6 +8,8 @@ use actix_web::{ web::{Data, Json}, HttpResponse, }; +use std::collections::{BTreeMap, BTreeSet}; +use time::OffsetDateTime; pub(crate) async fn allow( admin: Admin, @@ -69,3 +71,20 @@ pub(crate) async fn stats( ) -> Result, Error> { Ok(Json(collector.snapshot())) } + +pub(crate) async fn last_seen(admin: Admin) -> Result, Error> { + let nodes = admin.db_ref().last_seen().await?; + + let mut last_seen: BTreeMap> = BTreeMap::new(); + let mut never = Vec::new(); + + for (domain, datetime) in nodes { + if let Some(datetime) = datetime { + last_seen.entry(datetime).or_default().insert(domain); + } else { + never.push(domain); + } + } + + Ok(Json(LastSeen { last_seen, never })) +} diff --git a/src/apub.rs b/src/apub.rs index 2b922e3..a99cacc 100644 --- a/src/apub.rs +++ b/src/apub.rs @@ -34,11 +34,13 @@ pub struct PublicKey { #[serde(rename_all = "PascalCase")] pub enum ValidTypes { Accept, + Add, Announce, Create, Delete, Follow, Reject, + Remove, Undo, Update, } diff --git a/src/args.rs b/src/args.rs index 18a4059..155b296 100644 --- a/src/args.rs +++ b/src/args.rs @@ -17,11 +17,22 @@ pub(crate) struct Args { #[arg(short, long, help = "Get statistics from the server")] stats: bool, + + #[arg( + short, + long, + help = "List domains by when they were last succesfully contacted" + )] + contacted: bool, } impl Args { pub(crate) fn any(&self) -> bool { - !self.blocks.is_empty() || !self.allowed.is_empty() || self.list || self.stats + !self.blocks.is_empty() + || !self.allowed.is_empty() + || self.list + || self.stats + || self.contacted } pub(crate) fn new() -> Self { @@ -47,4 +58,8 @@ impl Args { pub(crate) fn stats(&self) -> bool { self.stats } + + pub(crate) fn contacted(&self) -> bool { + self.contacted + } } diff --git a/src/build.rs b/src/build.rs index 8db87a2..ebac904 100644 --- a/src/build.rs +++ b/src/build.rs @@ -6,6 +6,7 @@ fn git_info() { if output.status.success() { let git_hash = String::from_utf8_lossy(&output.stdout); println!("cargo:rustc-env=GIT_HASH={}", git_hash); + println!("cargo:rustc-env=GIT_SHORT_HASH={}", &git_hash[..8]) } } @@ -23,7 +24,7 @@ fn git_info() { fn version_info() -> Result<(), anyhow::Error> { let cargo_toml = Path::new(&std::env::var("CARGO_MANIFEST_DIR")?).join("Cargo.toml"); - let mut file = File::open(&cargo_toml)?; + let mut file = File::open(cargo_toml)?; let mut cargo_data = String::new(); file.read_to_string(&mut cargo_data)?; diff --git a/src/collector.rs b/src/collector.rs index 0d3536d..b2758b0 100644 --- a/src/collector.rs +++ b/src/collector.rs @@ -1,414 +1,5 @@ -use metrics::{Key, Recorder, SetRecorderError}; -use metrics_util::{ - registry::{AtomicStorage, GenerationalStorage, Recency, Registry}, - MetricKindMask, Summary, -}; -use quanta::Clock; -use std::{ - collections::{BTreeMap, HashMap}, - sync::{atomic::Ordering, Arc, RwLock}, - time::Duration, -}; +mod double; +mod stats; -const SECONDS: u64 = 1; -const MINUTES: u64 = 60 * SECONDS; -const HOURS: u64 = 60 * MINUTES; -const DAYS: u64 = 24 * HOURS; - -type DistributionMap = BTreeMap, Summary>; - -#[derive(Clone)] -pub struct MemoryCollector { - inner: Arc, -} - -struct Inner { - descriptions: RwLock>, - distributions: RwLock>, - recency: Recency, - registry: Registry>, -} - -#[derive(Debug, serde::Deserialize, serde::Serialize)] -struct Counter { - labels: BTreeMap, - value: u64, -} - -impl std::fmt::Display for Counter { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let labels = self - .labels - .iter() - .map(|(k, v)| format!("{}: {}", k, v)) - .collect::>() - .join(", "); - - write!(f, "{} - {}", labels, self.value) - } -} - -#[derive(Debug, serde::Deserialize, serde::Serialize)] -struct Gauge { - labels: BTreeMap, - value: f64, -} - -impl std::fmt::Display for Gauge { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let labels = self - .labels - .iter() - .map(|(k, v)| format!("{}: {}", k, v)) - .collect::>() - .join(", "); - - write!(f, "{} - {}", labels, self.value) - } -} - -#[derive(Debug, serde::Deserialize, serde::Serialize)] -struct Histogram { - labels: BTreeMap, - value: Vec<(f64, Option)>, -} - -impl std::fmt::Display for Histogram { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let labels = self - .labels - .iter() - .map(|(k, v)| format!("{}: {}", k, v)) - .collect::>() - .join(", "); - - let value = self - .value - .iter() - .map(|(k, v)| { - if let Some(v) = v { - format!("{}: {:.6}", k, v) - } else { - format!("{}: None,", k) - } - }) - .collect::>() - .join(", "); - - write!(f, "{} - {}", labels, value) - } -} - -#[derive(Debug, serde::Deserialize, serde::Serialize)] -pub(crate) struct Snapshot { - counters: HashMap>, - gauges: HashMap>, - histograms: HashMap>, -} - -const PAIRS: [((&str, &str), &str); 2] = [ - ( - ( - "background-jobs.worker.started", - "background-jobs.worker.finished", - ), - "background-jobs.worker.running", - ), - ( - ( - "background-jobs.job.started", - "background-jobs.job.finished", - ), - "background-jobs.job.running", - ), -]; - -#[derive(Default)] -struct MergeCounter { - start: Option, - finish: Option, -} - -impl MergeCounter { - fn merge(self) -> Option { - match (self.start, self.finish) { - (Some(start), Some(end)) => Some(Counter { - labels: start.labels, - value: start.value.saturating_sub(end.value), - }), - (Some(only), None) => Some(only), - (None, Some(only)) => Some(Counter { - labels: only.labels, - value: 0, - }), - (None, None) => None, - } - } -} - -impl Snapshot { - pub(crate) fn present(self) { - if !self.counters.is_empty() { - println!("Counters"); - let mut merging = HashMap::new(); - for (key, counters) in self.counters { - if let Some(((start, _), name)) = PAIRS - .iter() - .find(|((start, finish), _)| *start == key || *finish == key) - { - let entry = merging.entry(name).or_insert_with(HashMap::new); - - for counter in counters { - let mut merge_counter = entry - .entry(counter.labels.clone()) - .or_insert_with(MergeCounter::default); - if key == *start { - merge_counter.start = Some(counter); - } else { - merge_counter.finish = Some(counter); - } - } - - continue; - } - - println!("\t{}", key); - for counter in counters { - println!("\t\t{}", counter); - } - } - - for (key, counters) in merging { - println!("\t{}", key); - - for (_, counter) in counters { - if let Some(counter) = counter.merge() { - println!("\t\t{}", counter); - } - } - } - } - - if !self.gauges.is_empty() { - println!("Gauges"); - for (key, gauges) in self.gauges { - println!("\t{}", key); - - for gauge in gauges { - println!("\t\t{}", gauge); - } - } - } - - if !self.histograms.is_empty() { - println!("Histograms"); - for (key, histograms) in self.histograms { - println!("\t{}", key); - - for histogram in histograms { - println!("\t\t{}", histogram); - } - } - } - } -} - -fn key_to_parts(key: &Key) -> (String, Vec<(String, String)>) { - let labels = key - .labels() - .into_iter() - .map(|label| (label.key().to_string(), label.value().to_string())) - .collect(); - let name = key.name().to_string(); - (name, labels) -} - -impl Inner { - fn snapshot_counters(&self) -> HashMap> { - let mut counters = HashMap::new(); - - for (key, counter) in self.registry.get_counter_handles() { - let gen = counter.get_generation(); - if !self.recency.should_store_counter(&key, gen, &self.registry) { - continue; - } - - let (name, labels) = key_to_parts(&key); - let value = counter.get_inner().load(Ordering::Acquire); - counters.entry(name).or_insert_with(Vec::new).push(Counter { - labels: labels.into_iter().collect(), - value, - }); - } - - counters - } - - fn snapshot_gauges(&self) -> HashMap> { - let mut gauges = HashMap::new(); - - for (key, gauge) in self.registry.get_gauge_handles() { - let gen = gauge.get_generation(); - if !self.recency.should_store_gauge(&key, gen, &self.registry) { - continue; - } - - let (name, labels) = key_to_parts(&key); - let value = f64::from_bits(gauge.get_inner().load(Ordering::Acquire)); - gauges.entry(name).or_insert_with(Vec::new).push(Gauge { - labels: labels.into_iter().collect(), - value, - }) - } - - gauges - } - - fn snapshot_histograms(&self) -> HashMap> { - for (key, histogram) in self.registry.get_histogram_handles() { - let gen = histogram.get_generation(); - let (name, labels) = key_to_parts(&key); - - if !self - .recency - .should_store_histogram(&key, gen, &self.registry) - { - let mut d = self.distributions.write().unwrap(); - let delete_by_name = if let Some(by_name) = d.get_mut(&name) { - by_name.remove(&labels); - by_name.is_empty() - } else { - false - }; - drop(d); - - if delete_by_name { - self.descriptions.write().unwrap().remove(&name); - } - - continue; - } - - let mut d = self.distributions.write().unwrap(); - let outer_entry = d.entry(name.clone()).or_insert_with(BTreeMap::new); - - let entry = outer_entry - .entry(labels) - .or_insert_with(Summary::with_defaults); - - histogram.get_inner().clear_with(|samples| { - for sample in samples { - entry.add(*sample); - } - }) - } - - let d = self.distributions.read().unwrap().clone(); - d.into_iter() - .map(|(key, value)| { - ( - key, - value - .into_iter() - .map(|(labels, summary)| Histogram { - labels: labels.into_iter().collect(), - value: [0.001, 0.01, 0.05, 0.1, 0.5, 0.9, 0.99, 1.0] - .into_iter() - .map(|q| (q, summary.quantile(q))) - .collect(), - }) - .collect(), - ) - }) - .collect() - } - - fn snapshot(&self) -> Snapshot { - Snapshot { - counters: self.snapshot_counters(), - gauges: self.snapshot_gauges(), - histograms: self.snapshot_histograms(), - } - } -} - -impl MemoryCollector { - pub(crate) fn new() -> Self { - MemoryCollector { - inner: Arc::new(Inner { - descriptions: Default::default(), - distributions: Default::default(), - recency: Recency::new( - Clock::new(), - MetricKindMask::ALL, - Some(Duration::from_secs(5 * DAYS)), - ), - registry: Registry::new(GenerationalStorage::atomic()), - }), - } - } - - pub(crate) fn install(&self) -> Result<(), SetRecorderError> { - metrics::set_boxed_recorder(Box::new(self.clone())) - } - - pub(crate) fn snapshot(&self) -> Snapshot { - self.inner.snapshot() - } - - fn add_description_if_missing( - &self, - key: &metrics::KeyName, - description: metrics::SharedString, - ) { - let mut d = self.inner.descriptions.write().unwrap(); - d.entry(key.as_str().to_owned()).or_insert(description); - } -} - -impl Recorder for MemoryCollector { - fn describe_counter( - &self, - key: metrics::KeyName, - _: Option, - description: metrics::SharedString, - ) { - self.add_description_if_missing(&key, description) - } - - fn describe_gauge( - &self, - key: metrics::KeyName, - _: Option, - description: metrics::SharedString, - ) { - self.add_description_if_missing(&key, description) - } - - fn describe_histogram( - &self, - key: metrics::KeyName, - _: Option, - description: metrics::SharedString, - ) { - self.add_description_if_missing(&key, description) - } - - fn register_counter(&self, key: &Key) -> metrics::Counter { - self.inner - .registry - .get_or_create_counter(key, |c| c.clone().into()) - } - - fn register_gauge(&self, key: &Key) -> metrics::Gauge { - self.inner - .registry - .get_or_create_gauge(key, |c| c.clone().into()) - } - - fn register_histogram(&self, key: &Key) -> metrics::Histogram { - self.inner - .registry - .get_or_create_histogram(key, |c| c.clone().into()) - } -} +pub(crate) use double::DoubleRecorder; +pub(crate) use stats::{MemoryCollector, Snapshot}; diff --git a/src/collector/double.rs b/src/collector/double.rs new file mode 100644 index 0000000..b78dbff --- /dev/null +++ b/src/collector/double.rs @@ -0,0 +1,133 @@ +use metrics::{CounterFn, GaugeFn, HistogramFn, Key, Recorder, SetRecorderError}; +use std::sync::Arc; + +#[derive(Clone)] +pub(crate) struct DoubleRecorder { + first: R, + second: S, +} + +struct DoubleCounter { + first: metrics::Counter, + second: metrics::Counter, +} + +struct DoubleGauge { + first: metrics::Gauge, + second: metrics::Gauge, +} + +struct DoubleHistogram { + first: metrics::Histogram, + second: metrics::Histogram, +} + +impl DoubleRecorder { + pub(crate) fn new(first: R, second: S) -> Self { + DoubleRecorder { first, second } + } + + pub(crate) fn install(self) -> Result<(), SetRecorderError> + where + R: Recorder + 'static, + S: Recorder + 'static, + { + metrics::set_boxed_recorder(Box::new(self)) + } +} + +impl Recorder for DoubleRecorder +where + R: Recorder, + S: Recorder, +{ + fn describe_counter( + &self, + key: metrics::KeyName, + unit: Option, + description: metrics::SharedString, + ) { + self.first + .describe_counter(key.clone(), unit, description.clone()); + self.second.describe_counter(key, unit, description); + } + + fn describe_gauge( + &self, + key: metrics::KeyName, + unit: Option, + description: metrics::SharedString, + ) { + self.first + .describe_gauge(key.clone(), unit, description.clone()); + self.second.describe_gauge(key, unit, description); + } + + fn describe_histogram( + &self, + key: metrics::KeyName, + unit: Option, + description: metrics::SharedString, + ) { + self.first + .describe_histogram(key.clone(), unit, description.clone()); + self.second.describe_histogram(key, unit, description); + } + + fn register_counter(&self, key: &Key) -> metrics::Counter { + let first = self.first.register_counter(key); + let second = self.second.register_counter(key); + + metrics::Counter::from_arc(Arc::new(DoubleCounter { first, second })) + } + + fn register_gauge(&self, key: &Key) -> metrics::Gauge { + let first = self.first.register_gauge(key); + let second = self.second.register_gauge(key); + + metrics::Gauge::from_arc(Arc::new(DoubleGauge { first, second })) + } + + fn register_histogram(&self, key: &Key) -> metrics::Histogram { + let first = self.first.register_histogram(key); + let second = self.second.register_histogram(key); + + metrics::Histogram::from_arc(Arc::new(DoubleHistogram { first, second })) + } +} + +impl CounterFn for DoubleCounter { + fn increment(&self, value: u64) { + self.first.increment(value); + self.second.increment(value); + } + + fn absolute(&self, value: u64) { + self.first.absolute(value); + self.second.absolute(value); + } +} + +impl GaugeFn for DoubleGauge { + fn increment(&self, value: f64) { + self.first.increment(value); + self.second.increment(value); + } + + fn decrement(&self, value: f64) { + self.first.decrement(value); + self.second.decrement(value); + } + + fn set(&self, value: f64) { + self.first.set(value); + self.second.set(value); + } +} + +impl HistogramFn for DoubleHistogram { + fn record(&self, value: f64) { + self.first.record(value); + self.second.record(value); + } +} diff --git a/src/collector/stats.rs b/src/collector/stats.rs new file mode 100644 index 0000000..c8d1812 --- /dev/null +++ b/src/collector/stats.rs @@ -0,0 +1,414 @@ +use metrics::{Key, Recorder, SetRecorderError}; +use metrics_util::{ + registry::{AtomicStorage, GenerationalStorage, Recency, Registry}, + MetricKindMask, Summary, +}; +use quanta::Clock; +use std::{ + collections::{BTreeMap, HashMap}, + sync::{atomic::Ordering, Arc, RwLock}, + time::Duration, +}; + +const SECONDS: u64 = 1; +const MINUTES: u64 = 60 * SECONDS; +const HOURS: u64 = 60 * MINUTES; +const DAYS: u64 = 24 * HOURS; + +type DistributionMap = BTreeMap, Summary>; + +#[derive(Clone)] +pub struct MemoryCollector { + inner: Arc, +} + +struct Inner { + descriptions: RwLock>, + distributions: RwLock>, + recency: Recency, + registry: Registry>, +} + +#[derive(Debug, serde::Deserialize, serde::Serialize)] +struct Counter { + labels: BTreeMap, + value: u64, +} + +impl std::fmt::Display for Counter { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let labels = self + .labels + .iter() + .map(|(k, v)| format!("{}: {}", k, v)) + .collect::>() + .join(", "); + + write!(f, "{} - {}", labels, self.value) + } +} + +#[derive(Debug, serde::Deserialize, serde::Serialize)] +struct Gauge { + labels: BTreeMap, + value: f64, +} + +impl std::fmt::Display for Gauge { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let labels = self + .labels + .iter() + .map(|(k, v)| format!("{}: {}", k, v)) + .collect::>() + .join(", "); + + write!(f, "{} - {}", labels, self.value) + } +} + +#[derive(Debug, serde::Deserialize, serde::Serialize)] +struct Histogram { + labels: BTreeMap, + value: Vec<(f64, Option)>, +} + +impl std::fmt::Display for Histogram { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let labels = self + .labels + .iter() + .map(|(k, v)| format!("{}: {}", k, v)) + .collect::>() + .join(", "); + + let value = self + .value + .iter() + .map(|(k, v)| { + if let Some(v) = v { + format!("{}: {:.6}", k, v) + } else { + format!("{}: None,", k) + } + }) + .collect::>() + .join(", "); + + write!(f, "{} - {}", labels, value) + } +} + +#[derive(Debug, serde::Deserialize, serde::Serialize)] +pub(crate) struct Snapshot { + counters: HashMap>, + gauges: HashMap>, + histograms: HashMap>, +} + +const PAIRS: [((&str, &str), &str); 2] = [ + ( + ( + "background-jobs.worker.started", + "background-jobs.worker.finished", + ), + "background-jobs.worker.running", + ), + ( + ( + "background-jobs.job.started", + "background-jobs.job.finished", + ), + "background-jobs.job.running", + ), +]; + +#[derive(Default)] +struct MergeCounter { + start: Option, + finish: Option, +} + +impl MergeCounter { + fn merge(self) -> Option { + match (self.start, self.finish) { + (Some(start), Some(end)) => Some(Counter { + labels: start.labels, + value: start.value.saturating_sub(end.value), + }), + (Some(only), None) => Some(only), + (None, Some(only)) => Some(Counter { + labels: only.labels, + value: 0, + }), + (None, None) => None, + } + } +} + +impl Snapshot { + pub(crate) fn present(self) { + if !self.counters.is_empty() { + println!("Counters"); + let mut merging = HashMap::new(); + for (key, counters) in self.counters { + if let Some(((start, _), name)) = PAIRS + .iter() + .find(|((start, finish), _)| *start == key || *finish == key) + { + let entry = merging.entry(name).or_insert_with(HashMap::new); + + for counter in counters { + let mut merge_counter = entry + .entry(counter.labels.clone()) + .or_insert_with(MergeCounter::default); + if key == *start { + merge_counter.start = Some(counter); + } else { + merge_counter.finish = Some(counter); + } + } + + continue; + } + + println!("\t{}", key); + for counter in counters { + println!("\t\t{}", counter); + } + } + + for (key, counters) in merging { + println!("\t{}", key); + + for (_, counter) in counters { + if let Some(counter) = counter.merge() { + println!("\t\t{}", counter); + } + } + } + } + + if !self.gauges.is_empty() { + println!("Gauges"); + for (key, gauges) in self.gauges { + println!("\t{}", key); + + for gauge in gauges { + println!("\t\t{}", gauge); + } + } + } + + if !self.histograms.is_empty() { + println!("Histograms"); + for (key, histograms) in self.histograms { + println!("\t{}", key); + + for histogram in histograms { + println!("\t\t{}", histogram); + } + } + } + } +} + +fn key_to_parts(key: &Key) -> (String, Vec<(String, String)>) { + let labels = key + .labels() + .into_iter() + .map(|label| (label.key().to_string(), label.value().to_string())) + .collect(); + let name = key.name().to_string(); + (name, labels) +} + +impl Inner { + fn snapshot_counters(&self) -> HashMap> { + let mut counters = HashMap::new(); + + for (key, counter) in self.registry.get_counter_handles() { + let gen = counter.get_generation(); + if !self.recency.should_store_counter(&key, gen, &self.registry) { + continue; + } + + let (name, labels) = key_to_parts(&key); + let value = counter.get_inner().load(Ordering::Acquire); + counters.entry(name).or_insert_with(Vec::new).push(Counter { + labels: labels.into_iter().collect(), + value, + }); + } + + counters + } + + fn snapshot_gauges(&self) -> HashMap> { + let mut gauges = HashMap::new(); + + for (key, gauge) in self.registry.get_gauge_handles() { + let gen = gauge.get_generation(); + if !self.recency.should_store_gauge(&key, gen, &self.registry) { + continue; + } + + let (name, labels) = key_to_parts(&key); + let value = f64::from_bits(gauge.get_inner().load(Ordering::Acquire)); + gauges.entry(name).or_insert_with(Vec::new).push(Gauge { + labels: labels.into_iter().collect(), + value, + }) + } + + gauges + } + + fn snapshot_histograms(&self) -> HashMap> { + for (key, histogram) in self.registry.get_histogram_handles() { + let gen = histogram.get_generation(); + let (name, labels) = key_to_parts(&key); + + if !self + .recency + .should_store_histogram(&key, gen, &self.registry) + { + let mut d = self.distributions.write().unwrap(); + let delete_by_name = if let Some(by_name) = d.get_mut(&name) { + by_name.remove(&labels); + by_name.is_empty() + } else { + false + }; + drop(d); + + if delete_by_name { + self.descriptions.write().unwrap().remove(&name); + } + + continue; + } + + let mut d = self.distributions.write().unwrap(); + let outer_entry = d.entry(name.clone()).or_insert_with(BTreeMap::new); + + let entry = outer_entry + .entry(labels) + .or_insert_with(Summary::with_defaults); + + histogram.get_inner().clear_with(|samples| { + for sample in samples { + entry.add(*sample); + } + }) + } + + let d = self.distributions.read().unwrap().clone(); + d.into_iter() + .map(|(key, value)| { + ( + key, + value + .into_iter() + .map(|(labels, summary)| Histogram { + labels: labels.into_iter().collect(), + value: [0.001, 0.01, 0.05, 0.1, 0.5, 0.9, 0.99, 1.0] + .into_iter() + .map(|q| (q, summary.quantile(q))) + .collect(), + }) + .collect(), + ) + }) + .collect() + } + + fn snapshot(&self) -> Snapshot { + Snapshot { + counters: self.snapshot_counters(), + gauges: self.snapshot_gauges(), + histograms: self.snapshot_histograms(), + } + } +} + +impl MemoryCollector { + pub(crate) fn new() -> Self { + MemoryCollector { + inner: Arc::new(Inner { + descriptions: Default::default(), + distributions: Default::default(), + recency: Recency::new( + Clock::new(), + MetricKindMask::ALL, + Some(Duration::from_secs(5 * DAYS)), + ), + registry: Registry::new(GenerationalStorage::atomic()), + }), + } + } + + pub(crate) fn snapshot(&self) -> Snapshot { + self.inner.snapshot() + } + + fn add_description_if_missing( + &self, + key: &metrics::KeyName, + description: metrics::SharedString, + ) { + let mut d = self.inner.descriptions.write().unwrap(); + d.entry(key.as_str().to_owned()).or_insert(description); + } + + pub(crate) fn install(&self) -> Result<(), SetRecorderError> { + metrics::set_boxed_recorder(Box::new(self.clone())) + } +} + +impl Recorder for MemoryCollector { + fn describe_counter( + &self, + key: metrics::KeyName, + _: Option, + description: metrics::SharedString, + ) { + self.add_description_if_missing(&key, description) + } + + fn describe_gauge( + &self, + key: metrics::KeyName, + _: Option, + description: metrics::SharedString, + ) { + self.add_description_if_missing(&key, description) + } + + fn describe_histogram( + &self, + key: metrics::KeyName, + _: Option, + description: metrics::SharedString, + ) { + self.add_description_if_missing(&key, description) + } + + fn register_counter(&self, key: &Key) -> metrics::Counter { + self.inner + .registry + .get_or_create_counter(key, |c| c.clone().into()) + } + + fn register_gauge(&self, key: &Key) -> metrics::Gauge { + self.inner + .registry + .get_or_create_gauge(key, |c| c.clone().into()) + } + + fn register_histogram(&self, key: &Key) -> metrics::Histogram { + self.inner + .registry + .get_or_create_histogram(key, |c| c.clone().into()) + } +} diff --git a/src/config.rs b/src/config.rs index 1d4cf86..848cc5b 100644 --- a/src/config.rs +++ b/src/config.rs @@ -1,22 +1,24 @@ use crate::{ - data::{ActorCache, State}, error::Error, extractors::{AdminConfig, XApiToken}, - middleware::MyVerify, - requests::Requests, }; use activitystreams::{ iri, iri_string::{ + format::ToDedicatedString, resolve::FixedBaseResolver, types::{IriAbsoluteString, IriFragmentStr, IriRelativeStr, IriString}, }, }; use config::Environment; -use http_signature_normalization_actix::prelude::{VerifyDigest, VerifySignature}; +use http_signature_normalization_actix::prelude::VerifyDigest; use rustls::{Certificate, PrivateKey}; use sha2::{Digest, Sha256}; -use std::{io::BufReader, net::IpAddr, path::PathBuf}; +use std::{ + io::BufReader, + net::{IpAddr, SocketAddr}, + path::PathBuf, +}; use uuid::Uuid; #[derive(Clone, Debug, serde::Deserialize)] @@ -31,6 +33,7 @@ pub(crate) struct ParsedConfig { publish_blocks: bool, sled_path: PathBuf, source_repo: IriString, + repository_commit_base: String, opentelemetry_url: Option, telegram_token: Option, telegram_admin_handle: Option, @@ -40,6 +43,8 @@ pub(crate) struct ParsedConfig { footer_blurb: Option, local_domains: Option, local_blurb: Option, + prometheus_addr: Option, + prometheus_port: Option, } #[derive(Clone)] @@ -62,6 +67,7 @@ pub struct Config { footer_blurb: Option, local_domains: Vec, local_blurb: Option, + prometheus_config: Option, } #[derive(Clone)] @@ -70,6 +76,12 @@ struct TlsConfig { cert: PathBuf, } +#[derive(Clone, Debug)] +struct PrometheusConfig { + addr: IpAddr, + port: u16, +} + #[derive(Debug)] pub enum UrlKind { Activity, @@ -94,6 +106,7 @@ pub enum AdminUrlKind { Blocked, Connected, Stats, + LastSeen, } impl std::fmt::Debug for Config { @@ -121,6 +134,7 @@ impl std::fmt::Debug for Config { .field("footer_blurb", &self.footer_blurb) .field("local_domains", &self.local_domains) .field("local_blurb", &self.local_blurb) + .field("prometheus_config", &self.prometheus_config) .finish() } } @@ -138,6 +152,7 @@ impl Config { .set_default("publish_blocks", false)? .set_default("sled_path", "./sled/db-0-34")? .set_default("source_repo", "https://git.asonix.dog/asonix/relay")? + .set_default("repository_commit_base", "/src/commit/")? .set_default("opentelemetry_url", None as Option<&str>)? .set_default("telegram_token", None as Option<&str>)? .set_default("telegram_admin_handle", None as Option<&str>)? @@ -147,6 +162,8 @@ impl Config { .set_default("footer_blurb", None as Option<&str>)? .set_default("local_domains", None as Option<&str>)? .set_default("local_blurb", None as Option<&str>)? + .set_default("prometheus_addr", None as Option<&str>)? + .set_default("prometheus_port", None as Option)? .add_source(Environment::default()) .build()?; @@ -175,6 +192,29 @@ impl Config { .map(|d| d.to_string()) .collect(); + let prometheus_config = match (config.prometheus_addr, config.prometheus_port) { + (Some(addr), Some(port)) => Some(PrometheusConfig { addr, port }), + (Some(_), None) => { + tracing::warn!("PROMETHEUS_ADDR is set but PROMETHEUS_PORT is not set, not building Prometheus config"); + None + } + (None, Some(_)) => { + tracing::warn!("PROMETHEUS_PORT is set but PROMETHEUS_ADDR is not set, not building Prometheus config"); + None + } + (None, None) => None, + }; + + let source_url = match Self::git_hash() { + Some(hash) => format!( + "{}{}{}", + config.source_repo, config.repository_commit_base, hash + ) + .parse() + .expect("constructed source URL is valid"), + None => config.source_repo.clone(), + }; + Ok(Config { hostname: config.hostname, addr: config.addr, @@ -185,7 +225,7 @@ impl Config { publish_blocks: config.publish_blocks, base_uri, sled_path: config.sled_path, - source_repo: config.source_repo, + source_repo: source_url, opentelemetry_url: config.opentelemetry_url, telegram_token: config.telegram_token, telegram_admin_handle: config.telegram_admin_handle, @@ -194,9 +234,16 @@ impl Config { footer_blurb: config.footer_blurb, local_domains, local_blurb: config.local_blurb, + prometheus_config, }) } + pub(crate) fn prometheus_bind_address(&self) -> Option { + let config = self.prometheus_config.as_ref()?; + + Some((config.addr, config.port).into()) + } + pub(crate) fn open_keys(&self) -> Result, PrivateKey)>, Error> { let tls = if let Some(tls) = &self.tls { tls @@ -276,19 +323,6 @@ impl Config { } } - pub(crate) fn signature_middleware( - &self, - requests: Requests, - actors: ActorCache, - state: State, - ) -> VerifySignature { - if self.validate_signatures { - VerifySignature::new(MyVerify(requests, actors, state), Default::default()) - } else { - VerifySignature::new(MyVerify(requests, actors, state), Default::default()).optional() - } - } - pub(crate) fn x_api_token(&self) -> Option { self.api_token.clone().map(XApiToken::new) } @@ -345,7 +379,7 @@ impl Config { fn git_version() -> Option { let branch = Self::git_branch()?; - let hash = Self::git_hash()?; + let hash = Self::git_short_hash()?; Some(format!("{}-{}", branch, hash)) } @@ -366,6 +400,10 @@ impl Config { option_env!("GIT_HASH") } + fn git_short_hash() -> Option<&'static str> { + option_env!("GIT_SHORT_HASH") + } + pub(crate) fn user_agent(&self) -> String { format!( "{} ({}/{}; +{})", @@ -395,37 +433,44 @@ impl Config { self.do_generate_url(kind).expect("Generated valid IRI") } - #[tracing::instrument(level = "debug", skip_all, fields(base_uri = tracing::field::debug(&self.base_uri), kind = tracing::field::debug(&kind)))] fn do_generate_url(&self, kind: UrlKind) -> Result { let iri = match kind { - UrlKind::Activity => FixedBaseResolver::new(self.base_uri.as_ref()).try_resolve( - IriRelativeStr::new(&format!("activity/{}", Uuid::new_v4()))?.as_ref(), - )?, + UrlKind::Activity => FixedBaseResolver::new(self.base_uri.as_ref()) + .resolve(IriRelativeStr::new(&format!("activity/{}", Uuid::new_v4()))?.as_ref()) + .try_to_dedicated_string()?, UrlKind::Actor => FixedBaseResolver::new(self.base_uri.as_ref()) - .try_resolve(IriRelativeStr::new("actor")?.as_ref())?, + .resolve(IriRelativeStr::new("actor")?.as_ref()) + .try_to_dedicated_string()?, UrlKind::Followers => FixedBaseResolver::new(self.base_uri.as_ref()) - .try_resolve(IriRelativeStr::new("followers")?.as_ref())?, + .resolve(IriRelativeStr::new("followers")?.as_ref()) + .try_to_dedicated_string()?, UrlKind::Following => FixedBaseResolver::new(self.base_uri.as_ref()) - .try_resolve(IriRelativeStr::new("following")?.as_ref())?, + .resolve(IriRelativeStr::new("following")?.as_ref()) + .try_to_dedicated_string()?, UrlKind::Inbox => FixedBaseResolver::new(self.base_uri.as_ref()) - .try_resolve(IriRelativeStr::new("inbox")?.as_ref())?, + .resolve(IriRelativeStr::new("inbox")?.as_ref()) + .try_to_dedicated_string()?, UrlKind::Index => self.base_uri.clone().into(), UrlKind::MainKey => { let actor = IriRelativeStr::new("actor")?; let fragment = IriFragmentStr::new("main-key")?; - let mut resolved = - FixedBaseResolver::new(self.base_uri.as_ref()).try_resolve(actor.as_ref())?; + let mut resolved = FixedBaseResolver::new(self.base_uri.as_ref()) + .resolve(actor.as_ref()) + .try_to_dedicated_string()?; resolved.set_fragment(Some(fragment)); resolved } UrlKind::Media(uuid) => FixedBaseResolver::new(self.base_uri.as_ref()) - .try_resolve(IriRelativeStr::new(&format!("media/{}", uuid))?.as_ref())?, + .resolve(IriRelativeStr::new(&format!("media/{}", uuid))?.as_ref()) + .try_to_dedicated_string()?, UrlKind::NodeInfo => FixedBaseResolver::new(self.base_uri.as_ref()) - .try_resolve(IriRelativeStr::new("nodeinfo/2.0.json")?.as_ref())?, + .resolve(IriRelativeStr::new("nodeinfo/2.0.json")?.as_ref()) + .try_to_dedicated_string()?, UrlKind::Outbox => FixedBaseResolver::new(self.base_uri.as_ref()) - .try_resolve(IriRelativeStr::new("outbox")?.as_ref())?, + .resolve(IriRelativeStr::new("outbox")?.as_ref()) + .try_to_dedicated_string()?, }; Ok(iri) @@ -437,25 +482,22 @@ impl Config { } fn do_generate_admin_url(&self, kind: AdminUrlKind) -> Result { - let iri = match kind { - AdminUrlKind::Allow => FixedBaseResolver::new(self.base_uri.as_ref()) - .try_resolve(IriRelativeStr::new("api/v1/admin/allow")?.as_ref())?, - AdminUrlKind::Disallow => FixedBaseResolver::new(self.base_uri.as_ref()) - .try_resolve(IriRelativeStr::new("api/v1/admin/disallow")?.as_ref())?, - AdminUrlKind::Block => FixedBaseResolver::new(self.base_uri.as_ref()) - .try_resolve(IriRelativeStr::new("api/v1/admin/block")?.as_ref())?, - AdminUrlKind::Unblock => FixedBaseResolver::new(self.base_uri.as_ref()) - .try_resolve(IriRelativeStr::new("api/v1/admin/unblock")?.as_ref())?, - AdminUrlKind::Allowed => FixedBaseResolver::new(self.base_uri.as_ref()) - .try_resolve(IriRelativeStr::new("api/v1/admin/allowed")?.as_ref())?, - AdminUrlKind::Blocked => FixedBaseResolver::new(self.base_uri.as_ref()) - .try_resolve(IriRelativeStr::new("api/v1/admin/blocked")?.as_ref())?, - AdminUrlKind::Connected => FixedBaseResolver::new(self.base_uri.as_ref()) - .try_resolve(IriRelativeStr::new("api/v1/admin/connected")?.as_ref())?, - AdminUrlKind::Stats => FixedBaseResolver::new(self.base_uri.as_ref()) - .try_resolve(IriRelativeStr::new("api/v1/admin/stats")?.as_ref())?, + let path = match kind { + AdminUrlKind::Allow => "api/v1/admin/allow", + AdminUrlKind::Disallow => "api/v1/admin/disallow", + AdminUrlKind::Block => "api/v1/admin/block", + AdminUrlKind::Unblock => "api/v1/admin/unblock", + AdminUrlKind::Allowed => "api/v1/admin/allowed", + AdminUrlKind::Blocked => "api/v1/admin/blocked", + AdminUrlKind::Connected => "api/v1/admin/connected", + AdminUrlKind::Stats => "api/v1/admin/stats", + AdminUrlKind::LastSeen => "api/v1/admin/last_seen", }; + let iri = FixedBaseResolver::new(self.base_uri.as_ref()) + .resolve(IriRelativeStr::new(path)?.as_ref()) + .try_to_dedicated_string()?; + Ok(iri) } } diff --git a/src/data.rs b/src/data.rs index 918bdf7..1125149 100644 --- a/src/data.rs +++ b/src/data.rs @@ -1,9 +1,11 @@ mod actor; +mod last_online; mod media; mod node; mod state; pub(crate) use actor::ActorCache; +pub(crate) use last_online::LastOnline; pub(crate) use media::MediaCache; pub(crate) use node::{Node, NodeCache}; pub(crate) use state::State; diff --git a/src/data/actor.rs b/src/data/actor.rs index c0b3ddd..784e99f 100644 --- a/src/data/actor.rs +++ b/src/data/actor.rs @@ -37,7 +37,7 @@ impl ActorCache { ActorCache { db } } - #[tracing::instrument(level = "debug" name = "Get Actor", skip_all, fields(id = id.to_string().as_str(), requests))] + #[tracing::instrument(level = "debug" name = "Get Actor", skip_all, fields(id = id.to_string().as_str()))] pub(crate) async fn get( &self, id: &IriString, @@ -56,12 +56,8 @@ impl ActorCache { #[tracing::instrument(level = "debug", name = "Add Connection", skip(self))] pub(crate) async fn add_connection(&self, actor: Actor) -> Result<(), Error> { - let add_connection = self.db.add_connection(actor.id.clone()); - let save_actor = self.db.save_actor(actor); - - tokio::try_join!(add_connection, save_actor)?; - - Ok(()) + self.db.add_connection(actor.id.clone()).await?; + self.db.save_actor(actor).await } #[tracing::instrument(level = "debug", name = "Remove Connection", skip(self))] @@ -69,7 +65,7 @@ impl ActorCache { self.db.remove_connection(actor.id.clone()).await } - #[tracing::instrument(level = "debug", name = "Fetch remote actor", skip_all, fields(id = id.to_string().as_str(), requests))] + #[tracing::instrument(level = "debug", name = "Fetch remote actor", skip_all, fields(id = id.to_string().as_str()))] pub(crate) async fn get_no_cache( &self, id: &IriString, diff --git a/src/data/last_online.rs b/src/data/last_online.rs new file mode 100644 index 0000000..889d804 --- /dev/null +++ b/src/data/last_online.rs @@ -0,0 +1,28 @@ +use activitystreams::iri_string::types::IriStr; +use std::{collections::HashMap, sync::Mutex}; +use time::OffsetDateTime; + +pub(crate) struct LastOnline { + domains: Mutex>, +} + +impl LastOnline { + pub(crate) fn mark_seen(&self, iri: &IriStr) { + if let Some(authority) = iri.authority_str() { + self.domains + .lock() + .unwrap() + .insert(authority.to_string(), OffsetDateTime::now_utc()); + } + } + + pub(crate) fn take(&self) -> HashMap { + std::mem::take(&mut *self.domains.lock().unwrap()) + } + + pub(crate) fn empty() -> Self { + Self { + domains: Mutex::new(HashMap::default()), + } + } +} diff --git a/src/data/node.rs b/src/data/node.rs index 815cc7c..4ba6eb5 100644 --- a/src/data/node.rs +++ b/src/data/node.rs @@ -36,11 +36,9 @@ impl NodeCache { #[tracing::instrument(level = "debug", name = "Get nodes", skip(self))] pub(crate) async fn nodes(&self) -> Result, Error> { - let infos = self.db.connected_info(); - let instances = self.db.connected_instance(); - let contacts = self.db.connected_contact(); - - let (infos, instances, contacts) = tokio::try_join!(infos, instances, contacts)?; + let infos = self.db.connected_info().await?; + let instances = self.db.connected_instance().await?; + let contacts = self.db.connected_contact().await?; let vec = self .db diff --git a/src/data/state.rs b/src/data/state.rs index 93a3f9a..4387974 100644 --- a/src/data/state.rs +++ b/src/data/state.rs @@ -10,8 +10,9 @@ use actix_web::web; use lru::LruCache; use rand::thread_rng; use rsa::{RsaPrivateKey, RsaPublicKey}; -use std::sync::Arc; -use tokio::sync::RwLock; +use std::sync::{Arc, RwLock}; + +use super::LastOnline; #[derive(Clone)] pub struct State { @@ -20,6 +21,7 @@ pub struct State { object_cache: Arc>>, node_cache: NodeCache, breakers: Breakers, + pub(crate) last_online: Arc, pub(crate) db: Db, } @@ -44,6 +46,7 @@ impl State { self.private_key.clone(), config.user_agent(), self.breakers.clone(), + self.last_online.clone(), ) } @@ -78,12 +81,12 @@ impl State { .collect()) } - pub(crate) async fn is_cached(&self, object_id: &IriString) -> bool { - self.object_cache.read().await.contains(object_id) + pub(crate) fn is_cached(&self, object_id: &IriString) -> bool { + self.object_cache.read().unwrap().contains(object_id) } - pub(crate) async fn cache(&self, object_id: IriString, actor_id: IriString) { - self.object_cache.write().await.put(object_id, actor_id); + pub(crate) fn cache(&self, object_id: IriString, actor_id: IriString) { + self.object_cache.write().unwrap().put(object_id, actor_id); } #[tracing::instrument(level = "debug", name = "Building state", skip_all)] @@ -115,6 +118,7 @@ impl State { node_cache: NodeCache::new(db.clone()), breakers: Breakers::default(), db, + last_online: Arc::new(LastOnline::empty()), }; Ok(state) diff --git a/src/db.rs b/src/db.rs index 685405f..6d3064f 100644 --- a/src/db.rs +++ b/src/db.rs @@ -7,8 +7,13 @@ use rsa::{ pkcs8::{DecodePrivateKey, EncodePrivateKey}, RsaPrivateKey, }; -use sled::Tree; -use std::{collections::HashMap, sync::Arc, time::SystemTime}; +use sled::{Batch, Tree}; +use std::{ + collections::{BTreeMap, HashMap}, + sync::Arc, + time::SystemTime, +}; +use time::OffsetDateTime; use uuid::Uuid; #[derive(Clone, Debug)] @@ -28,6 +33,7 @@ struct Inner { actor_id_info: Tree, actor_id_instance: Tree, actor_id_contact: Tree, + last_seen: Tree, restricted_mode: bool, } @@ -247,6 +253,7 @@ impl Db { actor_id_info: db.open_tree("actor-id-info")?, actor_id_instance: db.open_tree("actor-id-instance")?, actor_id_contact: db.open_tree("actor-id-contact")?, + last_seen: db.open_tree("last-seen")?, restricted_mode, }), }) @@ -254,7 +261,7 @@ impl Db { async fn unblock( &self, - f: impl Fn(&Inner) -> Result + Send + 'static, + f: impl FnOnce(&Inner) -> Result + Send + 'static, ) -> Result where T: Send + 'static, @@ -266,6 +273,48 @@ impl Db { Ok(t) } + pub(crate) async fn mark_last_seen( + &self, + nodes: HashMap, + ) -> Result<(), Error> { + let mut batch = Batch::default(); + + for (domain, datetime) in nodes { + let datetime_string = serde_json::to_vec(&datetime)?; + + batch.insert(domain.as_bytes(), datetime_string); + } + + self.unblock(move |inner| inner.last_seen.apply_batch(batch).map_err(Error::from)) + .await + } + + pub(crate) async fn last_seen( + &self, + ) -> Result>, Error> { + self.unblock(|inner| { + let mut map = BTreeMap::new(); + + for iri in inner.connected() { + let Some(authority_str) = iri.authority_str() else { + continue; + }; + + if let Some(datetime) = inner.last_seen.get(authority_str)? { + map.insert( + authority_str.to_string(), + Some(serde_json::from_slice(&datetime)?), + ); + } else { + map.insert(authority_str.to_string(), None); + } + } + + Ok(map) + }) + .await + } + pub(crate) async fn connected_ids(&self) -> Result, Error> { self.unblock(|inner| Ok(inner.connected().collect())).await } diff --git a/src/error.rs b/src/error.rs index 8fcd05d..2f9e658 100644 --- a/src/error.rs +++ b/src/error.rs @@ -10,7 +10,7 @@ use std::{convert::Infallible, fmt::Debug, io}; use tracing_error::SpanTrace; pub(crate) struct Error { - context: SpanTrace, + context: String, kind: ErrorKind, } @@ -26,6 +26,10 @@ impl Error { pub(crate) fn is_bad_request(&self) -> bool { matches!(self.kind, ErrorKind::Status(_, StatusCode::BAD_REQUEST)) } + + pub(crate) fn is_gone(&self) -> bool { + matches!(self.kind, ErrorKind::Status(_, StatusCode::GONE)) + } } impl std::fmt::Debug for Error { @@ -53,7 +57,7 @@ where { fn from(error: T) -> Self { Error { - context: SpanTrace::capture(), + context: SpanTrace::capture().to_string(), kind: error.into(), } } @@ -77,10 +81,7 @@ pub(crate) enum ErrorKind { ParseIri(#[from] activitystreams::iri_string::validate::Error), #[error("Couldn't normalize IRI, {0}")] - NormalizeIri( - #[from] - activitystreams::iri_string::task::Error, - ), + NormalizeIri(#[from] std::collections::TryReserveError), #[error("Couldn't perform IO, {0}")] Io(#[from] io::Error), @@ -125,7 +126,7 @@ pub(crate) enum ErrorKind { BadActor(String, String), #[error("Signature verification is required, but no signature was given")] - NoSignature(String), + NoSignature(Option), #[error("Wrong ActivityPub kind, {0}")] Kind(String), @@ -196,7 +197,8 @@ impl ResponseError for Error { ErrorKind::Kind(_) | ErrorKind::MissingKind | ErrorKind::MissingId - | ErrorKind::ObjectCount => StatusCode::BAD_REQUEST, + | ErrorKind::ObjectCount + | ErrorKind::NoSignature(_) => StatusCode::BAD_REQUEST, _ => StatusCode::INTERNAL_SERVER_ERROR, } } diff --git a/src/extractors.rs b/src/extractors.rs index 572da34..f56b9af 100644 --- a/src/extractors.rs +++ b/src/extractors.rs @@ -80,7 +80,7 @@ impl Admin { #[derive(Debug, thiserror::Error)] #[error("Failed authentication")] pub(crate) struct Error { - context: SpanTrace, + context: String, #[source] kind: ErrorKind, } @@ -88,49 +88,49 @@ pub(crate) struct Error { impl Error { fn invalid() -> Self { Error { - context: SpanTrace::capture(), + context: SpanTrace::capture().to_string(), kind: ErrorKind::Invalid, } } fn missing_config() -> Self { Error { - context: SpanTrace::capture(), + context: SpanTrace::capture().to_string(), kind: ErrorKind::MissingConfig, } } fn missing_db() -> Self { Error { - context: SpanTrace::capture(), + context: SpanTrace::capture().to_string(), kind: ErrorKind::MissingDb, } } fn bcrypt_verify(e: BcryptError) -> Self { Error { - context: SpanTrace::capture(), + context: SpanTrace::capture().to_string(), kind: ErrorKind::BCryptVerify(e), } } fn bcrypt_hash(e: BcryptError) -> Self { Error { - context: SpanTrace::capture(), + context: SpanTrace::capture().to_string(), kind: ErrorKind::BCryptHash(e), } } fn parse_header(e: ParseError) -> Self { Error { - context: SpanTrace::capture(), + context: SpanTrace::capture().to_string(), kind: ErrorKind::ParseHeader(e), } } fn canceled(_: BlockingError) -> Self { Error { - context: SpanTrace::capture(), + context: SpanTrace::capture().to_string(), kind: ErrorKind::Canceled, } } diff --git a/src/jobs.rs b/src/jobs.rs index 014bd72..0bad1ac 100644 --- a/src/jobs.rs +++ b/src/jobs.rs @@ -5,6 +5,7 @@ mod deliver_many; mod instance; mod nodeinfo; mod process_listeners; +mod record_last_online; pub(crate) use self::{ contact::QueryContact, deliver::Deliver, deliver_many::DeliverMany, instance::QueryInstance, @@ -15,7 +16,7 @@ use crate::{ config::Config, data::{ActorCache, MediaCache, NodeCache, State}, error::{Error, ErrorKind}, - jobs::process_listeners::Listeners, + jobs::{process_listeners::Listeners, record_last_online::RecordLastOnline}, requests::Requests, }; use background_jobs::{ @@ -62,6 +63,7 @@ pub(crate) fn create_workers( .register::() .register::() .register::() + .register::() .register::() .register::() .register::() @@ -73,6 +75,7 @@ pub(crate) fn create_workers( .start_with_threads(parallelism); shared.every(Duration::from_secs(60 * 5), Listeners); + shared.every(Duration::from_secs(60 * 10), RecordLastOnline); let job_server = JobServer::new(shared.queue_handle().clone()); diff --git a/src/jobs/apub.rs b/src/jobs/apub.rs index d857b46..a5e195c 100644 --- a/src/jobs/apub.rs +++ b/src/jobs/apub.rs @@ -36,13 +36,13 @@ async fn get_inboxes( state.inboxes_without(&actor.inbox, &authority).await } -fn prepare_activity( +fn prepare_activity( mut t: T, id: impl TryInto, to: impl TryInto, ) -> Result where - T: ObjectExt + BaseExt, + T: ObjectExt + BaseExt, Error: From + From, { t.set_id(id.try_into()?) diff --git a/src/jobs/apub/announce.rs b/src/jobs/apub/announce.rs index 93dec67..480f8db 100644 --- a/src/jobs/apub/announce.rs +++ b/src/jobs/apub/announce.rs @@ -42,7 +42,7 @@ impl Announce { .queue(DeliverMany::new(inboxes, announce)?) .await?; - state.state.cache(self.object_id, activity_id).await; + state.state.cache(self.object_id, activity_id); Ok(()) } } diff --git a/src/jobs/instance.rs b/src/jobs/instance.rs index 4339845..2f1e5c7 100644 --- a/src/jobs/instance.rs +++ b/src/jobs/instance.rs @@ -148,8 +148,8 @@ struct Contact { mod tests { use super::Instance; - const ASONIX_INSTANCE: &'static str = r#"{"uri":"masto.asonix.dog","title":"asonix.dog","short_description":"The asonix of furry mastodon. For me and a few friends. DM me somewhere if u want an account lol","description":"A mastodon server that's only for me and nobody else sorry","email":"asonix@asonix.dog","version":"4.0.0rc2-asonix-changes","urls":{"streaming_api":"wss://masto.asonix.dog"},"stats":{"user_count":7,"status_count":12328,"domain_count":5146},"thumbnail":"https://masto.asonix.dog/system/site_uploads/files/000/000/002/@1x/32f51462a2b2bf2d.png","languages":["dog"],"registrations":false,"approval_required":false,"invites_enabled":false,"configuration":{"accounts":{"max_featured_tags":10},"statuses":{"max_characters":500,"max_media_attachments":4,"characters_reserved_per_url":23},"media_attachments":{"supported_mime_types":["image/jpeg","image/png","image/gif","image/heic","image/heif","image/webp","image/avif","video/webm","video/mp4","video/quicktime","video/ogg","audio/wave","audio/wav","audio/x-wav","audio/x-pn-wave","audio/vnd.wave","audio/ogg","audio/vorbis","audio/mpeg","audio/mp3","audio/webm","audio/flac","audio/aac","audio/m4a","audio/x-m4a","audio/mp4","audio/3gpp","video/x-ms-asf"],"image_size_limit":10485760,"image_matrix_limit":16777216,"video_size_limit":41943040,"video_frame_rate_limit":60,"video_matrix_limit":2304000},"polls":{"max_options":4,"max_characters_per_option":50,"min_expiration":300,"max_expiration":2629746}},"contact_account":{"id":"1","username":"asonix","acct":"asonix","display_name":"Liom on Mane :antiverified:","locked":true,"bot":false,"discoverable":true,"group":false,"created_at":"2021-02-09T00:00:00.000Z","note":"\u003cp\u003e26, local liom, friend, rust (lang) stan, bi \u003c/p\u003e\u003cp\u003eicon by \u003cspan class=\"h-card\"\u003e\u003ca href=\"https://furaffinity.net/user/lalupine\" target=\"blank\" rel=\"noopener noreferrer\" class=\"u-url mention\"\u003e@\u003cspan\u003elalupine@furaffinity.net\u003c/span\u003e\u003c/a\u003e\u003c/span\u003e\u003cbr /\u003eheader by \u003cspan class=\"h-card\"\u003e\u003ca href=\"https://furaffinity.net/user/tronixx\" target=\"blank\" rel=\"noopener noreferrer\" class=\"u-url mention\"\u003e@\u003cspan\u003etronixx@furaffinity.net\u003c/span\u003e\u003c/a\u003e\u003c/span\u003e\u003c/p\u003e\u003cp\u003eTestimonials:\u003c/p\u003e\u003cp\u003eStand: LIONS\u003cbr /\u003eStand User: AODE\u003cbr /\u003e- Keris (not on here)\u003c/p\u003e","url":"https://masto.asonix.dog/@asonix","avatar":"https://masto.asonix.dog/system/accounts/avatars/000/000/001/original/00852df0e6fee7e0.png","avatar_static":"https://masto.asonix.dog/system/accounts/avatars/000/000/001/original/00852df0e6fee7e0.png","header":"https://masto.asonix.dog/system/accounts/headers/000/000/001/original/8122ce3e5a745385.png","header_static":"https://masto.asonix.dog/system/accounts/headers/000/000/001/original/8122ce3e5a745385.png","followers_count":237,"following_count":474,"statuses_count":8798,"last_status_at":"2022-11-08","noindex":true,"emojis":[{"shortcode":"antiverified","url":"https://masto.asonix.dog/system/custom_emojis/images/000/030/053/original/bb0bc2e395b9a127.png","static_url":"https://masto.asonix.dog/system/custom_emojis/images/000/030/053/static/bb0bc2e395b9a127.png","visible_in_picker":true}],"fields":[{"name":"pronouns","value":"he/they","verified_at":null},{"name":"software","value":"bad","verified_at":null},{"name":"gitea","value":"\u003ca href=\"https://git.asonix.dog\" target=\"_blank\" rel=\"nofollow noopener noreferrer me\"\u003e\u003cspan class=\"invisible\"\u003ehttps://\u003c/span\u003e\u003cspan class=\"\"\u003egit.asonix.dog\u003c/span\u003e\u003cspan class=\"invisible\"\u003e\u003c/span\u003e\u003c/a\u003e","verified_at":null},{"name":"join my","value":"relay","verified_at":null}]},"rules":[]}"#; - const HYNET_INSTANCE: &'static str = r#"{"approval_required":false,"avatar_upload_limit":2000000,"background_image":"https://soc.hyena.network/images/city.jpg","background_upload_limit":4000000,"banner_upload_limit":4000000,"description":"Akkoma: The cooler fediverse server","description_limit":5000,"email":"me@hyena.network","languages":["en"],"max_toot_chars":"5000","pleroma":{"metadata":{"account_activation_required":true,"features":["pleroma_api","mastodon_api","mastodon_api_streaming","polls","v2_suggestions","pleroma_explicit_addressing","shareable_emoji_packs","multifetch","pleroma:api/v1/notifications:include_types_filter","chat","shout","relay","safe_dm_mentions","pleroma_emoji_reactions","pleroma_chat_messages","exposable_reactions","profile_directory","custom_emoji_reactions"],"federation":{"enabled":true,"exclusions":false,"mrf_hashtag":{"federated_timeline_removal":[],"reject":[],"sensitive":["nsfw"]},"mrf_policies":["SimplePolicy","EnsureRePrepended","HashtagPolicy"],"mrf_simple":{"accept":[],"avatar_removal":[],"banner_removal":[],"federated_timeline_removal":["botsin.space"],"followers_only":[],"media_nsfw":["mstdn.jp","wxw.moe","knzk.me","vipgirlfriend.xxx","humblr.social","switter.at","kinkyelephant.com","sinblr.com","kinky.business","rubber.social"],"media_removal":[],"reject":["*.10minutepleroma.com","101010.pl","13bells.com","2.distsn.org","2hu.club","2ndamendment.social","434.earth","4chan.icu","4qq.org","7td.org","80percent.social","a.nti.social","aaathats3as.com","accela.online","amala.schwartzwelt.xyz","angrytoday.com","anime.website","antitwitter.moe","antivaxxer.icu","archivefedifor.fun","artalley.social","bae.st","bajax.us","baraag.net","bbs.kawa-kun.com","beefyboys.club","beefyboys.win","bikeshed.party","bitcoinhackers.org","bleepp.com","blovice.bahnhof.cz","brighteon.social","buildthatwallandmakeamericagreatagain.trumpislovetrumpis.life","bungle.online","cawfee.club","censorship.icu","chungus.cc","club.darknight-coffee.org","clubcyberia.co","cock.fish","cock.li","comfyboy.club","contrapointsfan.club","coon.town","counter.social","cum.salon","d-fens.systems","definitely-not-archivefedifor.fun","degenerates.fail","desuposter.club","detroitriotcity.com","developer.gab.com","dogwhipping.day","eientei.org","enigmatic.observer","eveningzoo.club","exited.eu","federation.krowverse.services","fedi.cc","fedi.krowverse.services","fedi.pawlicker.com","fedi.vern.cc","freak.university","freeatlantis.com","freecumextremist.com","freesoftwareextremist.com","freespeech.firedragonstudios.com","freespeech.host","freespeechextremist.com","freevoice.space","freezepeach.xyz","froth.zone","fuckgov.org","gab.ai","gab.polaris-1.work","gab.protohype.net","gabfed.com","gameliberty.club","gearlandia.haus","gitmo.life","glindr.org","glittersluts.xyz","glowers.club","godspeed.moe","gorf.pub","goyim.app","gs.kawa-kun.com","hagra.net","hallsofamenti.io","hayu.sh","hentai.baby","honkwerx.tech","hunk.city","husk.site","iddqd.social","ika.moe","isexychat.space","jaeger.website","justicewarrior.social","kag.social","katiehopkinspolitical.icu","kiwifarms.cc","kiwifarms.is","kiwifarms.net","kohrville.net","koyu.space","kys.moe","lain.com","lain.sh","leafposter.club","lets.saynoto.lgbt","liberdon.com","libertarianism.club","ligma.pro","lolis.world","masochi.st","masthead.social","mastodon.digitalsuccess.dev","mastodon.fidonet.io","mastodon.grin.hu","mastodon.ml","midnightride.rs","milker.cafe","mobile.tmediatech.io","moon.holiday","mstdn.foxfam.club","mstdn.io","mstdn.starnix.network","mulmeyun.church","nazi.social","neckbeard.xyz","neenster.org","neko.ci","netzsphaere.xyz","newjack.city","nicecrew.digital","nnia.space","noagendasocial.com","norrebro.space","oursocialism.today","ovo.sc","pawoo.net","paypig.org","pedo.school","phreedom.tk","pieville.net","pkteerium.xyz","pl.murky.club","pl.spiderden.net","pl.tkammer.de","pl.zombiecats.run","pleroma.nobodyhasthe.biz","pleroma.runfox.tk","pleroma.site","plr.inferencium.net","pmth.us","poa.st","pod.vladtepesblog.com","political.icu","pooper.social","posting.lolicon.rocks","preteengirls.biz","prout.social","qoto.org","rage.lol","rakket.app","raplst.town","rdrama.cc","ryona.agency","s.sneak.berlin","seal.cafe","sealion.club","search.fedi.app","sementerrori.st","shitposter.club","shortstackran.ch","silkhe.art","sleepy.cafe","soc.mahodou.moe","soc.redeyes.site","social.076.ne.jp","social.anoxinon.de","social.chadland.net","social.freetalklive.com","social.getgle.org","social.handholding.io","social.headsca.la","social.imirhil.fr","social.lovingexpressions.net","social.manalejandro.com","social.midwaytrades.com","social.pseudo-whiskey.bar","social.targaryen.house","social.teci.world","societal.co","society.oftrolls.com","socks.pinnoto.org","socnet.supes.com","solagg.com","spinster.xyz","springbo.cc","stereophonic.space","sunshinegardens.org","theautisticinvestors.quest","thechad.zone","theduran.icu","theosis.church","toot.love","toots.alirezahayati.com","traboone.com","truthsocial.co.in","truthsocial.com","tuusin.misono-ya.info","tweety.icu","unbound.social","unsafe.space","varishangout.net","video.nobodyhasthe.biz","voicenews.icu","voluntaryism.club","waifu.social","weeaboo.space","whinge.town","wolfgirl.bar","workers.dev","wurm.host","xiii.ch","xn--p1abe3d.xn--80asehdb","yggdrasil.social","youjo.love"],"reject_deletes":[],"report_removal":[]},"mrf_simple_info":{"federated_timeline_removal":{"botsin.space":{"reason":"A lot of bot content"}},"media_nsfw":{"humblr.social":{"reason":"NSFW Instance, safe to assume most content is NSFW"},"kinky.business":{"reason":"NSFW Instance, safe to assume most content is NSFW"},"kinkyelephant.com":{"reason":"NSFW Instance, safe to assume most content is NSFW"},"knzk.me":{"reason":"Unmarked nsfw media"},"mstdn.jp":{"reason":"Not sure about the media policy"},"rubber.social":{"reason":"NSFW Instance, safe to assume most content is NSFW"},"sinblr.com":{"reason":"NSFW Instance, safe to assume most content is NSFW"},"switter.at":{"reason":"NSFW Instance, safe to assume most content is NSFW"},"vipgirlfriend.xxx":{"reason":"Unmarked nsfw media"},"wxw.moe":{"reason":"Unmarked nsfw media"}}},"quarantined_instances":[],"quarantined_instances_info":{"quarantined_instances":{}}},"fields_limits":{"max_fields":10,"max_remote_fields":20,"name_length":512,"value_length":2048},"post_formats":["text/plain","text/html","text/markdown","text/bbcode","text/x.misskeymarkdown"],"privileged_staff":false},"stats":{"mau":1},"vapid_public_key":"BMg4q-rT3rkMzc29F7OS5uM6t-Rx4HncMIB1NXrKwNlVRfX-W1kwgOuq5pDy-WhWmOZudaegftjBTCX3-pzdDFc"},"poll_limits":{"max_expiration":31536000,"max_option_chars":200,"max_options":20,"min_expiration":0},"registrations":"FALSE","shout_limit":5000,"stats":{"domain_count":1035,"status_count":7,"user_count":1},"thumbnail":"https://soc.hyena.network/instance/thumbnail.jpeg","title":"HyNET Social","upload_limit":16000000,"uri":"https://soc.hyena.network","urls":{"streaming_api":"wss://soc.hyena.network"},"version":"2.7.2 (compatible; Akkoma 3.0.0)"}"#; + const ASONIX_INSTANCE: &str = r#"{"uri":"masto.asonix.dog","title":"asonix.dog","short_description":"The asonix of furry mastodon. For me and a few friends. DM me somewhere if u want an account lol","description":"A mastodon server that's only for me and nobody else sorry","email":"asonix@asonix.dog","version":"4.0.0rc2-asonix-changes","urls":{"streaming_api":"wss://masto.asonix.dog"},"stats":{"user_count":7,"status_count":12328,"domain_count":5146},"thumbnail":"https://masto.asonix.dog/system/site_uploads/files/000/000/002/@1x/32f51462a2b2bf2d.png","languages":["dog"],"registrations":false,"approval_required":false,"invites_enabled":false,"configuration":{"accounts":{"max_featured_tags":10},"statuses":{"max_characters":500,"max_media_attachments":4,"characters_reserved_per_url":23},"media_attachments":{"supported_mime_types":["image/jpeg","image/png","image/gif","image/heic","image/heif","image/webp","image/avif","video/webm","video/mp4","video/quicktime","video/ogg","audio/wave","audio/wav","audio/x-wav","audio/x-pn-wave","audio/vnd.wave","audio/ogg","audio/vorbis","audio/mpeg","audio/mp3","audio/webm","audio/flac","audio/aac","audio/m4a","audio/x-m4a","audio/mp4","audio/3gpp","video/x-ms-asf"],"image_size_limit":10485760,"image_matrix_limit":16777216,"video_size_limit":41943040,"video_frame_rate_limit":60,"video_matrix_limit":2304000},"polls":{"max_options":4,"max_characters_per_option":50,"min_expiration":300,"max_expiration":2629746}},"contact_account":{"id":"1","username":"asonix","acct":"asonix","display_name":"Liom on Mane :antiverified:","locked":true,"bot":false,"discoverable":true,"group":false,"created_at":"2021-02-09T00:00:00.000Z","note":"\u003cp\u003e26, local liom, friend, rust (lang) stan, bi \u003c/p\u003e\u003cp\u003eicon by \u003cspan class=\"h-card\"\u003e\u003ca href=\"https://furaffinity.net/user/lalupine\" target=\"blank\" rel=\"noopener noreferrer\" class=\"u-url mention\"\u003e@\u003cspan\u003elalupine@furaffinity.net\u003c/span\u003e\u003c/a\u003e\u003c/span\u003e\u003cbr /\u003eheader by \u003cspan class=\"h-card\"\u003e\u003ca href=\"https://furaffinity.net/user/tronixx\" target=\"blank\" rel=\"noopener noreferrer\" class=\"u-url mention\"\u003e@\u003cspan\u003etronixx@furaffinity.net\u003c/span\u003e\u003c/a\u003e\u003c/span\u003e\u003c/p\u003e\u003cp\u003eTestimonials:\u003c/p\u003e\u003cp\u003eStand: LIONS\u003cbr /\u003eStand User: AODE\u003cbr /\u003e- Keris (not on here)\u003c/p\u003e","url":"https://masto.asonix.dog/@asonix","avatar":"https://masto.asonix.dog/system/accounts/avatars/000/000/001/original/00852df0e6fee7e0.png","avatar_static":"https://masto.asonix.dog/system/accounts/avatars/000/000/001/original/00852df0e6fee7e0.png","header":"https://masto.asonix.dog/system/accounts/headers/000/000/001/original/8122ce3e5a745385.png","header_static":"https://masto.asonix.dog/system/accounts/headers/000/000/001/original/8122ce3e5a745385.png","followers_count":237,"following_count":474,"statuses_count":8798,"last_status_at":"2022-11-08","noindex":true,"emojis":[{"shortcode":"antiverified","url":"https://masto.asonix.dog/system/custom_emojis/images/000/030/053/original/bb0bc2e395b9a127.png","static_url":"https://masto.asonix.dog/system/custom_emojis/images/000/030/053/static/bb0bc2e395b9a127.png","visible_in_picker":true}],"fields":[{"name":"pronouns","value":"he/they","verified_at":null},{"name":"software","value":"bad","verified_at":null},{"name":"gitea","value":"\u003ca href=\"https://git.asonix.dog\" target=\"_blank\" rel=\"nofollow noopener noreferrer me\"\u003e\u003cspan class=\"invisible\"\u003ehttps://\u003c/span\u003e\u003cspan class=\"\"\u003egit.asonix.dog\u003c/span\u003e\u003cspan class=\"invisible\"\u003e\u003c/span\u003e\u003c/a\u003e","verified_at":null},{"name":"join my","value":"relay","verified_at":null}]},"rules":[]}"#; + const HYNET_INSTANCE: &str = r#"{"approval_required":false,"avatar_upload_limit":2000000,"background_image":"https://soc.hyena.network/images/city.jpg","background_upload_limit":4000000,"banner_upload_limit":4000000,"description":"Akkoma: The cooler fediverse server","description_limit":5000,"email":"me@hyena.network","languages":["en"],"max_toot_chars":"5000","pleroma":{"metadata":{"account_activation_required":true,"features":["pleroma_api","mastodon_api","mastodon_api_streaming","polls","v2_suggestions","pleroma_explicit_addressing","shareable_emoji_packs","multifetch","pleroma:api/v1/notifications:include_types_filter","chat","shout","relay","safe_dm_mentions","pleroma_emoji_reactions","pleroma_chat_messages","exposable_reactions","profile_directory","custom_emoji_reactions"],"federation":{"enabled":true,"exclusions":false,"mrf_hashtag":{"federated_timeline_removal":[],"reject":[],"sensitive":["nsfw"]},"mrf_policies":["SimplePolicy","EnsureRePrepended","HashtagPolicy"],"mrf_simple":{"accept":[],"avatar_removal":[],"banner_removal":[],"federated_timeline_removal":["botsin.space"],"followers_only":[],"media_nsfw":["mstdn.jp","wxw.moe","knzk.me","vipgirlfriend.xxx","humblr.social","switter.at","kinkyelephant.com","sinblr.com","kinky.business","rubber.social"],"media_removal":[],"reject":["*.10minutepleroma.com","101010.pl","13bells.com","2.distsn.org","2hu.club","2ndamendment.social","434.earth","4chan.icu","4qq.org","7td.org","80percent.social","a.nti.social","aaathats3as.com","accela.online","amala.schwartzwelt.xyz","angrytoday.com","anime.website","antitwitter.moe","antivaxxer.icu","archivefedifor.fun","artalley.social","bae.st","bajax.us","baraag.net","bbs.kawa-kun.com","beefyboys.club","beefyboys.win","bikeshed.party","bitcoinhackers.org","bleepp.com","blovice.bahnhof.cz","brighteon.social","buildthatwallandmakeamericagreatagain.trumpislovetrumpis.life","bungle.online","cawfee.club","censorship.icu","chungus.cc","club.darknight-coffee.org","clubcyberia.co","cock.fish","cock.li","comfyboy.club","contrapointsfan.club","coon.town","counter.social","cum.salon","d-fens.systems","definitely-not-archivefedifor.fun","degenerates.fail","desuposter.club","detroitriotcity.com","developer.gab.com","dogwhipping.day","eientei.org","enigmatic.observer","eveningzoo.club","exited.eu","federation.krowverse.services","fedi.cc","fedi.krowverse.services","fedi.pawlicker.com","fedi.vern.cc","freak.university","freeatlantis.com","freecumextremist.com","freesoftwareextremist.com","freespeech.firedragonstudios.com","freespeech.host","freespeechextremist.com","freevoice.space","freezepeach.xyz","froth.zone","fuckgov.org","gab.ai","gab.polaris-1.work","gab.protohype.net","gabfed.com","gameliberty.club","gearlandia.haus","gitmo.life","glindr.org","glittersluts.xyz","glowers.club","godspeed.moe","gorf.pub","goyim.app","gs.kawa-kun.com","hagra.net","hallsofamenti.io","hayu.sh","hentai.baby","honkwerx.tech","hunk.city","husk.site","iddqd.social","ika.moe","isexychat.space","jaeger.website","justicewarrior.social","kag.social","katiehopkinspolitical.icu","kiwifarms.cc","kiwifarms.is","kiwifarms.net","kohrville.net","koyu.space","kys.moe","lain.com","lain.sh","leafposter.club","lets.saynoto.lgbt","liberdon.com","libertarianism.club","ligma.pro","lolis.world","masochi.st","masthead.social","mastodon.digitalsuccess.dev","mastodon.fidonet.io","mastodon.grin.hu","mastodon.ml","midnightride.rs","milker.cafe","mobile.tmediatech.io","moon.holiday","mstdn.foxfam.club","mstdn.io","mstdn.starnix.network","mulmeyun.church","nazi.social","neckbeard.xyz","neenster.org","neko.ci","netzsphaere.xyz","newjack.city","nicecrew.digital","nnia.space","noagendasocial.com","norrebro.space","oursocialism.today","ovo.sc","pawoo.net","paypig.org","pedo.school","phreedom.tk","pieville.net","pkteerium.xyz","pl.murky.club","pl.spiderden.net","pl.tkammer.de","pl.zombiecats.run","pleroma.nobodyhasthe.biz","pleroma.runfox.tk","pleroma.site","plr.inferencium.net","pmth.us","poa.st","pod.vladtepesblog.com","political.icu","pooper.social","posting.lolicon.rocks","preteengirls.biz","prout.social","qoto.org","rage.lol","rakket.app","raplst.town","rdrama.cc","ryona.agency","s.sneak.berlin","seal.cafe","sealion.club","search.fedi.app","sementerrori.st","shitposter.club","shortstackran.ch","silkhe.art","sleepy.cafe","soc.mahodou.moe","soc.redeyes.site","social.076.ne.jp","social.anoxinon.de","social.chadland.net","social.freetalklive.com","social.getgle.org","social.handholding.io","social.headsca.la","social.imirhil.fr","social.lovingexpressions.net","social.manalejandro.com","social.midwaytrades.com","social.pseudo-whiskey.bar","social.targaryen.house","social.teci.world","societal.co","society.oftrolls.com","socks.pinnoto.org","socnet.supes.com","solagg.com","spinster.xyz","springbo.cc","stereophonic.space","sunshinegardens.org","theautisticinvestors.quest","thechad.zone","theduran.icu","theosis.church","toot.love","toots.alirezahayati.com","traboone.com","truthsocial.co.in","truthsocial.com","tuusin.misono-ya.info","tweety.icu","unbound.social","unsafe.space","varishangout.net","video.nobodyhasthe.biz","voicenews.icu","voluntaryism.club","waifu.social","weeaboo.space","whinge.town","wolfgirl.bar","workers.dev","wurm.host","xiii.ch","xn--p1abe3d.xn--80asehdb","yggdrasil.social","youjo.love"],"reject_deletes":[],"report_removal":[]},"mrf_simple_info":{"federated_timeline_removal":{"botsin.space":{"reason":"A lot of bot content"}},"media_nsfw":{"humblr.social":{"reason":"NSFW Instance, safe to assume most content is NSFW"},"kinky.business":{"reason":"NSFW Instance, safe to assume most content is NSFW"},"kinkyelephant.com":{"reason":"NSFW Instance, safe to assume most content is NSFW"},"knzk.me":{"reason":"Unmarked nsfw media"},"mstdn.jp":{"reason":"Not sure about the media policy"},"rubber.social":{"reason":"NSFW Instance, safe to assume most content is NSFW"},"sinblr.com":{"reason":"NSFW Instance, safe to assume most content is NSFW"},"switter.at":{"reason":"NSFW Instance, safe to assume most content is NSFW"},"vipgirlfriend.xxx":{"reason":"Unmarked nsfw media"},"wxw.moe":{"reason":"Unmarked nsfw media"}}},"quarantined_instances":[],"quarantined_instances_info":{"quarantined_instances":{}}},"fields_limits":{"max_fields":10,"max_remote_fields":20,"name_length":512,"value_length":2048},"post_formats":["text/plain","text/html","text/markdown","text/bbcode","text/x.misskeymarkdown"],"privileged_staff":false},"stats":{"mau":1},"vapid_public_key":"BMg4q-rT3rkMzc29F7OS5uM6t-Rx4HncMIB1NXrKwNlVRfX-W1kwgOuq5pDy-WhWmOZudaegftjBTCX3-pzdDFc"},"poll_limits":{"max_expiration":31536000,"max_option_chars":200,"max_options":20,"min_expiration":0},"registrations":"FALSE","shout_limit":5000,"stats":{"domain_count":1035,"status_count":7,"user_count":1},"thumbnail":"https://soc.hyena.network/instance/thumbnail.jpeg","title":"HyNET Social","upload_limit":16000000,"uri":"https://soc.hyena.network","urls":{"streaming_api":"wss://soc.hyena.network"},"version":"2.7.2 (compatible; Akkoma 3.0.0)"}"#; #[test] fn deser_masto_instance_with_contact() { diff --git a/src/jobs/record_last_online.rs b/src/jobs/record_last_online.rs new file mode 100644 index 0000000..3c81b31 --- /dev/null +++ b/src/jobs/record_last_online.rs @@ -0,0 +1,28 @@ +use crate::{error::Error, jobs::JobState}; +use background_jobs::{ActixJob, Backoff}; +use std::{future::Future, pin::Pin}; + +#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)] +pub(crate) struct RecordLastOnline; + +impl RecordLastOnline { + #[tracing::instrument(skip(state))] + async fn perform(self, state: JobState) -> Result<(), Error> { + let nodes = state.state.last_online.take(); + + state.state.db.mark_last_seen(nodes).await + } +} + +impl ActixJob for RecordLastOnline { + type State = JobState; + type Future = Pin>>>; + + const NAME: &'static str = "relay::jobs::RecordLastOnline"; + const QUEUE: &'static str = "maintenance"; + const BACKOFF: Backoff = Backoff::Linear(1); + + fn run(self, state: Self::State) -> Self::Future { + Box::pin(async move { self.perform(state).await.map_err(Into::into) }) + } +} diff --git a/src/main.rs b/src/main.rs index a428991..39bba1d 100644 --- a/src/main.rs +++ b/src/main.rs @@ -2,10 +2,13 @@ #![allow(clippy::needless_borrow)] use activitystreams::iri_string::types::IriString; +use actix_rt::task::JoinHandle; use actix_web::{middleware::Compress, web, App, HttpServer}; -use collector::MemoryCollector; +use collector::{DoubleRecorder, MemoryCollector}; #[cfg(feature = "console")] use console_subscriber::ConsoleLayer; +use http_signature_normalization_actix::middleware::VerifySignature; +use metrics_exporter_prometheus::PrometheusBuilder; use opentelemetry::{sdk::Resource, KeyValue}; use opentelemetry_otlp::WithExportConfig; use rustls::ServerConfig; @@ -35,7 +38,7 @@ use self::{ data::{ActorCache, MediaCache, State}, db::Db, jobs::create_workers, - middleware::{DebugPayload, RelayResolver, Timings}, + middleware::{DebugPayload, MyVerify, RelayResolver, Timings}, routes::{actor, inbox, index, nodeinfo, nodeinfo_meta, statics}, }; @@ -94,19 +97,31 @@ fn init_subscriber( Ok(()) } -fn main() -> Result<(), anyhow::Error> { +#[actix_rt::main] +async fn main() -> Result<(), anyhow::Error> { dotenv::dotenv().ok(); let config = Config::build()?; init_subscriber(Config::software_name(), config.opentelemetry_url())?; - let collector = MemoryCollector::new(); - collector.install()?; let args = Args::new(); if args.any() { - return client_main(config, args); + return client_main(config, args).await?; + } + + let collector = MemoryCollector::new(); + + if let Some(bind_addr) = config.prometheus_bind_address() { + let (recorder, exporter) = PrometheusBuilder::new() + .with_http_listener(bind_addr) + .build()?; + + actix_rt::spawn(exporter); + DoubleRecorder::new(recorder, collector.clone()).install()?; + } else { + collector.install()?; } tracing::warn!("Opening DB"); @@ -116,16 +131,15 @@ fn main() -> Result<(), anyhow::Error> { let actors = ActorCache::new(db.clone()); let media = MediaCache::new(db.clone()); - server_main(db, actors, media, collector, config)?; + server_main(db, actors, media, collector, config).await??; tracing::warn!("Application exit"); Ok(()) } -#[actix_rt::main] -async fn client_main(config: Config, args: Args) -> Result<(), anyhow::Error> { - actix_rt::spawn(do_client_main(config, args)).await? +fn client_main(config: Config, args: Args) -> JoinHandle> { + actix_rt::spawn(do_client_main(config, args)) } async fn do_client_main(config: Config, args: Args) -> Result<(), anyhow::Error> { @@ -142,6 +156,39 @@ async fn do_client_main(config: Config, args: Args) -> Result<(), anyhow::Error> println!("Updated lists"); } + if args.contacted() { + let last_seen = admin::client::last_seen(&client, &config).await?; + + let mut report = String::from("Contacted:"); + + if !last_seen.never.is_empty() { + report += "\nNever seen:\n"; + } + + for domain in last_seen.never { + report += "\t"; + report += &domain; + report += "\n"; + } + + if !last_seen.last_seen.is_empty() { + report += "\nSeen:\n"; + } + + for (datetime, domains) in last_seen.last_seen { + for domain in domains { + report += "\t"; + report += &datetime.to_string(); + report += " - "; + report += &domain; + report += "\n"; + } + } + + report += "\n"; + println!("{report}"); + } + if args.list() { let (blocked, allowed, connected) = tokio::try_join!( admin::client::blocked(&client, &config), @@ -174,15 +221,14 @@ async fn do_client_main(config: Config, args: Args) -> Result<(), anyhow::Error> Ok(()) } -#[actix_rt::main] -async fn server_main( +fn server_main( db: Db, actors: ActorCache, media: MediaCache, collector: MemoryCollector, config: Config, -) -> Result<(), anyhow::Error> { - actix_rt::spawn(do_server_main(db, actors, media, collector, config)).await? +) -> JoinHandle> { + actix_rt::spawn(do_server_main(db, actors, media, collector, config)) } async fn do_server_main( @@ -232,10 +278,9 @@ async fn do_server_main( .service( web::resource("/inbox") .wrap(config.digest_middleware()) - .wrap(config.signature_middleware( - state.requests(&config), - actors.clone(), - state.clone(), + .wrap(VerifySignature::new( + MyVerify(state.requests(&config), actors.clone(), state.clone()), + Default::default(), )) .wrap(DebugPayload(config.debug())) .route(web::post().to(inbox)), @@ -258,7 +303,8 @@ async fn do_server_main( .route("/allowed", web::get().to(admin::routes::allowed)) .route("/blocked", web::get().to(admin::routes::blocked)) .route("/connected", web::get().to(admin::routes::connected)) - .route("/stats", web::get().to(admin::routes::stats)), + .route("/stats", web::get().to(admin::routes::stats)) + .route("/last_seen", web::get().to(admin::routes::last_seen)), ), ) }); diff --git a/src/middleware/verifier.rs b/src/middleware/verifier.rs index 21d5b06..a982563 100644 --- a/src/middleware/verifier.rs +++ b/src/middleware/verifier.rs @@ -65,11 +65,21 @@ impl MyVerify { actor_id } else { - self.0 + match self + .0 .fetch::(public_key_id.as_str()) - .await? - .actor_id() - .ok_or(ErrorKind::MissingId)? + .await + { + Ok(res) => res.actor_id().ok_or(ErrorKind::MissingId), + Err(e) => { + if e.is_gone() { + tracing::warn!("Actor gone: {}", public_key_id); + return Ok(false); + } else { + return Err(e); + } + } + }? }; // Previously we verified the sig from an actor's local cache @@ -159,10 +169,10 @@ mod tests { use crate::apub::AcceptedActors; use rsa::{pkcs8::DecodePublicKey, RsaPublicKey}; - const ASONIX_DOG_ACTOR: &'static str = r#"{"@context":["https://www.w3.org/ns/activitystreams","https://w3id.org/security/v1",{"manuallyApprovesFollowers":"as:manuallyApprovesFollowers","toot":"http://joinmastodon.org/ns#","featured":{"@id":"toot:featured","@type":"@id"},"featuredTags":{"@id":"toot:featuredTags","@type":"@id"},"alsoKnownAs":{"@id":"as:alsoKnownAs","@type":"@id"},"movedTo":{"@id":"as:movedTo","@type":"@id"},"schema":"http://schema.org#","PropertyValue":"schema:PropertyValue","value":"schema:value","discoverable":"toot:discoverable","Device":"toot:Device","Ed25519Signature":"toot:Ed25519Signature","Ed25519Key":"toot:Ed25519Key","Curve25519Key":"toot:Curve25519Key","EncryptedMessage":"toot:EncryptedMessage","publicKeyBase64":"toot:publicKeyBase64","deviceId":"toot:deviceId","claim":{"@type":"@id","@id":"toot:claim"},"fingerprintKey":{"@type":"@id","@id":"toot:fingerprintKey"},"identityKey":{"@type":"@id","@id":"toot:identityKey"},"devices":{"@type":"@id","@id":"toot:devices"},"messageFranking":"toot:messageFranking","messageType":"toot:messageType","cipherText":"toot:cipherText","suspended":"toot:suspended"}],"id":"https://masto.asonix.dog/actor","type":"Application","inbox":"https://masto.asonix.dog/actor/inbox","outbox":"https://masto.asonix.dog/actor/outbox","preferredUsername":"masto.asonix.dog","url":"https://masto.asonix.dog/about/more?instance_actor=true","manuallyApprovesFollowers":true,"publicKey":{"id":"https://masto.asonix.dog/actor#main-key","owner":"https://masto.asonix.dog/actor","publicKeyPem":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAx8zXS0QNg9YGUBsxAOBH\nJaxIn7i6t+Z4UOpSFDVa2kP0NvQgIJsq3wzRqvaiuncRWpkyFk1fTakiRGD32xnY\nt+juuAaIBlU8eswKyANFqhcLAvFHmT3rA1848M4/YM19djvlL/PR9T53tPNHU+el\nS9MlsG3o6Zsj8YaUJtCI8RgEuJoROLHUb/V9a3oMQ7CfuIoSvF3VEz3/dRT09RW6\n0wQX7yhka9WlKuayWLWmTcB9lAIX6neBk+qKc8VSEsO7mHkzB8mRgVcS2uYZl1eA\nD8/jTT+SlpeFNDZk0Oh35GNFoOxh9qjRw3NGxu7jJCVBehDODzasOv4xDxKAhONa\njQIDAQAB\n-----END PUBLIC KEY-----\n"},"endpoints":{"sharedInbox":"https://masto.asonix.dog/inbox"}}"#; - const KARJALAZET_RELAY: &'static str = r#"{"@context":["https://www.w3.org/ns/activitystreams","https://pleroma.karjalazet.se/schemas/litepub-0.1.jsonld",{"@language":"und"}],"alsoKnownAs":[],"attachment":[],"capabilities":{},"discoverable":false,"endpoints":{"oauthAuthorizationEndpoint":"https://pleroma.karjalazet.se/oauth/authorize","oauthRegistrationEndpoint":"https://pleroma.karjalazet.se/api/v1/apps","oauthTokenEndpoint":"https://pleroma.karjalazet.se/oauth/token","sharedInbox":"https://pleroma.karjalazet.se/inbox","uploadMedia":"https://pleroma.karjalazet.se/api/ap/upload_media"},"featured":"https://pleroma.karjalazet.se/relay/collections/featured","followers":"https://pleroma.karjalazet.se/relay/followers","following":"https://pleroma.karjalazet.se/relay/following","id":"https://pleroma.karjalazet.se/relay","inbox":"https://pleroma.karjalazet.se/relay/inbox","manuallyApprovesFollowers":false,"name":null,"outbox":"https://pleroma.karjalazet.se/relay/outbox","preferredUsername":"relay","publicKey":{"id":"https://pleroma.karjalazet.se/relay#main-key","owner":"https://pleroma.karjalazet.se/relay","publicKeyPem":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAucoyCht6QpEzUPdQWP/J\nJYxObSH3MCcXBnG4d0OX78QshloeAHhl78EZ5c8I0ePmIjDg2NFK3/pG0EvSrHe2\nIZHnHaN5emgCb2ifNya5W572yfQXo1tUQy+ZXtbTUA7BWbr4LuCvd+HUavMwbx72\neraSZTiQj//ObwpbXFoZO5I/+e5avGmVnfmr/y2cG95hqFDtI3438RgZyBjY5kJM\nY1MLWoY9itGSfYmBtxRj3umlC2bPuBB+hHUJi6TvP7NO6zuUZ66m4ETyuBDi8iP6\ngnUp3Q4+1/I3nDUmhjt7OXckUcX3r5M4UHD3VVUFG0aZw6WWMEAxlyFf/07fCkhR\nBwIDAQAB\n-----END PUBLIC KEY-----\n\n"},"summary":"","tag":[],"type":"Person","url":"https://pleroma.karjalazet.se/relay"}"#; - const ASONIX_DOG_KEY: &'static str = "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAx8zXS0QNg9YGUBsxAOBH\nJaxIn7i6t+Z4UOpSFDVa2kP0NvQgIJsq3wzRqvaiuncRWpkyFk1fTakiRGD32xnY\nt+juuAaIBlU8eswKyANFqhcLAvFHmT3rA1848M4/YM19djvlL/PR9T53tPNHU+el\nS9MlsG3o6Zsj8YaUJtCI8RgEuJoROLHUb/V9a3oMQ7CfuIoSvF3VEz3/dRT09RW6\n0wQX7yhka9WlKuayWLWmTcB9lAIX6neBk+qKc8VSEsO7mHkzB8mRgVcS2uYZl1eA\nD8/jTT+SlpeFNDZk0Oh35GNFoOxh9qjRw3NGxu7jJCVBehDODzasOv4xDxKAhONa\njQIDAQAB\n-----END PUBLIC KEY-----\n"; - const KARJALAZET_KEY: &'static str = "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAucoyCht6QpEzUPdQWP/J\nJYxObSH3MCcXBnG4d0OX78QshloeAHhl78EZ5c8I0ePmIjDg2NFK3/pG0EvSrHe2\nIZHnHaN5emgCb2ifNya5W572yfQXo1tUQy+ZXtbTUA7BWbr4LuCvd+HUavMwbx72\neraSZTiQj//ObwpbXFoZO5I/+e5avGmVnfmr/y2cG95hqFDtI3438RgZyBjY5kJM\nY1MLWoY9itGSfYmBtxRj3umlC2bPuBB+hHUJi6TvP7NO6zuUZ66m4ETyuBDi8iP6\ngnUp3Q4+1/I3nDUmhjt7OXckUcX3r5M4UHD3VVUFG0aZw6WWMEAxlyFf/07fCkhR\nBwIDAQAB\n-----END PUBLIC KEY-----\n\n"; + const ASONIX_DOG_ACTOR: &str = r#"{"@context":["https://www.w3.org/ns/activitystreams","https://w3id.org/security/v1",{"manuallyApprovesFollowers":"as:manuallyApprovesFollowers","toot":"http://joinmastodon.org/ns#","featured":{"@id":"toot:featured","@type":"@id"},"featuredTags":{"@id":"toot:featuredTags","@type":"@id"},"alsoKnownAs":{"@id":"as:alsoKnownAs","@type":"@id"},"movedTo":{"@id":"as:movedTo","@type":"@id"},"schema":"http://schema.org#","PropertyValue":"schema:PropertyValue","value":"schema:value","discoverable":"toot:discoverable","Device":"toot:Device","Ed25519Signature":"toot:Ed25519Signature","Ed25519Key":"toot:Ed25519Key","Curve25519Key":"toot:Curve25519Key","EncryptedMessage":"toot:EncryptedMessage","publicKeyBase64":"toot:publicKeyBase64","deviceId":"toot:deviceId","claim":{"@type":"@id","@id":"toot:claim"},"fingerprintKey":{"@type":"@id","@id":"toot:fingerprintKey"},"identityKey":{"@type":"@id","@id":"toot:identityKey"},"devices":{"@type":"@id","@id":"toot:devices"},"messageFranking":"toot:messageFranking","messageType":"toot:messageType","cipherText":"toot:cipherText","suspended":"toot:suspended"}],"id":"https://masto.asonix.dog/actor","type":"Application","inbox":"https://masto.asonix.dog/actor/inbox","outbox":"https://masto.asonix.dog/actor/outbox","preferredUsername":"masto.asonix.dog","url":"https://masto.asonix.dog/about/more?instance_actor=true","manuallyApprovesFollowers":true,"publicKey":{"id":"https://masto.asonix.dog/actor#main-key","owner":"https://masto.asonix.dog/actor","publicKeyPem":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAx8zXS0QNg9YGUBsxAOBH\nJaxIn7i6t+Z4UOpSFDVa2kP0NvQgIJsq3wzRqvaiuncRWpkyFk1fTakiRGD32xnY\nt+juuAaIBlU8eswKyANFqhcLAvFHmT3rA1848M4/YM19djvlL/PR9T53tPNHU+el\nS9MlsG3o6Zsj8YaUJtCI8RgEuJoROLHUb/V9a3oMQ7CfuIoSvF3VEz3/dRT09RW6\n0wQX7yhka9WlKuayWLWmTcB9lAIX6neBk+qKc8VSEsO7mHkzB8mRgVcS2uYZl1eA\nD8/jTT+SlpeFNDZk0Oh35GNFoOxh9qjRw3NGxu7jJCVBehDODzasOv4xDxKAhONa\njQIDAQAB\n-----END PUBLIC KEY-----\n"},"endpoints":{"sharedInbox":"https://masto.asonix.dog/inbox"}}"#; + const KARJALAZET_RELAY: &str = r#"{"@context":["https://www.w3.org/ns/activitystreams","https://pleroma.karjalazet.se/schemas/litepub-0.1.jsonld",{"@language":"und"}],"alsoKnownAs":[],"attachment":[],"capabilities":{},"discoverable":false,"endpoints":{"oauthAuthorizationEndpoint":"https://pleroma.karjalazet.se/oauth/authorize","oauthRegistrationEndpoint":"https://pleroma.karjalazet.se/api/v1/apps","oauthTokenEndpoint":"https://pleroma.karjalazet.se/oauth/token","sharedInbox":"https://pleroma.karjalazet.se/inbox","uploadMedia":"https://pleroma.karjalazet.se/api/ap/upload_media"},"featured":"https://pleroma.karjalazet.se/relay/collections/featured","followers":"https://pleroma.karjalazet.se/relay/followers","following":"https://pleroma.karjalazet.se/relay/following","id":"https://pleroma.karjalazet.se/relay","inbox":"https://pleroma.karjalazet.se/relay/inbox","manuallyApprovesFollowers":false,"name":null,"outbox":"https://pleroma.karjalazet.se/relay/outbox","preferredUsername":"relay","publicKey":{"id":"https://pleroma.karjalazet.se/relay#main-key","owner":"https://pleroma.karjalazet.se/relay","publicKeyPem":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAucoyCht6QpEzUPdQWP/J\nJYxObSH3MCcXBnG4d0OX78QshloeAHhl78EZ5c8I0ePmIjDg2NFK3/pG0EvSrHe2\nIZHnHaN5emgCb2ifNya5W572yfQXo1tUQy+ZXtbTUA7BWbr4LuCvd+HUavMwbx72\neraSZTiQj//ObwpbXFoZO5I/+e5avGmVnfmr/y2cG95hqFDtI3438RgZyBjY5kJM\nY1MLWoY9itGSfYmBtxRj3umlC2bPuBB+hHUJi6TvP7NO6zuUZ66m4ETyuBDi8iP6\ngnUp3Q4+1/I3nDUmhjt7OXckUcX3r5M4UHD3VVUFG0aZw6WWMEAxlyFf/07fCkhR\nBwIDAQAB\n-----END PUBLIC KEY-----\n\n"},"summary":"","tag":[],"type":"Person","url":"https://pleroma.karjalazet.se/relay"}"#; + const ASONIX_DOG_KEY: &str = "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAx8zXS0QNg9YGUBsxAOBH\nJaxIn7i6t+Z4UOpSFDVa2kP0NvQgIJsq3wzRqvaiuncRWpkyFk1fTakiRGD32xnY\nt+juuAaIBlU8eswKyANFqhcLAvFHmT3rA1848M4/YM19djvlL/PR9T53tPNHU+el\nS9MlsG3o6Zsj8YaUJtCI8RgEuJoROLHUb/V9a3oMQ7CfuIoSvF3VEz3/dRT09RW6\n0wQX7yhka9WlKuayWLWmTcB9lAIX6neBk+qKc8VSEsO7mHkzB8mRgVcS2uYZl1eA\nD8/jTT+SlpeFNDZk0Oh35GNFoOxh9qjRw3NGxu7jJCVBehDODzasOv4xDxKAhONa\njQIDAQAB\n-----END PUBLIC KEY-----\n"; + const KARJALAZET_KEY: &str = "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAucoyCht6QpEzUPdQWP/J\nJYxObSH3MCcXBnG4d0OX78QshloeAHhl78EZ5c8I0ePmIjDg2NFK3/pG0EvSrHe2\nIZHnHaN5emgCb2ifNya5W572yfQXo1tUQy+ZXtbTUA7BWbr4LuCvd+HUavMwbx72\neraSZTiQj//ObwpbXFoZO5I/+e5avGmVnfmr/y2cG95hqFDtI3438RgZyBjY5kJM\nY1MLWoY9itGSfYmBtxRj3umlC2bPuBB+hHUJi6TvP7NO6zuUZ66m4ETyuBDi8iP6\ngnUp3Q4+1/I3nDUmhjt7OXckUcX3r5M4UHD3VVUFG0aZw6WWMEAxlyFf/07fCkhR\nBwIDAQAB\n-----END PUBLIC KEY-----\n\n"; #[test] fn handles_masto_keys() { diff --git a/src/requests.rs b/src/requests.rs index 332024a..9a1bc50 100644 --- a/src/requests.rs +++ b/src/requests.rs @@ -1,4 +1,7 @@ -use crate::error::{Error, ErrorKind}; +use crate::{ + data::LastOnline, + error::{Error, ErrorKind}, +}; use activitystreams::iri_string::types::IriString; use actix_web::http::header::Date; use awc::{error::SendRequestError, Client, ClientResponse}; @@ -146,6 +149,7 @@ pub(crate) struct Requests { private_key: RsaPrivateKey, config: Config, breakers: Breakers, + last_online: Arc, } impl std::fmt::Debug for Requests { @@ -174,6 +178,7 @@ impl Requests { private_key: RsaPrivateKey, user_agent: String, breakers: Breakers, + last_online: Arc, ) -> Self { Requests { client: Rc::new(RefCell::new(build_client(&user_agent))), @@ -184,6 +189,7 @@ impl Requests { private_key, config: Config::default().mastodon_compat(), breakers, + last_online, } } @@ -233,6 +239,7 @@ impl Requests { return Err(ErrorKind::Status(parsed_url.to_string(), res.status()).into()); } + self.last_online.mark_seen(&parsed_url); self.breakers.succeed(&parsed_url); Ok(res) diff --git a/src/routes/inbox.rs b/src/routes/inbox.rs index 14286df..133d7dd 100644 --- a/src/routes/inbox.rs +++ b/src/routes/inbox.rs @@ -16,7 +16,8 @@ use activitystreams::{ use actix_web::{web, HttpResponse}; use http_signature_normalization_actix::prelude::{DigestVerified, SignatureVerified}; -#[tracing::instrument(name = "Inbox", skip_all)] +#[tracing::instrument(name = "Inbox", skip_all, fields(id = tracing::field::debug(&input.id_unchecked()), kind = tracing::field::debug(&input.kind())))] +#[allow(clippy::too_many_arguments)] pub(crate) async fn route( state: web::Data, actors: web::Data, @@ -24,22 +25,48 @@ pub(crate) async fn route( client: web::Data, jobs: web::Data, input: web::Json, - verified: Option<(SignatureVerified, DigestVerified)>, + digest_verified: Option, + signature_verified: Option, ) -> Result { let input = input.into_inner(); - let actor = actors - .get( - input.actor()?.as_single_id().ok_or(ErrorKind::MissingId)?, - &client, - ) - .await? - .into_inner(); + let kind = input.kind().ok_or(ErrorKind::MissingKind)?; - let is_allowed = state.db.is_allowed(actor.id.clone()); - let is_connected = state.db.is_connected(actor.id.clone()); + if digest_verified.is_some() && signature_verified.is_none() && *kind == ValidTypes::Delete { + return Ok(accepted(serde_json::json!({}))); + } else if config.validate_signatures() + && (digest_verified.is_none() || signature_verified.is_none()) + { + return Err(ErrorKind::NoSignature(None).into()); + } - let (is_allowed, is_connected) = tokio::try_join!(is_allowed, is_connected)?; + let actor_id = if input.id_unchecked().is_some() { + input.actor()?.as_single_id().ok_or(ErrorKind::MissingId)? + } else { + input + .actor_unchecked() + .as_single_id() + .ok_or(ErrorKind::MissingId)? + }; + + let actor = actors.get(actor_id, &client).await?.into_inner(); + + if let Some(verified) = signature_verified { + if actor.public_key_id.as_str() != verified.key_id() { + tracing::error!("Actor signed with wrong key"); + return Err(ErrorKind::BadActor( + actor.public_key_id.to_string(), + verified.key_id().to_owned(), + ) + .into()); + } + } else if config.validate_signatures() { + tracing::error!("This case should never be reachable, since I handle signature checks earlier in the flow. If you see this in a log it means I did it wrong"); + return Err(ErrorKind::NoSignature(Some(actor.public_key_id.to_string())).into()); + } + + let is_allowed = state.db.is_allowed(actor.id.clone()).await?; + let is_connected = state.db.is_connected(actor.id.clone()).await?; if !is_allowed { return Err(ErrorKind::NotAllowed(actor.id.to_string()).into()); @@ -49,29 +76,16 @@ pub(crate) async fn route( return Err(ErrorKind::NotSubscribed(actor.id.to_string()).into()); } - if config.validate_signatures() && verified.is_none() { - return Err(ErrorKind::NoSignature(actor.public_key_id.to_string()).into()); - } else if config.validate_signatures() { - if let Some((verified, _)) = verified { - if actor.public_key_id.as_str() != verified.key_id() { - tracing::error!("Bad actor, more info: {:?}", input); - return Err(ErrorKind::BadActor( - actor.public_key_id.to_string(), - verified.key_id().to_owned(), - ) - .into()); - } - } - } - - match input.kind().ok_or(ErrorKind::MissingKind)? { + match kind { ValidTypes::Accept => handle_accept(&config, input).await?, ValidTypes::Reject => handle_reject(&config, &jobs, input, actor).await?, ValidTypes::Announce | ValidTypes::Create => { handle_announce(&state, &jobs, input, actor).await? } ValidTypes::Follow => handle_follow(&config, &jobs, input, actor).await?, - ValidTypes::Delete | ValidTypes::Update => handle_forward(&jobs, input, actor).await?, + ValidTypes::Add | ValidTypes::Delete | ValidTypes::Remove | ValidTypes::Update => { + handle_forward(&jobs, input, actor).await? + } ValidTypes::Undo => handle_undo(&config, &jobs, input, actor, is_connected).await?, }; @@ -203,7 +217,7 @@ async fn handle_announce( .as_single_id() .ok_or(ErrorKind::MissingId)?; - if state.is_cached(object_id).await { + if state.is_cached(object_id) { return Err(ErrorKind::Duplicate.into()); } diff --git a/src/routes/nodeinfo.rs b/src/routes/nodeinfo.rs index 18b1c3a..a504496 100644 --- a/src/routes/nodeinfo.rs +++ b/src/routes/nodeinfo.rs @@ -24,18 +24,18 @@ struct Links { links: Vec, } -#[tracing::instrument(name = "NodeInfo")] +#[tracing::instrument(name = "NodeInfo", skip_all)] pub(crate) async fn route( config: web::Data, state: web::Data, ) -> web::Json { - let (inboxes, blocks) = tokio::join!(state.db.inboxes(), async { - if config.publish_blocks() { - Some(state.db.blocks().await.unwrap_or_default()) - } else { - None - } - }); + let inboxes = state.db.inboxes().await; + + let blocks = if config.publish_blocks() { + Some(state.db.blocks().await.unwrap_or_default()) + } else { + None + }; let peers = inboxes .unwrap_or_default() diff --git a/src/routes/statics.rs b/src/routes/statics.rs index 31e37c1..6c7788a 100644 --- a/src/routes/statics.rs +++ b/src/routes/statics.rs @@ -5,7 +5,7 @@ use actix_web::{ }; #[allow(clippy::async_yields_async)] -#[tracing::instrument(name = "Statistics")] +#[tracing::instrument(name = "Statics")] pub(crate) async fn route(filename: web::Path) -> HttpResponse { if let Some(data) = StaticFile::get(&filename.into_inner()) { HttpResponse::Ok() diff --git a/systemd/example-relay.service b/systemd/example-relay.service new file mode 100644 index 0000000..3e6f7ed --- /dev/null +++ b/systemd/example-relay.service @@ -0,0 +1,15 @@ +[Unit] +Description=Activitypub Relay +Documentation=https://git.asonix.dog/asonix/relay +Wants=network.target +After=network.target + +[Install] +WantedBy=multi-user.target + +[Service] +Type=simple +EnvironmentFile=/etc/systemd/system/example-relay.service.env +ExecStart=/path/to/relay +Restart=always + diff --git a/systemd/example-relay.service.env b/systemd/example-relay.service.env new file mode 100644 index 0000000..e74a6a0 --- /dev/null +++ b/systemd/example-relay.service.env @@ -0,0 +1,19 @@ +HOSTNAME='relay.example.com' +ADDR='0.0.0.0' +PORT='8080' +RESTRICTED_MODE='true' +VALIDATE_SIGNATURES='true' +HTTPS='true' +PRETTY_LOG='false' +PUBLISH_BLOCKS='true' +DEBUG='false' +SLED_PATH='/opt/sled' +TELEGRAM_ADMIN_HANDLE='myhandle' +RUST_BACKTRACE='full' +FOOTER_BLURB='Contact @example for inquiries.' +LOCAL_DOMAINS='masto.example.com' +LOCAL_BLURB='

An ActivityPub relay for servers. Currently running somewhere. Let me know if you want to join!

' +OPENTELEMETRY_URL='http://otel.example.com:4317' +API_TOKEN='blahblahblahblahblahblahblah' +TELEGRAM_TOKEN='blahblahblahblahblahblahblah' + diff --git a/systemd/example-relay.socket b/systemd/example-relay.socket new file mode 100644 index 0000000..9c6ea0b --- /dev/null +++ b/systemd/example-relay.socket @@ -0,0 +1,11 @@ +[Unit] +Description=Activitypub Relay Socket +Before=multi-user.target +After=network.target + +[Socket] +Service=example-relay.service +ListenStream=8080 + +[Install] +WantedBy=sockets.target diff --git a/templates/admin.rs.html b/templates/admin.rs.html index 1af6bc2..e457ce7 100644 --- a/templates/admin.rs.html +++ b/templates/admin.rs.html @@ -6,7 +6,7 @@
- @contact.display_name's avatar + @contact.display_name's avatar